mirror of
https://github.com/github/awesome-copilot.git
synced 2026-09-07 16:25:47 +00:00
chore: publish from main
This commit is contained in:
github-actions[bot]
@@ -0,0 +1,416 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a repository's source files against its tests, so coverage gaps are evidence, not guesses.
|
||||
|
||||
Read-only. Standard library only. Writes nothing. Runs no tests.
|
||||
|
||||
python coverage_map.py # text summary of the current repo
|
||||
python coverage_map.py --repo ../other # a different repo
|
||||
python coverage_map.py --format json # machine-readable
|
||||
python coverage_map.py --top 40 # more rows per section
|
||||
|
||||
Detects the test framework and naming convention, then matches each source file to
|
||||
tests by basename, mirrored path, and — most importantly — by scanning what the test
|
||||
files actually import. The import scan is what makes "no tests found for X" worth
|
||||
reporting instead of merely plausible.
|
||||
|
||||
HEURISTIC. It cannot see coverage through indirection, fixtures, or end-to-end tests
|
||||
that exercise a file without naming it. Treat an `untested` entry as a lead to confirm
|
||||
by hand, never as a finding on its own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
GIT_TIMEOUT = 30
|
||||
MAX_READ_BYTES = 400_000 # skip pathological files rather than stalling
|
||||
|
||||
IGNORED_DIRS = {
|
||||
".git", ".hg", ".svn", "node_modules", "vendor", "venv", ".venv", "env",
|
||||
"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox",
|
||||
"dist", "build", "target", "out", ".next", ".nuxt", ".svelte-kit",
|
||||
".idea", ".vscode", ".gradle", "Pods", ".terraform", "coverage",
|
||||
}
|
||||
|
||||
SOURCE_EXTS = {
|
||||
".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".vue", ".svelte",
|
||||
".go", ".rs", ".rb", ".php", ".java", ".kt", ".swift", ".scala", ".cs", ".ex", ".exs", ".dart",
|
||||
}
|
||||
|
||||
# Files that are configuration, generated, or entry-point glue rather than logic worth testing.
|
||||
NON_LOGIC = re.compile(
|
||||
r"(^|/)(setup|conftest|__init__|index|main|migrations?|__generated__)\.[A-Za-z]+$"
|
||||
r"|\.(config|d)\.[A-Za-z]+$"
|
||||
r"|(^|/)(migrations|__generated__|generated|\.storybook)/",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
TEST_PATH = re.compile(
|
||||
r"(^|/)(tests?|spec|specs|__tests__|e2e|integration|cypress|playwright)(/|$)"
|
||||
r"|(^|/)[^/]*[._-](test|spec)s?\.[A-Za-z0-9]+$"
|
||||
r"|(^|/)test_[^/]*\.[A-Za-z0-9]+$"
|
||||
r"|_test\.[A-Za-z0-9]+$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Framework marker -> label. Searched in manifests and config filenames.
|
||||
FRAMEWORKS = {
|
||||
"vitest": "Vitest", "jest": "Jest", "mocha": "Mocha", "jasmine": "Jasmine",
|
||||
"@playwright/test": "Playwright", "cypress": "Cypress", "ava": "AVA",
|
||||
"@testing-library": "Testing Library", "karma": "Karma", "node:test": "node:test",
|
||||
"pytest": "pytest", "unittest": "unittest", "nose": "nose", "tox": "tox",
|
||||
"rspec": "RSpec", "minitest": "Minitest", "phpunit": "PHPUnit", "pest": "Pest",
|
||||
"junit": "JUnit", "testng": "TestNG", "go test": "go test", "cargo test": "cargo test",
|
||||
"xunit": "xUnit", "nunit": "NUnit", "exunit": "ExUnit",
|
||||
}
|
||||
|
||||
ASSERTION_TOKENS = re.compile(
|
||||
r"\b(expect|assert|assert_|assertEqual|assertTrue|assertRaises|should|"
|
||||
r"toBe|toEqual|toThrow|toHaveBeenCalled|require\.Equal|assert\.|"
|
||||
r"refute|is_a|must_equal|shouldBe|verify)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# Actual test cases only. `describe`, `context` and `class Test` are grouping
|
||||
# constructs, and counting them inflates the case count -- a file with one it()
|
||||
# inside one describe() would report two cases, which then skews the
|
||||
# assertions-per-case ratio below.
|
||||
TEST_CASE_TOKENS = re.compile(
|
||||
r"^\s*(it|test|def test_|func Test|scenario|@Test|it\.each|test\.each)\b",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
RISK_KEYWORDS = (
|
||||
"auth", "login", "session", "token", "password", "permission", "role", "admin",
|
||||
"billing", "payment", "invoice", "subscription", "checkout", "charge", "refund",
|
||||
"migration", "delete", "destroy", "export", "import", "webhook", "crypto", "wallet",
|
||||
"security", "secret", "upload",
|
||||
)
|
||||
|
||||
# Import forms across the languages we handle. Group 1 is always the module reference.
|
||||
IMPORT_RES = [
|
||||
re.compile(r"""(?:from|import)\s+['"]([^'"]+)['"]"""), # JS/TS
|
||||
re.compile(r"""require\(\s*['"]([^'"]+)['"]\s*\)"""), # CJS
|
||||
re.compile(r"""^\s*from\s+([A-Za-z0-9_.]+)\s+import""", re.M), # Python from-import
|
||||
re.compile(r"""^\s*import\s+([A-Za-z0-9_.]+)""", re.M), # Python/Java/Go import
|
||||
re.compile(r"""use\s+([A-Za-z0-9_:]+)"""), # Rust
|
||||
re.compile(r"""require(?:_relative)?\s+['"]([^'"]+)['"]"""), # Ruby
|
||||
]
|
||||
|
||||
warnings: list[str] = []
|
||||
|
||||
|
||||
def run_git(args: list[str], repo: Path) -> str | None:
|
||||
git = shutil.which("git")
|
||||
if git is None:
|
||||
return None
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[git, *args], cwd=str(repo), text=True, timeout=GIT_TIMEOUT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
warnings.append(f"git {' '.join(args)} failed: {exc}")
|
||||
return None
|
||||
return r.stdout if r.returncode == 0 else None
|
||||
|
||||
|
||||
def list_files(repo: Path) -> list[str]:
|
||||
out = run_git(["ls-files", "--cached", "--other", "--exclude-standard"], repo)
|
||||
if out is not None:
|
||||
files = [ln.strip() for ln in out.splitlines() if ln.strip()]
|
||||
if files:
|
||||
return sorted(files)
|
||||
warnings.append("git unavailable or empty index; using a filesystem walk (ignore rules approximated)")
|
||||
files = []
|
||||
for root, dirnames, filenames in os.walk(repo):
|
||||
dirnames[:] = [d for d in dirnames if d not in IGNORED_DIRS and not d.startswith(".")]
|
||||
for fn in filenames:
|
||||
files.append(Path(root, fn).relative_to(repo).as_posix())
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def read(path: Path) -> str:
|
||||
try:
|
||||
if path.stat().st_size > MAX_READ_BYTES:
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def detect_frameworks(repo: Path, files: list[str]) -> list[str]:
|
||||
found: set[str] = set()
|
||||
haystacks: list[str] = []
|
||||
for name in ("package.json", "pyproject.toml", "requirements.txt", "Gemfile",
|
||||
"composer.json", "pom.xml", "build.gradle", "mix.exs", "Cargo.toml"):
|
||||
for rel in [f for f in files if Path(f).name == name][:5]:
|
||||
haystacks.append(read(repo / rel).lower())
|
||||
config_names = " ".join(Path(f).name.lower() for f in files)
|
||||
haystacks.append(config_names)
|
||||
blob = "\n".join(haystacks)
|
||||
for marker, label in FRAMEWORKS.items():
|
||||
m = marker.lower()
|
||||
# Short bare words such as "ava", "nose" and "pest" substring-match inside
|
||||
# "available", "javascript" and so on, so require word boundaries for them.
|
||||
# Markers carrying punctuation ("@playwright/test", "node:test") are already
|
||||
# specific enough, and \b would not behave around those characters anyway.
|
||||
if m.isalnum():
|
||||
if re.search(rf"\b{re.escape(m)}\b", blob):
|
||||
found.add(label)
|
||||
elif m in blob:
|
||||
found.add(label)
|
||||
# Language-implied runners that need no manifest entry.
|
||||
if any(f.endswith("_test.go") for f in files):
|
||||
found.add("go test")
|
||||
if any(f.endswith(".rs") for f in files) and any("#[test]" in read(repo / f) for f in
|
||||
[x for x in files if x.endswith(".rs")][:20]):
|
||||
found.add("cargo test")
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def infer_convention(test_files: list[str]) -> list[str]:
|
||||
patterns: Counter[str] = Counter()
|
||||
for rel in test_files:
|
||||
name = Path(rel).name
|
||||
if re.match(r"^test_.*\.py$", name):
|
||||
patterns["test_*.py"] += 1
|
||||
elif re.search(r"_test\.go$", name):
|
||||
patterns["*_test.go"] += 1
|
||||
elif re.search(r"\.(test|spec)\.[jt]sx?$", name):
|
||||
patterns[f"*.{'test' if '.test.' in name else 'spec'}.[jt]s(x)"] += 1
|
||||
elif re.search(r"_spec\.rb$", name):
|
||||
patterns["*_spec.rb"] += 1
|
||||
elif re.search(r"Test\.(java|kt|cs)$", name):
|
||||
patterns["*Test.{java,kt,cs}"] += 1
|
||||
if "__tests__/" in rel:
|
||||
patterns["__tests__/ directory"] += 1
|
||||
elif re.match(r"^tests?/", rel):
|
||||
patterns["tests/ directory"] += 1
|
||||
elif re.match(r"^spec/", rel):
|
||||
patterns["spec/ directory"] += 1
|
||||
return [f"{p} ({n})" for p, n in patterns.most_common()]
|
||||
|
||||
|
||||
def module_tokens(rel: str) -> set[str]:
|
||||
"""Identifiers by which a test might refer to this source file."""
|
||||
p = Path(rel)
|
||||
stem = p.stem
|
||||
tokens = {stem.lower()}
|
||||
# A component at foo/Button/index.tsx is referred to as "Button".
|
||||
if stem.lower() in ("index", "__init__", "mod"):
|
||||
tokens.add(p.parent.name.lower())
|
||||
tokens.add(rel.lower())
|
||||
tokens.add(p.with_suffix("").as_posix().lower())
|
||||
return {t for t in tokens if t and t not in ("", ".")}
|
||||
|
||||
|
||||
def build(repo: Path, files: list[str]) -> dict:
|
||||
test_files = [f for f in files if TEST_PATH.search(f) and Path(f).suffix in SOURCE_EXTS]
|
||||
test_set = set(test_files)
|
||||
source_files = [
|
||||
f for f in files
|
||||
if Path(f).suffix in SOURCE_EXTS and f not in test_set and not NON_LOGIC.search(f)
|
||||
]
|
||||
|
||||
# Index what the tests import, plus every bare identifier they mention.
|
||||
imported: set[str] = set()
|
||||
mentioned: set[str] = set()
|
||||
weak: list[dict] = []
|
||||
for rel in test_files:
|
||||
text = read(repo / rel)
|
||||
if not text:
|
||||
continue
|
||||
for regex in IMPORT_RES:
|
||||
for m in regex.findall(text):
|
||||
ref = m.strip()
|
||||
imported.add(ref.lower())
|
||||
imported.add(Path(ref).name.lower())
|
||||
imported.add(Path(ref).stem.lower())
|
||||
for part in re.split(r"[./:\\]", ref):
|
||||
if part and part not in (".", "..", "src", "lib", "app"):
|
||||
mentioned.add(part.lower())
|
||||
assertions = len(ASSERTION_TOKENS.findall(text))
|
||||
cases = len(TEST_CASE_TOKENS.findall(text))
|
||||
lines = text.count("\n") + 1
|
||||
# A test file with cases but almost no assertions is usually asserting nothing useful.
|
||||
if cases and assertions <= max(1, cases // 4):
|
||||
weak.append({"path": rel, "test_cases": cases, "assertions": assertions, "lines": lines})
|
||||
|
||||
# Test files whose filename itself marks them as a test, as opposed to files that
|
||||
# merely live under tests/ or e2e/ (fixtures, helpers, page objects, factories).
|
||||
named_test_re = re.compile(r"([._-](test|spec)s?$)|(^test_)|(^test$)", re.IGNORECASE)
|
||||
named_tests = [t for t in test_files if named_test_re.search(Path(t).stem)]
|
||||
|
||||
matched: dict[str, list[str]] = {}
|
||||
untested: list[dict] = []
|
||||
for rel in source_files:
|
||||
tokens = module_tokens(rel)
|
||||
how: list[str] = []
|
||||
|
||||
stem = Path(rel).stem.lower()
|
||||
mirror = Path(rel).with_suffix("").as_posix().lower()
|
||||
|
||||
# 1. a test file whose *own name* carries a test marker and embeds this file's name.
|
||||
# Requiring the marker matters: a helper like e2e/lib/api.mjs is classified as a
|
||||
# test file because of its directory, and would otherwise "cover" every api.js
|
||||
# in the repo. A false match here hides a real gap, so keep this rule strict.
|
||||
for t in named_tests:
|
||||
tl = Path(t).stem.lower()
|
||||
if tl in {f"{stem}test", f"test{stem}", f"{stem}spec", f"{stem}_test",
|
||||
f"test_{stem}", f"{stem}.test", f"{stem}.spec"} or \
|
||||
re.sub(r"[._-]?(test|spec)s?$", "", tl) == stem:
|
||||
how.append(f"name match: {t}")
|
||||
break
|
||||
# 2. mirrored directory layout, e.g. src/a/b.ts -> tests/a/b.test.ts
|
||||
if not how:
|
||||
tail = "/".join(mirror.split("/")[1:]) if "/" in mirror else mirror
|
||||
if tail and any(tail in t.lower() for t in test_files):
|
||||
how.append("mirrored path match")
|
||||
# 3. a test actually imports it
|
||||
if not how and (tokens & imported):
|
||||
how.append("imported by a test")
|
||||
# 4. weakest signal: a test mentions the identifier
|
||||
if not how and stem in mentioned and len(stem) > 3:
|
||||
how.append("mentioned in a test (weak signal)")
|
||||
|
||||
if how:
|
||||
matched[rel] = how
|
||||
else:
|
||||
lines = read(repo / rel).count("\n") + 1
|
||||
hits = [k for k in RISK_KEYWORDS if k in rel.lower()]
|
||||
untested.append({
|
||||
"path": rel,
|
||||
"lines": lines,
|
||||
"dir": str(Path(rel).parent).replace("\\", "/"),
|
||||
"risk_keywords": hits,
|
||||
})
|
||||
|
||||
untested.sort(key=lambda x: (not x["risk_keywords"], -x["lines"]))
|
||||
weak.sort(key=lambda x: -x["test_cases"])
|
||||
|
||||
by_dir: dict[str, dict] = defaultdict(lambda: {"untested": 0, "lines": 0})
|
||||
for u in untested:
|
||||
by_dir[u["dir"]]["untested"] += 1
|
||||
by_dir[u["dir"]]["lines"] += u["lines"]
|
||||
|
||||
return {
|
||||
"repo": str(repo),
|
||||
"frameworks": detect_frameworks(repo, files),
|
||||
"conventions": infer_convention(test_files),
|
||||
"totals": {
|
||||
"source_files": len(source_files),
|
||||
"test_files": len(test_files),
|
||||
"matched": len(matched),
|
||||
"untested": len(untested),
|
||||
"coverage_ratio": round(len(matched) / len(source_files), 3) if source_files else None,
|
||||
},
|
||||
"untested": untested,
|
||||
"untested_by_directory": sorted(
|
||||
({"dir": d, **v} for d, v in by_dir.items()),
|
||||
key=lambda x: -x["untested"],
|
||||
),
|
||||
"weak_tests": weak,
|
||||
"matched_sample": [{"path": p, "why": w} for p, w in list(matched.items())[:15]],
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
def truncate(items: list, limit: int, label: str, out: list[str]) -> list:
|
||||
if len(items) > limit:
|
||||
out.append(f" TRUNCATED: showing top {limit} of {len(items)} {label}")
|
||||
return items[:limit]
|
||||
return items
|
||||
|
||||
|
||||
def render(d: dict, top: int) -> str:
|
||||
t = d["totals"]
|
||||
L = ["# Test Coverage Map", "", f"Repo: {d['repo']}"]
|
||||
ratio = f"{t['coverage_ratio']:.0%}" if t["coverage_ratio"] is not None else "n/a"
|
||||
L.append(f"Source files: {t['source_files']} Test files: {t['test_files']} "
|
||||
f"Matched: {t['matched']} ({ratio}) Unmatched: {t['untested']}")
|
||||
L.append("")
|
||||
L.append(f"Frameworks detected: {', '.join(d['frameworks']) or 'none detected'}")
|
||||
L.append(f"Naming conventions: {', '.join(d['conventions']) or 'none inferred'}")
|
||||
L.append("")
|
||||
|
||||
L.append("## Unmatched source files (risk-flagged first, then largest)")
|
||||
if d["untested"]:
|
||||
for u in truncate(d["untested"], top, "unmatched files", L):
|
||||
flag = f" [risk: {', '.join(u['risk_keywords'])}]" if u["risk_keywords"] else ""
|
||||
L.append(f"- {u['path']}: {u['lines']} lines{flag}")
|
||||
else:
|
||||
L.append("- none; every source file matched at least one test signal")
|
||||
L.append("")
|
||||
|
||||
L.append("## Unmatched by directory")
|
||||
for row in truncate(d["untested_by_directory"], top, "directories", L):
|
||||
L.append(f"- {row['dir']}: {row['untested']} files, {row['lines']} lines")
|
||||
L.append("")
|
||||
|
||||
L.append("## Test files with test cases but few assertions")
|
||||
if d["weak_tests"]:
|
||||
for w in truncate(d["weak_tests"], top, "test files", L):
|
||||
L.append(f"- {w['path']}: {w['test_cases']} cases, {w['assertions']} assertions")
|
||||
else:
|
||||
L.append("- none flagged")
|
||||
L.append("")
|
||||
|
||||
if d["warnings"]:
|
||||
L.append("## Warnings")
|
||||
L.extend(f"- {w}" for w in d["warnings"])
|
||||
L.append("")
|
||||
|
||||
L.append("HEURISTIC MATCHER. An unmatched file is a lead, not a finding: coverage through")
|
||||
L.append("fixtures, end-to-end tests, or indirection is invisible here. Confirm by grepping for")
|
||||
L.append("the module name before reporting anything as P0 or P1.")
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Map source files against tests. Read-only.")
|
||||
ap.add_argument("--repo", default=".", help="Path inside the repository.")
|
||||
ap.add_argument("--format", choices=["text", "json"], default="text", help="Output format.")
|
||||
ap.add_argument("--top", type=int, default=25, help="Rows per section. Default 25.")
|
||||
ap.add_argument(
|
||||
"--no-git-root",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Treat --repo literally instead of expanding to the enclosing git repository root. "
|
||||
"Use this to scope the survey to one package or subdirectory of a monorepo."
|
||||
),
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
repo = Path(args.repo).resolve()
|
||||
if not repo.is_dir():
|
||||
print(f"error: not a directory: {repo}", file=sys.stderr)
|
||||
return 2
|
||||
if not args.no_git_root:
|
||||
root = run_git(["rev-parse", "--show-toplevel"], repo)
|
||||
if root and root.strip():
|
||||
repo = Path(root.strip()).resolve()
|
||||
|
||||
files = list_files(repo)
|
||||
if not files:
|
||||
print(f"error: no files found under {repo}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
data = build(repo, files)
|
||||
print(json.dumps(data, indent=2) if args.format == "json" else render(data, args.top))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user