diff --git a/.gitignore b/.gitignore index 3ce76ade683..9fec2ee4305 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ /lua/output /lua/spec/snapshots/diffs /package-lock.json -/luacov.report.out -/luacov.stats.out +luacov.report.out +luacov.stats.out **/__pycache__/ .env **/*.ck diff --git a/lua/.luacov b/lua/.luacov index 369a7ea5610..bab9bbe1a85 100644 --- a/lua/.luacov +++ b/lua/.luacov @@ -1,8 +1,18 @@ return { configfile = '.luacov', - includeuntestedfiles = {'standard'}, - include = {'standard/*'}, - exclude = {'wikis/*'}, - runreport = false, -- Don't work properly for some reason, have to do a manual `luacov` call afterwards - deletestats = true, + -- Report modules no spec ever loads as 0%, so coverage reflects all of commons + -- rather than only the files the suite happens to touch. + includeuntestedfiles = {'wikis/commons'}, + include = {'wikis/commons/'}, + exclude = { + 'spec/', + 'definitions/', + '3rd/', + 'plugins/', + 'test_assets/', + -- ScribuntoUnit testcases that run on-wiki, not under busted. + 'wikis/commons/test/', + }, + runreport = false, -- Doesn't work properly for some reason, have to do a manual `luacov` call afterwards + deletestats = false, -- Keep stats so the report can be regenerated / inspected after a run } diff --git a/lua/wikis/commons/Widget/Table2/Row.lua b/lua/wikis/commons/Widget/Table2/Row.lua index 31e8ec2bff0..929eb7bf464 100644 --- a/lua/wikis/commons/Widget/Table2/Row.lua +++ b/lua/wikis/commons/Widget/Table2/Row.lua @@ -136,4 +136,4 @@ end return Component.component( Table2Row -) \ No newline at end of file +) diff --git a/scripts/metrics/README.md b/scripts/metrics/README.md new file mode 100644 index 00000000000..0935bb7bc2e --- /dev/null +++ b/scripts/metrics/README.md @@ -0,0 +1,47 @@ +# Metrics + +Scripts that measure the codebase. Run them by hand whenever you want a number. + +Each one prints a table. Add `--csv` to get rows you can append to a file instead. + +## What's here + +| Script | What it tells you | +|-|-| +| `repo_loc.py` | Lua LOC per wiki, in this repo | +| `onwiki_loc.py` | Lua LOC that lives on-wiki and isn't in this repo | +| `deprecated_patterns.py` | Call sites still using legacy patterns | +| `coverage.py` | Test coverage of `lua/wikis/commons` | +| `annotation_coverage.py` | How much of commons has `@param` / `@return` | +| `customization_metrics.py` | How much per-wiki override code there is | + +## Running them + +Most need nothing but Python: + +```bash +python3 scripts/metrics/repo_loc.py +``` + +`coverage.py` runs the test suite, so it needs busted and luacov on Lua 5.1: + +```bash +luarocks install --lua-version=5.1 busted +luarocks install --lua-version=5.1 luacov +python3 scripts/metrics/coverage.py +``` + +`onwiki_loc.py` calls the live wiki API: + +```bash +pip install -r requirements.txt +WIKI_BASE_URL=https://liquipedia.net python3 scripts/metrics/onwiki_loc.py +``` + +## Tracking over time + +```bash +python3 scripts/metrics/repo_loc.py --csv --no-header >> metrics.csv +``` + +Run any script with `--help` for the rest of its options. diff --git a/scripts/metrics/annotation_coverage.py b/scripts/metrics/annotation_coverage.py new file mode 100755 index 00000000000..d764cdd6077 --- /dev/null +++ b/scripts/metrics/annotation_coverage.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Metric 5: LuaLS annotation coverage of exported functions in commons. + +Annotations are what make commons modules statically understandable, so this +tracks how much of the exported surface carries them. + +A function counts as annotated when the contiguous comment block directly above +it mentions `@param` or `@return`. Only `function M.name(...)` and +`function M:name(...)` forms are counted -- that is the shape the doc-block +convention applies to in this codebase. + +A function taking no parameters and returning no value needs neither tag, so it +is reported as exempt and left out of the ratio rather than counted against it. + +`params_total` / `params_annotated` count individual parameters, not functions, +so a five-parameter signature carries five times the weight of a one-parameter +one. +Whether a body returns a value is decided by scanning to the `end` at the +function's own indentation, which is a heuristic: when in doubt it assumes a +value is returned, so a function is more likely to be asked for an annotation +than excused from one. + +Usage: + python3 scripts/metrics/annotation_coverage.py [--csv] [--no-header] + [--base BASE_ROOT] [root] + +Intended to be run on a schedule (e.g. weekly CI job) with --csv appended to a +time-series file, so standardization / Phoenix progress can be charted. +""" + +import argparse +import csv +import re +import sys +from datetime import date +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_ROOT = REPO_ROOT / "lua" / "wikis" / "commons" + +FUNCTION = re.compile(r"^(\s*)function\s+[A-Za-z_][\w.]*[.:]\w+\s*\(([^)]*)\)") +RETURNS_VALUE = re.compile(r"\breturn\s+\S") + +FIELDS = [ + "functions", + "annotated", + "exempt", + "needs_annotation", + "unannotated", + "coverage_pct", + "params_total", + "params_annotated", + "params_pct", +] + + +def doc_block(lines: list[str], index: int) -> str: + """Return the contiguous comment block immediately above lines[index].""" + block = [] + cursor = index - 1 + while cursor >= 0 and lines[cursor].lstrip().startswith("--"): + block.append(lines[cursor]) + cursor -= 1 + return "\n".join(block) + + +def returns_value(lines: list[str], index: int, indent: str) -> bool: + """Whether the function starting at lines[index] returns a value.""" + end = re.compile(rf"^{indent}end\b") + for line in lines[index + 1 :]: + if end.match(line): + return False + if RETURNS_VALUE.search(line): + return True + return True # no clear end found: assume it returns, so we still ask for a tag + + +def collect(root: Path) -> dict: + functions = annotated = exempt = params_total = params_annotated = 0 + + for path in sorted(root.rglob("*.lua")): + # ScribuntoUnit testcases run on-wiki and are not part of the surface. + if "test" in path.parts: + continue + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + for index, line in enumerate(lines): + match = FUNCTION.match(line) + if not match: + continue + indent, raw = match.group(1), match.group(2) + params = [p.strip() for p in raw.split(",") if p.strip()] + params = [p for p in params if p != "self"] + block = doc_block(lines, index) + + functions += 1 + if "@param" in block or "@return" in block: + annotated += 1 + elif not params and not returns_value(lines, index, indent): + exempt += 1 + # Per parameter, not per function: a five-parameter function with + # one tag is not as annotated as a one-parameter function with one. + # Clamped, since a doc block may carry more @param than the + # signature takes. + params_total += len(params) + params_annotated += min(block.count("@param"), len(params)) + + needs = functions - exempt + return { + "functions": functions, + "annotated": annotated, + "exempt": exempt, + "needs_annotation": needs, + "unannotated": needs - annotated, + "coverage_pct": round(annotated / needs * 100, 2) if needs else 0.0, + "params_total": params_total, + "params_annotated": params_annotated, + "params_pct": ( + round(params_annotated / params_total * 100, 2) if params_total else 0.0 + ), + } + + +def print_table(head: dict, base: Optional[dict]) -> None: + for field in FIELDS: + value = head[field] + cell = f"{value:.2f}%" if field.endswith("_pct") else str(value) + if base is not None: + delta = value - base[field] + cell += ( + f" ({delta:+.2f} pp)" if field.endswith("_pct") else f" ({delta:+d})" + ) + print(f"{field.replace('_', ' '):<20}{cell:>22}") + + +def print_csv(head: dict, header: bool) -> None: + writer = csv.writer(sys.stdout) + if header: + writer.writerow(["date"] + FIELDS) + writer.writerow([date.today().isoformat()] + [head[f] for f in FIELDS]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", nargs="?", type=Path, default=DEFAULT_ROOT) + parser.add_argument( + "--csv", action="store_true", help="CSV output (for appending to a time series)" + ) + parser.add_argument( + "--header", + action=argparse.BooleanOptionalAction, + default=True, + help="write the CSV header row (--no-header when appending " + "to an existing time-series file)", + ) + parser.add_argument( + "--base", type=Path, help="compare against this tree (table mode only)" + ) + args = parser.parse_args() + + if not args.root.is_dir(): + print(f"annotation root not found: {args.root}", file=sys.stderr) + return 1 + + head = collect(args.root) + if args.csv: + print_csv(head, header=args.header) + return 0 + + base = None + if args.base: + if args.base.is_dir(): + base = collect(args.base) + else: + print(f"base root not found, omitting deltas: {args.base}", file=sys.stderr) + print_table(head, base) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/metrics/coverage.py b/scripts/metrics/coverage.py new file mode 100755 index 00000000000..71e64c6a90b --- /dev/null +++ b/scripts/metrics/coverage.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Metric 6: busted test coverage of lua/wikis/commons. + +Runs the test suite under luacov and reports how much of commons it exercises. +Coverage instrumentation is a per-line debug hook, so the suite takes roughly +20x longer than normal -- expect about half a minute rather than a second. + +Needs busted and luacov on PATH, both for Lua 5.1: + + luarocks install --lua-version=5.1 busted + luarocks install --lua-version=5.1 luacov + +Scope and exclusions come from lua/.luacov. Modules no spec ever loads count as +0% rather than being left out, so the figure covers all of commons rather than +only the files the suite happens to touch. + +Usage: + python3 scripts/metrics/coverage.py [--csv] [--no-header] [--skip-run] + +Intended to be run on a schedule (e.g. weekly CI job) with --csv appended to a +time-series file, so standardization / Phoenix progress can be charted. +""" + +import argparse +import csv +import re +import subprocess +import sys +from datetime import date +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +LUA_DIR = REPO_ROOT / "lua" +REPORT = LUA_DIR / "luacov.report.out" +STATS = LUA_DIR / "luacov.stats.out" + +TOTAL = re.compile(r"^Total\s+(\d+)\s+(\d+)\s+([\d.]+)%") +FILE_ROW = re.compile(r"^wikis/commons/\S+\s+\d+\s+\d+\s+([\d.]+)%$") + +FIELDS = ["coverage_pct", "hits", "missed", "files", "files_at_zero"] + +INSTALL_HINT = ( + "not found on PATH. Install both for Lua 5.1:\n" + " luarocks install --lua-version=5.1 busted\n" + " luarocks install --lua-version=5.1 luacov" +) + + +def run(command: list[str], cwd: Path) -> None: + try: + subprocess.run(command, cwd=cwd, check=True) + except FileNotFoundError: + raise SystemExit(f"{command[0]} {INSTALL_HINT}") from None + except subprocess.CalledProcessError as error: + raise SystemExit(f"{command[0]} failed (exit {error.returncode})") from None + + +def run_suite() -> None: + """Run the test suite under coverage, then render the luacov report.""" + # lua/.luacov keeps stats so a report can be regenerated, which means a + # stale file would be reused and report the previous run's coverage. + STATS.unlink(missing_ok=True) + REPORT.unlink(missing_ok=True) + run(["busted", "-C", "lua", "--run=ci", "-c"], REPO_ROOT) + run(["luacov"], LUA_DIR) + + +def collect(report: Path) -> dict: + if not report.exists(): + raise SystemExit(f"no report at {report}; luacov produced nothing") + lines = report.read_text(encoding="utf-8", errors="replace").splitlines() + total = next((TOTAL.match(line) for line in lines if TOTAL.match(line)), None) + if total is None: + raise SystemExit(f"no Total line in {report}; did luacov run?") + + shares = [m.group(1) for m in map(FILE_ROW.match, lines) if m] + return { + "coverage_pct": float(total.group(3)), + "hits": int(total.group(1)), + "missed": int(total.group(2)), + "files": len(shares), + "files_at_zero": sum(1 for s in shares if float(s) == 0), + } + + +def print_table(metrics: dict) -> None: + for field in FIELDS: + value = metrics[field] + cell = f"{value:.2f}%" if field.endswith("_pct") else str(value) + print(f"{field.replace('_', ' '):<16}{cell:>12}") + + +def print_csv(metrics: dict, header: bool) -> None: + writer = csv.writer(sys.stdout) + if header: + writer.writerow(["date"] + FIELDS) + writer.writerow([date.today().isoformat()] + [metrics[f] for f in FIELDS]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--csv", action="store_true", help="CSV output (for appending to a time series)" + ) + parser.add_argument( + "--header", + action=argparse.BooleanOptionalAction, + default=True, + help="write the CSV header row (--no-header when appending " + "to an existing time-series file)", + ) + parser.add_argument( + "--skip-run", + action="store_true", + help="reuse the existing luacov report instead of re-running the suite", + ) + args = parser.parse_args() + + if not args.skip_run: + run_suite() + elif not REPORT.exists(): + print(f"no report at {REPORT}; run without --skip-run first", file=sys.stderr) + return 1 + + metrics = collect(REPORT) + if args.csv: + print_csv(metrics, header=args.header) + else: + print_table(metrics) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/metrics/customization_metrics.py b/scripts/metrics/customization_metrics.py new file mode 100755 index 00000000000..a9b8e2d61dd --- /dev/null +++ b/scripts/metrics/customization_metrics.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Metric 4: per-wiki customization of repo-managed Lua code. + +Everything in this repo is standardized; what varies is how much a wiki has to +override to get the behaviour it wants. Override code is the part that carries +maintenance cost, so it is the number worth watching. Declarative data/config +and legacy shims are counted separately -- a wiki adding 500 lines of faction +data is not the same event as one adding 500 lines of overrides. + +Categories are decided by file *content*, not filename: `GetMatchGroupCopyPaste/ +wiki.lua`, `FilterButtons/Config.lua` and `NotabilityChecker/config.lua` all +look declarative and are not. A file is override code when it defines a +non-local function, or exports a `local function` through a `return` statement +(the widget pattern: `local function X` ... `return wrap(X)`). Purely-local +helpers in an otherwise declarative file do not promote it. Any path containing +`Legacy` is counted as legacy before the content check. + +commons is counted too, so shares have the whole of lua/wikis as denominator +and "override code is 37% of all Lua" is answerable. LOC matches metric 1: +non-blank, non-comment-only lines, with total physical lines also reported. + +Usage: + python3 scripts/metrics/customization_metrics.py [--csv] [--no-header] + [--base BASE_ROOT] [root] + +Intended to be run on a schedule (e.g. weekly CI job) with --csv appended to a +time-series file, so standardization / Phoenix progress can be charted. +""" + +import argparse +import csv +import re +import sys +from datetime import date +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_ROOT = REPO_ROOT / "lua" / "wikis" +COMMONS = "commons" + +# A definition is the `function` keyword followed by an optional name and `(`. +# Matching the bare keyword would also catch prose in comments. +FUNCTION_DEF = re.compile(r"\bfunction\b\s*[A-Za-z_][\w.:]*\s*\(|\bfunction\b\s*\(") +# Both local forms, so a local exported via `return` is found either way. +LOCAL_DEF = re.compile(r"^[ \t]*local[ \t]+function[ \t]+([A-Za-z_]\w*)") +LOCAL_ASSIGN = re.compile(r"^[ \t]*local[ \t]+([A-Za-z_]\w*)[ \t]*=[ \t]*function\b") +RETURN_STATEMENT = re.compile(r"^[ \t]*return\b") +LINE_COMMENT = re.compile(r"--.*$") + +# Order is the display order; per-wiki categories first, then commons. +CATEGORIES = ["override", "declarative", "legacy", "commons"] + + +def is_override_code(lines: list[str]) -> bool: + """True when the file exposes behaviour rather than just data or config.""" + local_names = [] + returns = [] + for raw in lines: + line = LINE_COMMENT.sub("", raw) + if not FUNCTION_DEF.search(line): + if RETURN_STATEMENT.match(line): + returns.append(line) + continue + local = LOCAL_DEF.match(line) or LOCAL_ASSIGN.match(line) + if not local: + return True # a non-local function definition + local_names.append(local.group(1)) + + # A local function handed out through `return` is the module's interface. + # `name` not followed by `(` distinguishes handing it out (`return F`, + # `return wrap(F)`) from calling it to build data (`return {a = h()}`), + # which leaves the file declarative. Word-anchored so a short local name + # cannot match an unrelated identifier. + exported = [re.compile(rf"\b{re.escape(n)}\b\s*(?!\()") for n in local_names] + return any(pattern.search(line) for line in returns for pattern in exported) + + +def categorise(path: Path, lines: list[str]) -> str: + if COMMONS in path.parts: + return "commons" + # Substring, not a path component: legacy lives in directories + # (TeamCard/Legacy/Custom.lua) *and* in filenames (Match/Legacy.lua). + # `"Legacy" in path.parts` would miss the latter -- 52 of 78 files. + if "Legacy" in path.as_posix(): + return "legacy" + return "override" if is_override_code(lines) else "declarative" + + +def count_file(lines: list[str]) -> tuple[int, int]: + """Return (physical_lines, loc) -- loc as metric 1 defines it.""" + loc = sum(1 for line in lines if line.strip() and not line.strip().startswith("--")) + return len(lines), loc + + +def collect(root: Path) -> list[dict]: + totals = {c: {"files": 0, "lines": 0, "loc": 0} for c in CATEGORIES} + + for path in sorted(root.rglob("*.lua")): + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + physical, loc = count_file(lines) + bucket = totals[categorise(path, lines)] + bucket["files"] += 1 + bucket["lines"] += physical + bucket["loc"] += loc + + total_loc = sum(b["loc"] for b in totals.values()) + rows = [] + for category in CATEGORIES: + bucket = totals[category] + rows.append( + { + "category": category, + "files": bucket["files"], + "lines": bucket["lines"], + "loc": bucket["loc"], + "share": round(bucket["loc"] / total_loc * 100, 2) + if total_loc + else 0.0, + } + ) + return rows + + +def print_table(rows: list[dict], base: Optional[list[dict]]) -> None: + by_category = {r["category"]: r for r in base} if base else {} + print(f"{'category':<14}{'files':>7}{'lines':>10}{'loc':>10}{'share':>9}") + for row in rows: + share = f"{row['share']:.2f}%" + if row["category"] in by_category: + delta = row["loc"] - by_category[row["category"]]["loc"] + share += f" ({delta:+d} loc)" + print( + f"{row['category']:<14}{row['files']:>7}{row['lines']:>10}" + f"{row['loc']:>10}{share:>9}" + ) + per_wiki = [r for r in rows if r["category"] != "commons"] + per_wiki_loc = sum(r["loc"] for r in per_wiki) + total_loc = sum(r["loc"] for r in rows) + # From the loc totals, not by adding up the rounded per-category shares. + share = per_wiki_loc / total_loc * 100 if total_loc else 0.0 + print( + f"{'per-wiki total':<14}{sum(r['files'] for r in per_wiki):>7}" + f"{sum(r['lines'] for r in per_wiki):>10}{per_wiki_loc:>10}" + f"{share:>8.2f}%" + ) + + +def print_csv(rows: list[dict], header: bool) -> None: + writer = csv.writer(sys.stdout) + if header: + writer.writerow(["date", "category", "files", "lines", "loc", "share"]) + today = date.today().isoformat() + for row in rows: + writer.writerow( + [ + today, + row["category"], + row["files"], + row["lines"], + row["loc"], + row["share"], + ] + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", nargs="?", type=Path, default=DEFAULT_ROOT) + parser.add_argument( + "--csv", action="store_true", help="CSV output (for appending to a time series)" + ) + parser.add_argument( + "--header", + action=argparse.BooleanOptionalAction, + default=True, + help="write the CSV header row (--no-header when appending " + "to an existing time-series file)", + ) + parser.add_argument( + "--base", type=Path, help="compare loc against this tree (table mode only)" + ) + args = parser.parse_args() + + if not args.root.is_dir(): + print(f"wikis root not found: {args.root}", file=sys.stderr) + return 1 + + rows = collect(args.root) + if args.csv: + print_csv(rows, header=args.header) + return 0 + + base = None + if args.base: + if args.base.is_dir(): + base = collect(args.base) + else: + print(f"base root not found, omitting deltas: {args.base}", file=sys.stderr) + print_table(rows, base) + return 0 + + +if __name__ == "__main__": + sys.exit(main())