Split metric-bench's main into run_scores and measured, find the checkout from the history, drop an ambiguous shortened path, write places from the repository root, and give item 85 the history's reader

This commit is contained in:
2026-10-08 21:35:54 +02:00
parent 382472bd39
commit 8a2dbd70f1
7 changed files with 56 additions and 38 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
from typing import Callable
import collections
import dataclasses
import statistics
from typing import Callable
from metric_bench.places import Place
from metric_bench.python_units import Unit
+29 -27
View File
@@ -2,8 +2,9 @@
usage: metric-bench [history]
Run it inside a project checkout. history defaults to docs/comprehension-history.md there. For each
scoring run in it, at the commit the seats read, it measures every Python function and method git
history defaults to docs/comprehension-history.md in the checkout it runs in, and the bench reads
the checkout that holds the history. For each scoring run in it, at the commit the seats read, it
measures every Python function and method git
tracks, test files left out: cognitive complexity (SonarSource's white paper, read for Python),
length in lines, shared state (attributes on self another method of the class also writes, plus
names declared global or nonlocal), and change spread (how many other files changed in a commit with
@@ -22,8 +23,8 @@ import sys
from metric_bench.bench import bench, co_changed, medians, METRICS, RunScores, Score
from failure import Failure
from metric_bench.places import resolved, scoring_runs
from metric_bench.python_units import python_units, Unit
from metric_bench.python_units import measured, python_units
from metric_bench.places import resolved, scoring_runs, ScoringRun
def main(argv: list[str]) -> int:
@@ -33,26 +34,22 @@ def main(argv: list[str]) -> int:
if len(argv) > 1 or argv[:1] and argv[0].startswith("-"):
print(f"unknown arguments {' '.join(argv)}; usage: metric-bench [history]", file=sys.stderr)
return 1
top = git(["rev-parse", "--show-toplevel"], None)
if isinstance(top, Failure):
return failed(top)
history = pathlib.Path(argv[0]) if argv else pathlib.Path(top.strip(), "docs", "comprehension-history.md")
given = pathlib.Path(argv[0]).resolve() if argv else None
found = git(["rev-parse", "--show-toplevel"], str(given.parent) if given else None)
if isinstance(found, Failure):
return failed(found)
top = found.strip()
history = given or pathlib.Path(top, "docs", "comprehension-history.md")
try:
text = history.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as failure:
return failed(Failure(f"could not read {history}: {failure}"))
rows = []
for run in scoring_runs(text):
tracked = git(["ls-tree", "-r", "--name-only", run.commit], top.strip())
if isinstance(tracked, Failure):
return failed(tracked)
units = commit_units(run.commit, tracked.split("\n"), top.strip())
if isinstance(units, Failure):
return failed(units)
log = git(["log", "--no-merges", "--format=format:@@", "--name-only", run.commit], top.strip())
if isinstance(log, Failure):
return failed(log)
if scores := bench(units, resolved(run.places, tracked.split("\n")), co_changed(log)):
scores = run_scores(run, top)
if isinstance(scores, Failure):
return failed(scores)
if scores:
rows.append((run.heading, scores))
if not rows:
return failed(Failure(f"no scoring run in {history} names a Python function or method the bench measures"))
@@ -60,16 +57,21 @@ def main(argv: list[str]) -> int:
return 0
def commit_units(commit: str, tracked: list[str], top: str) -> list[Unit] | Failure:
def run_scores(run: ScoringRun, top: str) -> RunScores | None | Failure:
listed = git(["ls-tree", "-r", "--name-only", run.commit], top)
if isinstance(listed, Failure):
return listed
tracked = listed.split("\n")
units = []
for path in tracked:
name = path.rsplit("/", 1)[-1]
if path.endswith(".py") and not name.startswith("test_") and not name.endswith("_test.py") and "tests" not in path.split("/")[:-1]:
source = git(["show", f"{commit}:{path}"], top)
if isinstance(source, Failure):
return source
units += python_units(path, source)
return units
for path in filter(measured, tracked):
source = git(["show", f"{run.commit}:{path}"], top)
if isinstance(source, Failure):
return source
units += python_units(path, source)
log = git(["log", "--no-merges", "--format=format:@@", "--name-only", run.commit], top)
if isinstance(log, Failure):
return log
return bench(units, resolved(run.places, tracked), co_changed(log))
def table(rows: list[tuple[str, RunScores]]) -> str:
+4 -5
View File
@@ -57,14 +57,13 @@ def bullet_places(bullet: str) -> list[Place]:
def resolved(places: list[Place], tracked: list[str]) -> list[Place]:
"""Seats shorten a path once they have named its folder: `cycle/run.py` after `bin/loop_claude/cycle/states.py`. Each place resolves to the one tracked file it names, or, where several match, the one sharing most of the path resolved before it; a place naming no tracked file is dropped."""
"""Seats shorten a path once they have named its folder: `cycle/run.py` after `bin/loop_claude/cycle/states.py`. Each place resolves to the one tracked file it names, or, where several match, the one in the folder of the place resolved before it; a place naming no file, or several, is dropped."""
found = []
before = ""
for place in places:
candidates = [path for path in tracked if path == place.path or path.endswith("/" + place.path)]
if len(candidates) > 1:
candidates = [max(candidates, key=lambda path: len(os.path.commonprefix([path, before])))]
if candidates:
before = candidates[0]
nearest = [path for path in candidates if path.startswith(os.path.dirname(before) + "/")] if len(candidates) > 1 else candidates
if len(nearest) == 1:
before = nearest[0]
found.append(Place(before, place.name))
return found
+6
View File
@@ -14,6 +14,12 @@ class Unit:
shared_state: int
def measured(path: str) -> bool:
"""A Python file outside the tests."""
folders, _, name = path.rpartition("/")
return name.endswith(".py") and not name.startswith("test_") and not name.endswith("_test.py") and "tests" not in folders.split("/")
def python_units(path: str, source: str) -> list[Unit]:
"""Each module-level function and each method in source, with its metrics; a source that does not parse has none."""
try:
+14 -3
View File
@@ -10,9 +10,9 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from metric_bench.bench import auc, bench, co_changed
from metric_bench.places import Place, resolved, scoring_runs
from metric_bench.python_units import cognitive, python_units, Unit
from metric_bench.python_units import cognitive, measured, python_units, Unit
from test_loop_claude_black_box import host_env
from metric_bench.places import Place, resolved, scoring_runs
TOOL = pathlib.Path(__file__).resolve().parent / "metric-bench"
@@ -85,6 +85,9 @@ class Units(unittest.TestCase):
"""))
self.assertEqual(units, [Unit("a.py", "bump", 0, 3, 1), Unit("a.py", "__init__", 0, 2, 1), Unit("a.py", "grow", 0, 3, 1)])
def test_measures_python_files_outside_the_tests(self):
self.assertEqual([path for path in ["a.py", "bin/test_a.py", "a_test.py", "tests/b.py", "e2e-tests/c.py", "README.md"] if measured(path)], ["a.py", "e2e-tests/c.py"])
def test_a_source_that_does_not_parse_has_no_units(self):
self.assertEqual(python_units("a.py", "def ("), [])
@@ -121,6 +124,9 @@ class Places(unittest.TestCase):
places = [Place("bin/tool/cycle/states.py", "transition"), Place("cycle/run.py", "step"), Place("gone.py", "x")]
self.assertEqual(resolved(places, tracked), [Place("bin/tool/cycle/states.py", "transition"), Place("bin/tool/cycle/run.py", "step")])
def test_drops_a_shortened_path_two_files_match_with_no_place_before_it(self):
self.assertEqual(resolved([Place("cycle/run.py", "step")], ["bin/tool/cycle/run.py", "bin/other/cycle/run.py"]), [])
class Bench(unittest.TestCase):
def test_auc_is_one_for_a_perfect_ranking_and_counts_a_tie_half(self):
@@ -178,5 +184,10 @@ class BlackBox(unittest.TestCase):
def test_exits_1_naming_a_history_it_cannot_read_and_unknown_arguments(self):
top = self.project("")
self.assertIn("could not read", self.run_tool(top, "missing.md").stderr)
self.assertIn("could not read", self.run_tool(top, str(pathlib.Path(top, "missing.md"))).stderr)
self.assertEqual(self.run_tool(top, "--x").returncode, 1)
def test_reads_the_checkout_that_holds_a_history_named_from_elsewhere(self):
top = self.project("# Comprehension history\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
ran = self.run_tool(tempfile.gettempdir(), str(pathlib.Path(top, "docs", "comprehension-history.md")))
self.assertEqual(ran.returncode, 0, ran.stderr)
+1 -1
View File
@@ -108,7 +108,7 @@ Fix a worse ruling at the places the seats voting worse name, and the floor at t
Two files of the project keep the records:
- **History.** `docs/comprehension-history.md` keeps every paired ruling and scoring run, appended and never edited. Each opens with a `## <UTC time, ISO 8601>, <PR or branch> at <commit>` heading naming when the seats finished and the commit they read; a paired ruling's heading ends `, against <base>`. A paired ruling holds a `Ruling: <better, same or worse>` line, a table of each seat's vote, and under each seat, labelled `<seat>, decided by:`, its places. A scoring run holds a table of each seat's five scores, with a last row holding the four seats' mean of each, and under each seat, labelled `<seat>, hardest first:` or, for the architect, `<seat>, would restructure first:`, its places. A place is `path:line` and the unit's name. Appending re-enters no pass. Resolve a merge conflict in the file by putting the default branch's entries first.
- **History.** `docs/comprehension-history.md` keeps every paired ruling and scoring run, appended and never edited. Each opens with a `## <UTC time, ISO 8601>, <PR or branch> at <commit>` heading naming when the seats finished and the commit they read; a paired ruling's heading ends `, against <base>`. A paired ruling holds a `Ruling: <better, same or worse>` line, a table of each seat's vote, and under each seat, labelled `<seat>, decided by:`, its places. A scoring run holds a table of each seat's five scores, with a last row holding the four seats' mean of each, and under each seat, labelled `<seat>, hardest first:` or, for the architect, `<seat>, would restructure first:`, its places. A place is `path:line`, the path written from the repository's root, and the unit's name. Appending re-enters no pass. Resolve a merge conflict in the file by putting the default branch's entries first.
- **Entry.** `docs/decisions.md` keeps one entry whose title opens `comprehension floor:`. It holds each decision to ship under the floor, with its scores.
A `comprehension floor:` entry written before the history may still hold a scoring run's five scores. The next scoring run moves them, and only them, into the history as its first run: a heading of the date or time the entry gives and whatever it names the run by, its commit included where named, and a table with the mean alone. The rest of the entry stays.
+1 -1
View File
@@ -60,7 +60,7 @@ A hand-written question bank goes stale as the project grows and leaves everythi
### 85. Move larv-review's mechanical rules — the paired ruling's base and vote count, each pass's round count, the scoring run's trigger and the comprehension history's format — into one tested tool under `bin/`, so the skill says only when to run it and what each answer means.
2026-10-08, lilleman: next after the items that lower token use (73, 77, 78). Every comprehension run since PR #224 named `skills/larv-review/SKILL.md` §3 "Architecture" the hardest place, and PR #228's paired ruling ruled worse on rules a program should hold: the base's `git log` search and fallbacks, the vote arithmetic, a trigger written out three times, and the round count PR #225's seats said only prose enforces. PR #228's reviewers found about twenty edge cases in that text, among them squash against merge commits, entry order, a vote with no place, and a trigger judged before items are removed; tests can hold each, prose could not. Keep the skill's judgement calls in prose, and give the tool the history's format, so the base search does not hang on the wording of a line.
2026-10-08, lilleman: next after the items that lower token use (73, 77, 78). Every comprehension run since PR #224 named `skills/larv-review/SKILL.md` §3 "Architecture" the hardest place, and PR #228's paired ruling ruled worse on rules a program should hold: the base's `git log` search and fallbacks, the vote arithmetic, a trigger written out three times, and the round count PR #225's seats said only prose enforces. PR #228's reviewers found about twenty edge cases in that text, among them squash against merge commits, entry order, a vote with no place, and a trigger judged before items are removed; tests can hold each, prose could not. Keep the skill's judgement calls in prose, and give the tool the history's format, so the base search does not hang on the wording of a line. The history's reader in `bin/metric_bench/places.py` moves into that tool, so the format has one reader.
### 1. Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.