Split metric-bench's main into run_scores and measured, find the checkout from the history, drop an ambiguous shortened path, write places from the repository root, and give item 85 the history's reader
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
from typing import Callable
|
||||
import collections
|
||||
import dataclasses
|
||||
import statistics
|
||||
from typing import Callable
|
||||
|
||||
from metric_bench.places import Place
|
||||
from metric_bench.python_units import Unit
|
||||
|
||||
+29
-27
@@ -2,8 +2,9 @@
|
||||
|
||||
usage: metric-bench [history]
|
||||
|
||||
Run it inside a project checkout. history defaults to docs/comprehension-history.md there. For each
|
||||
scoring run in it, at the commit the seats read, it measures every Python function and method git
|
||||
history defaults to docs/comprehension-history.md in the checkout it runs in, and the bench reads
|
||||
the checkout that holds the history. For each scoring run in it, at the commit the seats read, it
|
||||
measures every Python function and method git
|
||||
tracks, test files left out: cognitive complexity (SonarSource's white paper, read for Python),
|
||||
length in lines, shared state (attributes on self another method of the class also writes, plus
|
||||
names declared global or nonlocal), and change spread (how many other files changed in a commit with
|
||||
@@ -22,8 +23,8 @@ import sys
|
||||
|
||||
from metric_bench.bench import bench, co_changed, medians, METRICS, RunScores, Score
|
||||
from failure import Failure
|
||||
from metric_bench.places import resolved, scoring_runs
|
||||
from metric_bench.python_units import python_units, Unit
|
||||
from metric_bench.python_units import measured, python_units
|
||||
from metric_bench.places import resolved, scoring_runs, ScoringRun
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
@@ -33,26 +34,22 @@ def main(argv: list[str]) -> int:
|
||||
if len(argv) > 1 or argv[:1] and argv[0].startswith("-"):
|
||||
print(f"unknown arguments {' '.join(argv)}; usage: metric-bench [history]", file=sys.stderr)
|
||||
return 1
|
||||
top = git(["rev-parse", "--show-toplevel"], None)
|
||||
if isinstance(top, Failure):
|
||||
return failed(top)
|
||||
history = pathlib.Path(argv[0]) if argv else pathlib.Path(top.strip(), "docs", "comprehension-history.md")
|
||||
given = pathlib.Path(argv[0]).resolve() if argv else None
|
||||
found = git(["rev-parse", "--show-toplevel"], str(given.parent) if given else None)
|
||||
if isinstance(found, Failure):
|
||||
return failed(found)
|
||||
top = found.strip()
|
||||
history = given or pathlib.Path(top, "docs", "comprehension-history.md")
|
||||
try:
|
||||
text = history.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
return failed(Failure(f"could not read {history}: {failure}"))
|
||||
rows = []
|
||||
for run in scoring_runs(text):
|
||||
tracked = git(["ls-tree", "-r", "--name-only", run.commit], top.strip())
|
||||
if isinstance(tracked, Failure):
|
||||
return failed(tracked)
|
||||
units = commit_units(run.commit, tracked.split("\n"), top.strip())
|
||||
if isinstance(units, Failure):
|
||||
return failed(units)
|
||||
log = git(["log", "--no-merges", "--format=format:@@", "--name-only", run.commit], top.strip())
|
||||
if isinstance(log, Failure):
|
||||
return failed(log)
|
||||
if scores := bench(units, resolved(run.places, tracked.split("\n")), co_changed(log)):
|
||||
scores = run_scores(run, top)
|
||||
if isinstance(scores, Failure):
|
||||
return failed(scores)
|
||||
if scores:
|
||||
rows.append((run.heading, scores))
|
||||
if not rows:
|
||||
return failed(Failure(f"no scoring run in {history} names a Python function or method the bench measures"))
|
||||
@@ -60,16 +57,21 @@ def main(argv: list[str]) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def commit_units(commit: str, tracked: list[str], top: str) -> list[Unit] | Failure:
|
||||
def run_scores(run: ScoringRun, top: str) -> RunScores | None | Failure:
|
||||
listed = git(["ls-tree", "-r", "--name-only", run.commit], top)
|
||||
if isinstance(listed, Failure):
|
||||
return listed
|
||||
tracked = listed.split("\n")
|
||||
units = []
|
||||
for path in tracked:
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
if path.endswith(".py") and not name.startswith("test_") and not name.endswith("_test.py") and "tests" not in path.split("/")[:-1]:
|
||||
source = git(["show", f"{commit}:{path}"], top)
|
||||
if isinstance(source, Failure):
|
||||
return source
|
||||
units += python_units(path, source)
|
||||
return units
|
||||
for path in filter(measured, tracked):
|
||||
source = git(["show", f"{run.commit}:{path}"], top)
|
||||
if isinstance(source, Failure):
|
||||
return source
|
||||
units += python_units(path, source)
|
||||
log = git(["log", "--no-merges", "--format=format:@@", "--name-only", run.commit], top)
|
||||
if isinstance(log, Failure):
|
||||
return log
|
||||
return bench(units, resolved(run.places, tracked), co_changed(log))
|
||||
|
||||
|
||||
def table(rows: list[tuple[str, RunScores]]) -> str:
|
||||
|
||||
@@ -57,14 +57,13 @@ def bullet_places(bullet: str) -> list[Place]:
|
||||
|
||||
|
||||
def resolved(places: list[Place], tracked: list[str]) -> list[Place]:
|
||||
"""Seats shorten a path once they have named its folder: `cycle/run.py` after `bin/loop_claude/cycle/states.py`. Each place resolves to the one tracked file it names, or, where several match, the one sharing most of the path resolved before it; a place naming no tracked file is dropped."""
|
||||
"""Seats shorten a path once they have named its folder: `cycle/run.py` after `bin/loop_claude/cycle/states.py`. Each place resolves to the one tracked file it names, or, where several match, the one in the folder of the place resolved before it; a place naming no file, or several, is dropped."""
|
||||
found = []
|
||||
before = ""
|
||||
for place in places:
|
||||
candidates = [path for path in tracked if path == place.path or path.endswith("/" + place.path)]
|
||||
if len(candidates) > 1:
|
||||
candidates = [max(candidates, key=lambda path: len(os.path.commonprefix([path, before])))]
|
||||
if candidates:
|
||||
before = candidates[0]
|
||||
nearest = [path for path in candidates if path.startswith(os.path.dirname(before) + "/")] if len(candidates) > 1 else candidates
|
||||
if len(nearest) == 1:
|
||||
before = nearest[0]
|
||||
found.append(Place(before, place.name))
|
||||
return found
|
||||
|
||||
@@ -14,6 +14,12 @@ class Unit:
|
||||
shared_state: int
|
||||
|
||||
|
||||
def measured(path: str) -> bool:
|
||||
"""A Python file outside the tests."""
|
||||
folders, _, name = path.rpartition("/")
|
||||
return name.endswith(".py") and not name.startswith("test_") and not name.endswith("_test.py") and "tests" not in folders.split("/")
|
||||
|
||||
|
||||
def python_units(path: str, source: str) -> list[Unit]:
|
||||
"""Each module-level function and each method in source, with its metrics; a source that does not parse has none."""
|
||||
try:
|
||||
|
||||
@@ -10,9 +10,9 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from metric_bench.bench import auc, bench, co_changed
|
||||
from metric_bench.places import Place, resolved, scoring_runs
|
||||
from metric_bench.python_units import cognitive, python_units, Unit
|
||||
from metric_bench.python_units import cognitive, measured, python_units, Unit
|
||||
from test_loop_claude_black_box import host_env
|
||||
from metric_bench.places import Place, resolved, scoring_runs
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parent / "metric-bench"
|
||||
|
||||
@@ -85,6 +85,9 @@ class Units(unittest.TestCase):
|
||||
"""))
|
||||
self.assertEqual(units, [Unit("a.py", "bump", 0, 3, 1), Unit("a.py", "__init__", 0, 2, 1), Unit("a.py", "grow", 0, 3, 1)])
|
||||
|
||||
def test_measures_python_files_outside_the_tests(self):
|
||||
self.assertEqual([path for path in ["a.py", "bin/test_a.py", "a_test.py", "tests/b.py", "e2e-tests/c.py", "README.md"] if measured(path)], ["a.py", "e2e-tests/c.py"])
|
||||
|
||||
def test_a_source_that_does_not_parse_has_no_units(self):
|
||||
self.assertEqual(python_units("a.py", "def ("), [])
|
||||
|
||||
@@ -121,6 +124,9 @@ class Places(unittest.TestCase):
|
||||
places = [Place("bin/tool/cycle/states.py", "transition"), Place("cycle/run.py", "step"), Place("gone.py", "x")]
|
||||
self.assertEqual(resolved(places, tracked), [Place("bin/tool/cycle/states.py", "transition"), Place("bin/tool/cycle/run.py", "step")])
|
||||
|
||||
def test_drops_a_shortened_path_two_files_match_with_no_place_before_it(self):
|
||||
self.assertEqual(resolved([Place("cycle/run.py", "step")], ["bin/tool/cycle/run.py", "bin/other/cycle/run.py"]), [])
|
||||
|
||||
|
||||
class Bench(unittest.TestCase):
|
||||
def test_auc_is_one_for_a_perfect_ranking_and_counts_a_tie_half(self):
|
||||
@@ -178,5 +184,10 @@ class BlackBox(unittest.TestCase):
|
||||
|
||||
def test_exits_1_naming_a_history_it_cannot_read_and_unknown_arguments(self):
|
||||
top = self.project("")
|
||||
self.assertIn("could not read", self.run_tool(top, "missing.md").stderr)
|
||||
self.assertIn("could not read", self.run_tool(top, str(pathlib.Path(top, "missing.md"))).stderr)
|
||||
self.assertEqual(self.run_tool(top, "--x").returncode, 1)
|
||||
|
||||
def test_reads_the_checkout_that_holds_a_history_named_from_elsewhere(self):
|
||||
top = self.project("# Comprehension history\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
|
||||
ran = self.run_tool(tempfile.gettempdir(), str(pathlib.Path(top, "docs", "comprehension-history.md")))
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
|
||||
@@ -108,7 +108,7 @@ Fix a worse ruling at the places the seats voting worse name, and the floor at t
|
||||
|
||||
Two files of the project keep the records:
|
||||
|
||||
- **History.** `docs/comprehension-history.md` keeps every paired ruling and scoring run, appended and never edited. Each opens with a `## <UTC time, ISO 8601>, <PR or branch> at <commit>` heading naming when the seats finished and the commit they read; a paired ruling's heading ends `, against <base>`. A paired ruling holds a `Ruling: <better, same or worse>` line, a table of each seat's vote, and under each seat, labelled `<seat>, decided by:`, its places. A scoring run holds a table of each seat's five scores, with a last row holding the four seats' mean of each, and under each seat, labelled `<seat>, hardest first:` or, for the architect, `<seat>, would restructure first:`, its places. A place is `path:line` and the unit's name. Appending re-enters no pass. Resolve a merge conflict in the file by putting the default branch's entries first.
|
||||
- **History.** `docs/comprehension-history.md` keeps every paired ruling and scoring run, appended and never edited. Each opens with a `## <UTC time, ISO 8601>, <PR or branch> at <commit>` heading naming when the seats finished and the commit they read; a paired ruling's heading ends `, against <base>`. A paired ruling holds a `Ruling: <better, same or worse>` line, a table of each seat's vote, and under each seat, labelled `<seat>, decided by:`, its places. A scoring run holds a table of each seat's five scores, with a last row holding the four seats' mean of each, and under each seat, labelled `<seat>, hardest first:` or, for the architect, `<seat>, would restructure first:`, its places. A place is `path:line`, the path written from the repository's root, and the unit's name. Appending re-enters no pass. Resolve a merge conflict in the file by putting the default branch's entries first.
|
||||
- **Entry.** `docs/decisions.md` keeps one entry whose title opens `comprehension floor:`. It holds each decision to ship under the floor, with its scores.
|
||||
|
||||
A `comprehension floor:` entry written before the history may still hold a scoring run's five scores. The next scoring run moves them, and only them, into the history as its first run: a heading of the date or time the entry gives and whatever it names the run by, its commit included where named, and a table with the mean alone. The rest of the entry stays.
|
||||
|
||||
@@ -60,7 +60,7 @@ A hand-written question bank goes stale as the project grows and leaves everythi
|
||||
|
||||
### 85. Move larv-review's mechanical rules — the paired ruling's base and vote count, each pass's round count, the scoring run's trigger and the comprehension history's format — into one tested tool under `bin/`, so the skill says only when to run it and what each answer means.
|
||||
|
||||
2026-10-08, lilleman: next after the items that lower token use (73, 77, 78). Every comprehension run since PR #224 named `skills/larv-review/SKILL.md` §3 "Architecture" the hardest place, and PR #228's paired ruling ruled worse on rules a program should hold: the base's `git log` search and fallbacks, the vote arithmetic, a trigger written out three times, and the round count PR #225's seats said only prose enforces. PR #228's reviewers found about twenty edge cases in that text, among them squash against merge commits, entry order, a vote with no place, and a trigger judged before items are removed; tests can hold each, prose could not. Keep the skill's judgement calls in prose, and give the tool the history's format, so the base search does not hang on the wording of a line.
|
||||
2026-10-08, lilleman: next after the items that lower token use (73, 77, 78). Every comprehension run since PR #224 named `skills/larv-review/SKILL.md` §3 "Architecture" the hardest place, and PR #228's paired ruling ruled worse on rules a program should hold: the base's `git log` search and fallbacks, the vote arithmetic, a trigger written out three times, and the round count PR #225's seats said only prose enforces. PR #228's reviewers found about twenty edge cases in that text, among them squash against merge commits, entry order, a vote with no place, and a trigger judged before items are removed; tests can hold each, prose could not. Keep the skill's judgement calls in prose, and give the tool the history's format, so the base search does not hang on the wording of a line. The history's reader in `bin/metric_bench/places.py` moves into that tool, so the format has one reader.
|
||||
|
||||
### 1. Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user