250 lines
11 KiB
Python
250 lines
11 KiB
Python
import ast
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
|
|
from metric_bench.bench import auc, bench, co_changed, top_k
|
|
from metric_bench.python_units import cognitive, measured, python_units, Unit
|
|
from failure import Failure
|
|
from test_loop_claude_black_box import host_env
|
|
from larv_review.history import Place, resolved, scoring_runs
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parent / "metric-bench"
|
|
|
|
|
|
def scored(source: str) -> int:
|
|
function = ast.parse(textwrap.dedent(source)).body[0]
|
|
assert isinstance(function, ast.FunctionDef)
|
|
return cognitive(function)
|
|
|
|
|
|
class Cognitive(unittest.TestCase):
|
|
def test_scores_the_white_papers_sum_of_primes_without_its_labelled_continue(self):
|
|
self.assertEqual(scored("""
|
|
def sum_of_primes(limit):
|
|
total = 0
|
|
for i in range(1, limit + 1):
|
|
for j in range(2, i):
|
|
if i % j == 0:
|
|
break
|
|
total += i
|
|
return total
|
|
"""), 6)
|
|
|
|
def test_counts_elif_and_else_flat_and_a_nested_ternary_by_its_depth(self):
|
|
self.assertEqual(scored("""
|
|
def f(a):
|
|
if a:
|
|
return 1 if a > 1 else 2
|
|
elif a is None:
|
|
return 3
|
|
else:
|
|
return 4
|
|
"""), 5)
|
|
|
|
def test_counts_each_boolean_operator_sequence_a_handler_a_match_and_recursion(self):
|
|
self.assertEqual(scored("""
|
|
def f(a, b, c):
|
|
try:
|
|
if a and b and c or a:
|
|
match a:
|
|
case 1:
|
|
return f(b, c, a)
|
|
except ValueError:
|
|
pass
|
|
"""), 1 + 2 + 2 + 1 + 1)
|
|
|
|
def test_nests_a_while_a_lambda_and_an_inner_function(self):
|
|
self.assertEqual(scored("""
|
|
def f(a):
|
|
while a:
|
|
g = lambda b: 1 if b else 2
|
|
def h():
|
|
if a:
|
|
pass
|
|
"""), 1 + 3 + 3)
|
|
|
|
def test_counts_a_call_to_itself_on_self_as_recursion(self):
|
|
self.assertEqual(scored("""
|
|
def walk(self, node):
|
|
return self.walk(node.child)
|
|
"""), 1)
|
|
|
|
def test_counts_a_comprehension_as_a_loop_and_its_conditions(self):
|
|
self.assertEqual(scored("""
|
|
def f(rows):
|
|
return [row for row in rows if row if row > 1]
|
|
"""), 3)
|
|
|
|
|
|
class Units(unittest.TestCase):
|
|
def test_measures_functions_and_methods_with_the_state_methods_share(self):
|
|
units = python_units("a.py", textwrap.dedent("""
|
|
COUNT = 0
|
|
|
|
def bump():
|
|
global COUNT
|
|
COUNT += 1
|
|
|
|
class Box:
|
|
def __init__(self):
|
|
self.size = 0
|
|
|
|
def grow(self):
|
|
self.size += 1
|
|
self.name = "x"
|
|
|
|
def rename(self):
|
|
self.name, self.other = "y", 1
|
|
self.buf[self.size] = 0
|
|
"""))
|
|
self.assertEqual(units, [Unit("a.py", "bump", 0, 3, 1), Unit("a.py", "__init__", 0, 2, 1), Unit("a.py", "grow", 0, 3, 2), Unit("a.py", "rename", 0, 3, 1)])
|
|
|
|
def test_measures_python_files_outside_the_tests(self):
|
|
self.assertEqual([path for path in ["a.py", "bin/test_a.py", "a_test.py", "tests/b.py", "e2e-tests/c.py", "README.md"] if measured(path)], ["a.py", "e2e-tests/c.py"])
|
|
|
|
def test_a_source_that_does_not_parse_is_a_failure_naming_it(self):
|
|
failure = python_units("a.py", "def (")
|
|
assert isinstance(failure, Failure)
|
|
self.assertIn("a.py does not parse on Python", failure.reason)
|
|
|
|
|
|
class Places(unittest.TestCase):
|
|
HISTORY = textwrap.dedent("""
|
|
# Comprehension history
|
|
|
|
## 2026-10-08T12:50:49Z, PR #224 at a58a0a8
|
|
|
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
|
|---|---|---|---|---|---|
|
|
| Mean | 7 | 7 | 7 | 7 | 7 |
|
|
|
|
Junior A, hardest first:
|
|
|
|
- `skills/x/SKILL.md:77` "Architecture"
|
|
- `bin/tool/cycle/states.py:69` `transition`, with `cycle/run.py:55` `step` and `:90` `close`
|
|
|
|
Inherited architect, would restructure first:
|
|
|
|
- `bin/tool/cycle/judge.py:31` `run_judge`
|
|
|
|
## 2026-10-08T18:37:21Z, PR #228 at 2682668, against 8c9bf09
|
|
|
|
Mid A, decided by:
|
|
|
|
- `bin/tool/cycle/run.py:1` `step`
|
|
""")
|
|
|
|
def test_reads_each_scoring_runs_places_and_leaves_paired_rulings_out(self):
|
|
runs = scoring_runs(self.HISTORY)
|
|
self.assertEqual([(run.heading, run.commit) for run in runs], [("2026-10-08T12:50:49Z, PR #224 at a58a0a8", "a58a0a8")])
|
|
self.assertEqual(runs[0].places, [Place("skills/x/SKILL.md", None), Place("bin/tool/cycle/states.py", "transition"), Place("cycle/run.py", "step"), Place("cycle/run.py", "close"), Place("bin/tool/cycle/judge.py", "run_judge")])
|
|
|
|
def test_resolves_a_shortened_path_to_the_file_nearest_the_one_before_it(self):
|
|
tracked = ["bin/tool/cycle/states.py", "bin/tool/cycle/run.py", "bin/other/cycle/run.py"]
|
|
places = [Place("bin/tool/cycle/states.py", "transition"), Place("cycle/run.py", "step"), Place("gone.py", "x")]
|
|
self.assertEqual(resolved(places, tracked), ([Place("bin/tool/cycle/states.py", "transition"), Place("bin/tool/cycle/run.py", "step")], [Place("gone.py", "x")]))
|
|
|
|
def test_a_full_path_wins_over_a_file_in_the_folder_before_it(self):
|
|
self.assertEqual(resolved([Place("pkg/x.py", "a"), Place("main.py", "f")], ["pkg/x.py", "main.py", "pkg/main.py"])[0][-1], Place("main.py", "f"))
|
|
|
|
def test_resolves_a_shortened_path_after_a_repeated_place_by_the_repeat(self):
|
|
tracked = ["bin/a/main.py", "bin/b/main.py", "bin/b/bench.py"]
|
|
places = [Place("bin/a/main.py", "main"), Place("bin/b/bench.py", "bench"), Place("bin/a/main.py", "main"), Place("main.py", "figures")]
|
|
self.assertEqual(resolved(places, tracked)[0][-1], Place("bin/a/main.py", "figures"))
|
|
|
|
def test_drops_a_shortened_path_two_files_match_with_no_place_before_it(self):
|
|
self.assertEqual(resolved([Place("cycle/run.py", "step")], ["bin/tool/cycle/run.py", "bin/other/cycle/run.py"]), ([], [Place("cycle/run.py", "step")]))
|
|
|
|
|
|
class Bench(unittest.TestCase):
|
|
def test_auc_is_one_for_a_perfect_ranking_and_counts_a_tie_half(self):
|
|
self.assertEqual(auc([5, 4], [1, 2]), 1.0)
|
|
self.assertEqual(auc([3], [3]), 0.5)
|
|
|
|
def test_co_changed_counts_each_files_distinct_partners_past_a_sweep(self):
|
|
sweep = "".join(f"f{i}.py\n" for i in range(31))
|
|
self.assertEqual(co_changed(f"@@\na.py\nmy b.py\n@@\na.py\nc.py\n@@\na.py\n@@\n{sweep}"), {"a.py": 2, "my b.py": 1, "c.py": 1})
|
|
|
|
def test_top_k_shares_the_seats_at_a_tied_cut_evenly(self):
|
|
self.assertEqual(top_k([0, 0], [0, 0, 0, 0]), 1 / 3)
|
|
self.assertEqual(top_k([5, 1], [1, 1]), 2 / 3)
|
|
|
|
def test_scores_the_named_units_and_counts_places_no_unit_measures(self):
|
|
units = [Unit("a.py", "hard", 9, 40, 0), Unit("a.py", "easy", 1, 5, 0), Unit("b.py", "plain", 2, 6, 0)]
|
|
run = bench(units, [Place("a.py", "hard"), Place("doc.md", None)], 2, {"a.py": 1})
|
|
assert run is not None
|
|
self.assertEqual((run.units, run.named, run.unmeasured), (3, 1, 3))
|
|
self.assertEqual((run.scores["cognitive complexity"].auc, run.scores["cognitive complexity"].top_k), (1.0, 1.0))
|
|
self.assertEqual((run.scores["shared state"].auc, run.scores["shared state"].top_k), (0.5, 1 / 3))
|
|
|
|
def test_a_run_naming_no_unit_or_every_unit_is_not_scored(self):
|
|
self.assertIsNone(bench([Unit("a.py", "f", 1, 1, 0)], [Place("doc.md", None)], 0, {}))
|
|
self.assertIsNone(bench([Unit("a.py", "f", 1, 1, 0)], [Place("a.py", "f")], 0, {}))
|
|
|
|
|
|
class BlackBox(unittest.TestCase):
|
|
def project(self, history: str) -> str:
|
|
directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(directory.cleanup)
|
|
top = directory.name
|
|
env = host_env()
|
|
for args in (["init", "-q"], ["config", "user.email", "t@t"], ["config", "user.name", "t"]):
|
|
subprocess.run(["git", *args], cwd=top, check=True, env=env)
|
|
pathlib.Path(top, "a.py").write_text("def hard(a):\n\tif a:\n\t\tfor x in a:\n\t\t\tif x:\n\t\t\t\treturn x\n\n\ndef easy():\n\treturn 1\n")
|
|
pathlib.Path(top, "test_a.py").write_text("def test_hard():\n\tif 1:\n\t\tpass\n")
|
|
subprocess.run(["git", "add", "."], cwd=top, check=True, env=env)
|
|
subprocess.run(["git", "commit", "-qm", "a"], cwd=top, check=True, env=env)
|
|
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=top, check=True, capture_output=True, text=True, env=env).stdout.strip()
|
|
os.makedirs(pathlib.Path(top, "docs"))
|
|
pathlib.Path(top, "docs", "comprehension-history.md").write_text(history.replace("COMMIT", commit))
|
|
return top
|
|
|
|
def run_tool(self, top: str, *args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run([sys.executable, str(TOOL), *args], cwd=top, capture_output=True, text=True, env=host_env())
|
|
|
|
def test_prints_each_metrics_score_on_each_run_and_the_medians(self):
|
|
top = self.project("# Comprehension history\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
|
|
ran = self.run_tool(top)
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertIn("| 2026-10-08T12:50:49Z, PR #1 at ", ran.stdout)
|
|
self.assertIn("| 2 | 1 | 0 | top k 0.50 | AUC 1.00, top k 1.00 | AUC 1.00, top k 1.00 | constant | constant |", ran.stdout)
|
|
self.assertIn("| Median |", ran.stdout)
|
|
|
|
def test_exits_1_when_no_run_names_a_measured_unit(self):
|
|
top = self.project("# Comprehension history\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `README.md:1` \"Install\"\n")
|
|
ran = self.run_tool(top)
|
|
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
|
|
self.assertIn("no scoring run in", ran.stderr)
|
|
|
|
def test_exits_1_naming_a_history_it_cannot_read_and_unknown_arguments(self):
|
|
top = self.project("")
|
|
self.assertIn("could not read", self.run_tool(top, str(pathlib.Path(top, "missing.md"))).stderr)
|
|
pathlib.Path(top, "docs", "comprehension-history.md").unlink()
|
|
self.assertIn("no comprehension history at", self.run_tool(top).stderr)
|
|
self.assertIn("could not read", self.run_tool(top, str(pathlib.Path(top, "gone", "x.md"))).stderr)
|
|
self.assertEqual(self.run_tool(top, "--x").returncode, 1)
|
|
|
|
def test_skips_and_names_a_run_whose_commit_the_checkout_lacks(self):
|
|
top = self.project("# Comprehension history\n\n## 2026-10-07T12:00:00Z, PR #0 at 0123456\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
|
|
ran = self.run_tool(top)
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertIn("Skipped, its commit is not in", ran.stdout)
|
|
self.assertIn("PR #0 at 0123456", ran.stdout.split("Skipped")[1])
|
|
|
|
def test_names_a_run_it_cannot_score_below_the_table(self):
|
|
top = self.project("# Comprehension history\n\n## 2026-10-07T12:00:00Z, PR #0 at COMMIT\n\nJunior A, hardest first:\n\n- `README.md:1` \"Install\"\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
|
|
ran = self.run_tool(top)
|
|
self.assertIn("Not scored, its places name no measured unit or every one: 2026-10-07T12:00:00Z, PR #0", ran.stdout)
|
|
|
|
def test_reads_the_checkout_that_holds_a_history_named_from_elsewhere(self):
|
|
top = self.project("# Comprehension history\n\n## 2026-10-08T12:50:49Z, PR #1 at COMMIT\n\nJunior A, hardest first:\n\n- `a.py:1` `hard`\n")
|
|
ran = self.run_tool(tempfile.gettempdir(), str(pathlib.Path(top, "docs", "comprehension-history.md")))
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|