From b474ec2f5a19b57fc699491052cfe09869bd7981 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 21 Nov 2026 04:24:52 +0100 Subject: [PATCH] Test the digest's cuts, the median pick and gather's refusals, and drop project_rung's checkout parameter --- bin/task_bench/bench_folder.py | 2 +- bin/task_bench/digest.py | 2 +- bin/task_bench/gather.py | 22 +++++++++++----------- bin/task_bench/test_digest.py | 16 +++++++++++++++- bin/task_bench/test_gather.py | 12 +++++++++++- bin/task_bench/test_task_bench.py | 9 +++++++++ 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/bin/task_bench/bench_folder.py b/bin/task_bench/bench_folder.py index 5a6ea7b9..b03be624 100644 --- a/bin/task_bench/bench_folder.py +++ b/bin/task_bench/bench_folder.py @@ -8,8 +8,8 @@ import pathlib import shutil import typing -from chunk_driver.claims import locked from failure import Failure +from chunk_driver.claims import locked FOLDER = "task-bench" # Beside FOLDER, not in it, since a bench empties its folder. It marks FOLDER as a bench's, and stays once the folder is gone. diff --git a/bin/task_bench/digest.py b/bin/task_bench/digest.py index 4ec038cd..dc20d3a7 100644 --- a/bin/task_bench/digest.py +++ b/bin/task_bench/digest.py @@ -6,8 +6,8 @@ import re import typing from claude_code.entries import AssistantTurn, parse_entry, Text, tool_calls, ToolCall, ToolResult, ToolResults -from reading_cost.measure import reads_of from task_bench.figures import HOME, WORK +from reading_cost.measure import reads_of # How much of the model's text between two calls a trace keeps, and of each line, of a search's pattern or of a shell command. TEXT_LINES = 3 diff --git a/bin/task_bench/gather.py b/bin/task_bench/gather.py index f53ee001..cd35c3d1 100644 --- a/bin/task_bench/gather.py +++ b/bin/task_bench/gather.py @@ -7,17 +7,17 @@ import shutil import subprocess import typing -from chunk_driver.project.model_choice import read_models_file from claude_code.reply import Claude, claude_reply, Limited from failure import Failure -from git_cli import git_repo from git_cli.git_repo import Git, top -from loop_state.paths import paths_from -from model_routing.ladder import Rung -from model_routing.ladder_file import project_ladder -from review_rules.review_folder import review_folder +from git_cli import git_repo from task_bench.bench_folder import held +from loop_state.paths import paths_from from transcript_archive.project_key import project_key +from model_routing.ladder_file import project_ladder +from chunk_driver.project.model_choice import read_models_file +from review_rules.review_folder import review_folder +from model_routing.ladder import Rung DIGEST = "digest.md" PROMPT = pathlib.Path(__file__).with_name("gather-prompt.md") @@ -30,7 +30,7 @@ def gather(env: typing.Mapping[str, str], home: pathlib.Path, claude: Claude) -> checkout = top(git) if isinstance(checkout, Failure): return checkout - rung = project_rung(checkout, home, git) + rung = project_rung(home, git) if isinstance(rung, Failure): return rung with held(pathlib.Path(review_folder(str(home), checkout))) as folder: @@ -56,9 +56,9 @@ def run_claude(args: list[str], cwd: str, prompt: str) -> subprocess.CompletedPr return subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=prompt, text=True) -def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure: - """The cheapest rung of the model loop-claude runs the project on, on the project's routing ladder.""" - common = git("rev-parse", "--path-format=absolute", "--git-common-dir", cwd=checkout) +def project_rung(home: pathlib.Path, git: Git) -> Rung | Failure: + """The cheapest rung of the model loop-claude runs the project on, on the routing ladder of the project the process runs in.""" + common = git("rev-parse", "--path-format=absolute", "--git-common-dir") if isinstance(common, Failure): return common # A worktree's project is its main checkout's, which holds the folder every worktree shares. @@ -69,7 +69,7 @@ def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure: return models if key not in models: return Failure(f"{saved} names no model for {key}: run loop-claude in the project once, which asks for it.") - ladder = project_ladder(checkout, home) + ladder = project_ladder(None, home) if isinstance(ladder, Failure): return ladder project = ladder.project_rung(models[key]) diff --git a/bin/task_bench/test_digest.py b/bin/task_bench/test_digest.py index 5cd77c60..5953bef1 100644 --- a/bin/task_bench/test_digest.py +++ b/bin/task_bench/test_digest.py @@ -49,6 +49,13 @@ class TraceTest(unittest.TestCase): def test_a_shell_command_shows_its_first_line_cut_short(self): self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"]) self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"]) + self.assertEqual(trace(stream(call("Bash", command="x" * 200 + "\nls"))), [f"Bash {'x' * 120} …"]) + self.assertEqual(trace(stream(call("Bash", command="x" * 120 + "\nls"))), [f"Bash {'x' * 120} …"]) + self.assertEqual(trace(stream(call("Bash", command="x" * 120))), [f"Bash {'x' * 120}"]) + self.assertEqual(trace(stream(call("Bash", command="ls"), result("1", "a.go\nb.go"))), ["Bash ls"]) + + def test_a_call_without_its_pattern_or_command_shows_none(self): + self.assertEqual(trace(stream(call("Grep"), call("Bash", "2"))), ['Grep ""', "Bash "]) def test_any_other_tool_shows_its_name_and_the_file_it_names(self): self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"]) @@ -61,6 +68,9 @@ class TraceTest(unittest.TestCase): told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it.")) self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."]) self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"]) + self.assertEqual(trace(stream(said("one\ntwo\nthree"))), ["> one", "> two", "> three"]) + self.assertEqual(trace(stream(said(f"one\n{'y' * 300}\nthree\nfour"))), ["> one", f"> {'y' * 200} …", "> three …"]) + self.assertEqual(trace(stream(said(f"one\ntwo\n{'y' * 300}\nfour"))), ["> one", "> two", f"> {'y' * 200} …"]) def test_a_stream_without_a_call_or_text_has_no_trace(self): self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), []) @@ -127,9 +137,13 @@ class DigestTest(unittest.TestCase): self.assertNotIn("failing run that", held) self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines()) + def test_the_median_is_by_tokens_whatever_the_order_the_runs_ran_in(self): + runs = [Run("bug-2-base-1", True, 30, b""), Run("bug-2-head-1", True, 10, b""), Run("bug-2-base-2", True, 20, b"")] + self.assertEqual(digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()[-2:], ["### bug-2-base-2, the passing run with the median tokens, 20", ""]) + def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self): held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True) - self.assertIn("0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", held.splitlines()) + self.assertEqual(held.splitlines()[10:14], ["0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", "", "### bug-3-base-1, the failing run that spent the most tokens, 10", ""]) self.assertIn(" at line 2", held.splitlines()) # Of two failing runs that spent the same, the first. self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines()) diff --git a/bin/task_bench/test_gather.py b/bin/task_bench/test_gather.py index 322d39eb..0aa8e96e 100644 --- a/bin/task_bench/test_gather.py +++ b/bin/task_bench/test_gather.py @@ -11,9 +11,9 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from failure import Failure -from task_bench import main from task_bench.gather import gather, PROMPT from test_env import isolated_env +from task_bench import main class GatherTest(unittest.TestCase): @@ -80,6 +80,16 @@ class GatherTest(unittest.TestCase): self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "claude hit a usage limit: You've hit your session limit") self.assertTrue((self.bench / "digest.md").exists()) + def test_a_digest_that_cannot_be_read_is_refused_with_the_reason(self): + (self.bench / "digest.md").unlink() + (self.bench / "digest.md").mkdir() + found = gather(self.env, self.home, self.claude) + self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"could not read {self.bench / 'digest.md'}: Is a directory", [])) + + def test_says_so_where_claude_ends_without_an_answer(self): + self.reply["result"] = "" + self.assertEqual(gather(self.env, self.home, self.claude)[0], "The gathering claude didn't say anything.") + def test_refuses_where_the_project_has_no_model_on_its_ladder(self): for saved, said in (({}, f"{self.models} names no model for calc: run loop-claude in the project once, which asks for it."), ({"calc": "claude-haiku-5-5"}, "calc runs on claude-haiku-5-5, which is not on its routing ladder.")): self.models.write_text(json.dumps(saved)) diff --git a/bin/task_bench/test_task_bench.py b/bin/task_bench/test_task_bench.py index bd03be58..56dec0ac 100644 --- a/bin/task_bench/test_task_bench.py +++ b/bin/task_bench/test_task_bench.py @@ -200,6 +200,7 @@ class BenchTest(unittest.TestCase): self.assertEqual(running, reports / "running/task-bench") # A project named task-bench has a review folder of that name, without a lock beside it. self.assertEqual([(reports / name / "task-bench").exists() for name in ("crashed", "running", "a-project")], [False, True, True]) + self.assertTrue((reports / "crashed/task-bench.lock").exists()) def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self): argv = [part for name, value in self.given.items() for part in (name, value)] @@ -371,6 +372,14 @@ class MainTest(unittest.TestCase): self.assertEqual(out.getvalue().count("usage: task-bench --image --test "), 2) + def test_a_run_that_did_not_finish_counts_as_spending_no_tokens_in_the_digest(self): + with tempfile.TemporaryDirectory() as folder: + pathlib.Path(folder, "runs").mkdir() + pathlib.Path(folder, "runs/bug-1-base-2.jsonl").write_bytes(b"{}") + task = main.Task(1, Mutant("calc.go", 4, 11, "ARITHMETIC_BASE"), {}, {"base": "boom", "head": "bang"}) + bug = main.digested(task, [Result(1, "base", 2, False, None), Result(2, "base", 1, True, None)], pathlib.Path(folder)) + self.assertEqual(bug, ("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "boom", [("bug-1-base-2", False, 0, b"{}")])) + def test_records_a_run_that_did_not_finish_without_figures(self): self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})