Test the digest's cuts, the median pick and gather's refusals, and drop project_rung's checkout parameter

This commit is contained in:
2026-11-21 04:24:52 +01:00
parent 7a568f80af
commit b474ec2f5a
6 changed files with 48 additions and 15 deletions
+1 -1
View File
@@ -8,8 +8,8 @@ import pathlib
import shutil
import typing
from chunk_driver.claims import locked
from failure import Failure
from chunk_driver.claims import locked
FOLDER = "task-bench"
# Beside FOLDER, not in it, since a bench empties its folder. It marks FOLDER as a bench's, and stays once the folder is gone.
+1 -1
View File
@@ -6,8 +6,8 @@ import re
import typing
from claude_code.entries import AssistantTurn, parse_entry, Text, tool_calls, ToolCall, ToolResult, ToolResults
from reading_cost.measure import reads_of
from task_bench.figures import HOME, WORK
from reading_cost.measure import reads_of
# How much of the model's text between two calls a trace keeps, and of each line, of a search's pattern or of a shell command.
TEXT_LINES = 3
+11 -11
View File
@@ -7,17 +7,17 @@ import shutil
import subprocess
import typing
from chunk_driver.project.model_choice import read_models_file
from claude_code.reply import Claude, claude_reply, Limited
from failure import Failure
from git_cli import git_repo
from git_cli.git_repo import Git, top
from loop_state.paths import paths_from
from model_routing.ladder import Rung
from model_routing.ladder_file import project_ladder
from review_rules.review_folder import review_folder
from git_cli import git_repo
from task_bench.bench_folder import held
from loop_state.paths import paths_from
from transcript_archive.project_key import project_key
from model_routing.ladder_file import project_ladder
from chunk_driver.project.model_choice import read_models_file
from review_rules.review_folder import review_folder
from model_routing.ladder import Rung
DIGEST = "digest.md"
PROMPT = pathlib.Path(__file__).with_name("gather-prompt.md")
@@ -30,7 +30,7 @@ def gather(env: typing.Mapping[str, str], home: pathlib.Path, claude: Claude) ->
checkout = top(git)
if isinstance(checkout, Failure):
return checkout
rung = project_rung(checkout, home, git)
rung = project_rung(home, git)
if isinstance(rung, Failure):
return rung
with held(pathlib.Path(review_folder(str(home), checkout))) as folder:
@@ -56,9 +56,9 @@ def run_claude(args: list[str], cwd: str, prompt: str) -> subprocess.CompletedPr
return subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=prompt, text=True)
def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure:
"""The cheapest rung of the model loop-claude runs the project on, on the project's routing ladder."""
common = git("rev-parse", "--path-format=absolute", "--git-common-dir", cwd=checkout)
def project_rung(home: pathlib.Path, git: Git) -> Rung | Failure:
"""The cheapest rung of the model loop-claude runs the project on, on the routing ladder of the project the process runs in."""
common = git("rev-parse", "--path-format=absolute", "--git-common-dir")
if isinstance(common, Failure):
return common
# A worktree's project is its main checkout's, which holds the folder every worktree shares.
@@ -69,7 +69,7 @@ def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure:
return models
if key not in models:
return Failure(f"{saved} names no model for {key}: run loop-claude in the project once, which asks for it.")
ladder = project_ladder(checkout, home)
ladder = project_ladder(None, home)
if isinstance(ladder, Failure):
return ladder
project = ladder.project_rung(models[key])
+15 -1
View File
@@ -49,6 +49,13 @@ class TraceTest(unittest.TestCase):
def test_a_shell_command_shows_its_first_line_cut_short(self):
self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 200 + "\nls"))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 120 + "\nls"))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 120))), [f"Bash {'x' * 120}"])
self.assertEqual(trace(stream(call("Bash", command="ls"), result("1", "a.go\nb.go"))), ["Bash ls"])
def test_a_call_without_its_pattern_or_command_shows_none(self):
self.assertEqual(trace(stream(call("Grep"), call("Bash", "2"))), ['Grep ""', "Bash "])
def test_any_other_tool_shows_its_name_and_the_file_it_names(self):
self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"])
@@ -61,6 +68,9 @@ class TraceTest(unittest.TestCase):
told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it."))
self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."])
self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"])
self.assertEqual(trace(stream(said("one\ntwo\nthree"))), ["> one", "> two", "> three"])
self.assertEqual(trace(stream(said(f"one\n{'y' * 300}\nthree\nfour"))), ["> one", f"> {'y' * 200} …", "> three …"])
self.assertEqual(trace(stream(said(f"one\ntwo\n{'y' * 300}\nfour"))), ["> one", "> two", f"> {'y' * 200} …"])
def test_a_stream_without_a_call_or_text_has_no_trace(self):
self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), [])
@@ -127,9 +137,13 @@ class DigestTest(unittest.TestCase):
self.assertNotIn("failing run that", held)
self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines())
def test_the_median_is_by_tokens_whatever_the_order_the_runs_ran_in(self):
runs = [Run("bug-2-base-1", True, 30, b""), Run("bug-2-head-1", True, 10, b""), Run("bug-2-base-2", True, 20, b"")]
self.assertEqual(digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()[-2:], ["### bug-2-base-2, the passing run with the median tokens, 20", ""])
def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self):
held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True)
self.assertIn("0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", held.splitlines())
self.assertEqual(held.splitlines()[10:14], ["0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", "", "### bug-3-base-1, the failing run that spent the most tokens, 10", ""])
self.assertIn(" at line 2", held.splitlines())
# Of two failing runs that spent the same, the first.
self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines())
+11 -1
View File
@@ -11,9 +11,9 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from task_bench import main
from task_bench.gather import gather, PROMPT
from test_env import isolated_env
from task_bench import main
class GatherTest(unittest.TestCase):
@@ -80,6 +80,16 @@ class GatherTest(unittest.TestCase):
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "claude hit a usage limit: You've hit your session limit")
self.assertTrue((self.bench / "digest.md").exists())
def test_a_digest_that_cannot_be_read_is_refused_with_the_reason(self):
(self.bench / "digest.md").unlink()
(self.bench / "digest.md").mkdir()
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"could not read {self.bench / 'digest.md'}: Is a directory", []))
def test_says_so_where_claude_ends_without_an_answer(self):
self.reply["result"] = ""
self.assertEqual(gather(self.env, self.home, self.claude)[0], "The gathering claude didn't say anything.")
def test_refuses_where_the_project_has_no_model_on_its_ladder(self):
for saved, said in (({}, f"{self.models} names no model for calc: run loop-claude in the project once, which asks for it."), ({"calc": "claude-haiku-5-5"}, "calc runs on claude-haiku-5-5, which is not on its routing ladder.")):
self.models.write_text(json.dumps(saved))
+9
View File
@@ -200,6 +200,7 @@ class BenchTest(unittest.TestCase):
self.assertEqual(running, reports / "running/task-bench")
# A project named task-bench has a review folder of that name, without a lock beside it.
self.assertEqual([(reports / name / "task-bench").exists() for name in ("crashed", "running", "a-project")], [False, True, True])
self.assertTrue((reports / "crashed/task-bench.lock").exists())
def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self):
argv = [part for name, value in self.given.items() for part in (name, value)]
@@ -371,6 +372,14 @@ class MainTest(unittest.TestCase):
self.assertEqual(out.getvalue().count("usage: task-bench --image <image> --test <shell text>"), 2)
def test_a_run_that_did_not_finish_counts_as_spending_no_tokens_in_the_digest(self):
with tempfile.TemporaryDirectory() as folder:
pathlib.Path(folder, "runs").mkdir()
pathlib.Path(folder, "runs/bug-1-base-2.jsonl").write_bytes(b"{}")
task = main.Task(1, Mutant("calc.go", 4, 11, "ARITHMETIC_BASE"), {}, {"base": "boom", "head": "bang"})
bug = main.digested(task, [Result(1, "base", 2, False, None), Result(2, "base", 1, True, None)], pathlib.Path(folder))
self.assertEqual(bug, ("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "boom", [("bug-1-base-2", False, 0, b"{}")]))
def test_records_a_run_that_did_not_finish_without_figures(self):
self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})