Test the digest's cuts, the median pick and gather's refusals, and drop project_rung's checkout parameter
This commit is contained in:
@@ -8,8 +8,8 @@ import pathlib
|
||||
import shutil
|
||||
import typing
|
||||
|
||||
from chunk_driver.claims import locked
|
||||
from failure import Failure
|
||||
from chunk_driver.claims import locked
|
||||
|
||||
FOLDER = "task-bench"
|
||||
# Beside FOLDER, not in it, since a bench empties its folder. It marks FOLDER as a bench's, and stays once the folder is gone.
|
||||
|
||||
@@ -6,8 +6,8 @@ import re
|
||||
import typing
|
||||
|
||||
from claude_code.entries import AssistantTurn, parse_entry, Text, tool_calls, ToolCall, ToolResult, ToolResults
|
||||
from reading_cost.measure import reads_of
|
||||
from task_bench.figures import HOME, WORK
|
||||
from reading_cost.measure import reads_of
|
||||
|
||||
# How much of the model's text between two calls a trace keeps, and of each line, of a search's pattern or of a shell command.
|
||||
TEXT_LINES = 3
|
||||
|
||||
+11
-11
@@ -7,17 +7,17 @@ import shutil
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
from chunk_driver.project.model_choice import read_models_file
|
||||
from claude_code.reply import Claude, claude_reply, Limited
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from git_cli.git_repo import Git, top
|
||||
from loop_state.paths import paths_from
|
||||
from model_routing.ladder import Rung
|
||||
from model_routing.ladder_file import project_ladder
|
||||
from review_rules.review_folder import review_folder
|
||||
from git_cli import git_repo
|
||||
from task_bench.bench_folder import held
|
||||
from loop_state.paths import paths_from
|
||||
from transcript_archive.project_key import project_key
|
||||
from model_routing.ladder_file import project_ladder
|
||||
from chunk_driver.project.model_choice import read_models_file
|
||||
from review_rules.review_folder import review_folder
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
DIGEST = "digest.md"
|
||||
PROMPT = pathlib.Path(__file__).with_name("gather-prompt.md")
|
||||
@@ -30,7 +30,7 @@ def gather(env: typing.Mapping[str, str], home: pathlib.Path, claude: Claude) ->
|
||||
checkout = top(git)
|
||||
if isinstance(checkout, Failure):
|
||||
return checkout
|
||||
rung = project_rung(checkout, home, git)
|
||||
rung = project_rung(home, git)
|
||||
if isinstance(rung, Failure):
|
||||
return rung
|
||||
with held(pathlib.Path(review_folder(str(home), checkout))) as folder:
|
||||
@@ -56,9 +56,9 @@ def run_claude(args: list[str], cwd: str, prompt: str) -> subprocess.CompletedPr
|
||||
return subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=prompt, text=True)
|
||||
|
||||
|
||||
def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure:
|
||||
"""The cheapest rung of the model loop-claude runs the project on, on the project's routing ladder."""
|
||||
common = git("rev-parse", "--path-format=absolute", "--git-common-dir", cwd=checkout)
|
||||
def project_rung(home: pathlib.Path, git: Git) -> Rung | Failure:
|
||||
"""The cheapest rung of the model loop-claude runs the project on, on the routing ladder of the project the process runs in."""
|
||||
common = git("rev-parse", "--path-format=absolute", "--git-common-dir")
|
||||
if isinstance(common, Failure):
|
||||
return common
|
||||
# A worktree's project is its main checkout's, which holds the folder every worktree shares.
|
||||
@@ -69,7 +69,7 @@ def project_rung(checkout: str, home: pathlib.Path, git: Git) -> Rung | Failure:
|
||||
return models
|
||||
if key not in models:
|
||||
return Failure(f"{saved} names no model for {key}: run loop-claude in the project once, which asks for it.")
|
||||
ladder = project_ladder(checkout, home)
|
||||
ladder = project_ladder(None, home)
|
||||
if isinstance(ladder, Failure):
|
||||
return ladder
|
||||
project = ladder.project_rung(models[key])
|
||||
|
||||
@@ -49,6 +49,13 @@ class TraceTest(unittest.TestCase):
|
||||
def test_a_shell_command_shows_its_first_line_cut_short(self):
|
||||
self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"])
|
||||
self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"])
|
||||
self.assertEqual(trace(stream(call("Bash", command="x" * 200 + "\nls"))), [f"Bash {'x' * 120} …"])
|
||||
self.assertEqual(trace(stream(call("Bash", command="x" * 120 + "\nls"))), [f"Bash {'x' * 120} …"])
|
||||
self.assertEqual(trace(stream(call("Bash", command="x" * 120))), [f"Bash {'x' * 120}"])
|
||||
self.assertEqual(trace(stream(call("Bash", command="ls"), result("1", "a.go\nb.go"))), ["Bash ls"])
|
||||
|
||||
def test_a_call_without_its_pattern_or_command_shows_none(self):
|
||||
self.assertEqual(trace(stream(call("Grep"), call("Bash", "2"))), ['Grep ""', "Bash "])
|
||||
|
||||
def test_any_other_tool_shows_its_name_and_the_file_it_names(self):
|
||||
self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"])
|
||||
@@ -61,6 +68,9 @@ class TraceTest(unittest.TestCase):
|
||||
told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it."))
|
||||
self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."])
|
||||
self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"])
|
||||
self.assertEqual(trace(stream(said("one\ntwo\nthree"))), ["> one", "> two", "> three"])
|
||||
self.assertEqual(trace(stream(said(f"one\n{'y' * 300}\nthree\nfour"))), ["> one", f"> {'y' * 200} …", "> three …"])
|
||||
self.assertEqual(trace(stream(said(f"one\ntwo\n{'y' * 300}\nfour"))), ["> one", "> two", f"> {'y' * 200} …"])
|
||||
|
||||
def test_a_stream_without_a_call_or_text_has_no_trace(self):
|
||||
self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), [])
|
||||
@@ -127,9 +137,13 @@ class DigestTest(unittest.TestCase):
|
||||
self.assertNotIn("failing run that", held)
|
||||
self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines())
|
||||
|
||||
def test_the_median_is_by_tokens_whatever_the_order_the_runs_ran_in(self):
|
||||
runs = [Run("bug-2-base-1", True, 30, b""), Run("bug-2-head-1", True, 10, b""), Run("bug-2-base-2", True, 20, b"")]
|
||||
self.assertEqual(digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()[-2:], ["### bug-2-base-2, the passing run with the median tokens, 20", ""])
|
||||
|
||||
def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self):
|
||||
held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True)
|
||||
self.assertIn("0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", held.splitlines())
|
||||
self.assertEqual(held.splitlines()[10:14], ["0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", "", "### bug-3-base-1, the failing run that spent the most tokens, 10", ""])
|
||||
self.assertIn(" at line 2", held.splitlines())
|
||||
# Of two failing runs that spent the same, the first.
|
||||
self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines())
|
||||
|
||||
@@ -11,9 +11,9 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from failure import Failure
|
||||
from task_bench import main
|
||||
from task_bench.gather import gather, PROMPT
|
||||
from test_env import isolated_env
|
||||
from task_bench import main
|
||||
|
||||
|
||||
class GatherTest(unittest.TestCase):
|
||||
@@ -80,6 +80,16 @@ class GatherTest(unittest.TestCase):
|
||||
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "claude hit a usage limit: You've hit your session limit")
|
||||
self.assertTrue((self.bench / "digest.md").exists())
|
||||
|
||||
def test_a_digest_that_cannot_be_read_is_refused_with_the_reason(self):
|
||||
(self.bench / "digest.md").unlink()
|
||||
(self.bench / "digest.md").mkdir()
|
||||
found = gather(self.env, self.home, self.claude)
|
||||
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"could not read {self.bench / 'digest.md'}: Is a directory", []))
|
||||
|
||||
def test_says_so_where_claude_ends_without_an_answer(self):
|
||||
self.reply["result"] = ""
|
||||
self.assertEqual(gather(self.env, self.home, self.claude)[0], "The gathering claude didn't say anything.")
|
||||
|
||||
def test_refuses_where_the_project_has_no_model_on_its_ladder(self):
|
||||
for saved, said in (({}, f"{self.models} names no model for calc: run loop-claude in the project once, which asks for it."), ({"calc": "claude-haiku-5-5"}, "calc runs on claude-haiku-5-5, which is not on its routing ladder.")):
|
||||
self.models.write_text(json.dumps(saved))
|
||||
|
||||
@@ -200,6 +200,7 @@ class BenchTest(unittest.TestCase):
|
||||
self.assertEqual(running, reports / "running/task-bench")
|
||||
# A project named task-bench has a review folder of that name, without a lock beside it.
|
||||
self.assertEqual([(reports / name / "task-bench").exists() for name in ("crashed", "running", "a-project")], [False, True, True])
|
||||
self.assertTrue((reports / "crashed/task-bench.lock").exists())
|
||||
|
||||
def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self):
|
||||
argv = [part for name, value in self.given.items() for part in (name, value)]
|
||||
@@ -371,6 +372,14 @@ class MainTest(unittest.TestCase):
|
||||
self.assertEqual(out.getvalue().count("usage: task-bench --image <image> --test <shell text>"), 2)
|
||||
|
||||
|
||||
def test_a_run_that_did_not_finish_counts_as_spending_no_tokens_in_the_digest(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
pathlib.Path(folder, "runs").mkdir()
|
||||
pathlib.Path(folder, "runs/bug-1-base-2.jsonl").write_bytes(b"{}")
|
||||
task = main.Task(1, Mutant("calc.go", 4, 11, "ARITHMETIC_BASE"), {}, {"base": "boom", "head": "bang"})
|
||||
bug = main.digested(task, [Result(1, "base", 2, False, None), Result(2, "base", 1, True, None)], pathlib.Path(folder))
|
||||
self.assertEqual(bug, ("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "boom", [("bug-1-base-2", False, 0, b"{}")]))
|
||||
|
||||
def test_records_a_run_that_did_not_finish_without_figures(self):
|
||||
self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user