Files
claude-config/bin/task_bench/test_digest.py
T

155 lines
9.0 KiB
Python

"""The digest task-bench distils from its runs' streams: a run's trace, a bug's decoys, and the two runs whose traces it shows."""
import json
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from task_bench.digest import Bug, decoys, digest, Run, trace
def said(text: str) -> dict:
return {"type": "assistant", "message": {"id": "msg_text", "content": [{"type": "text", "text": text}], "usage": {}}}
def call(name: str, at: str = "1", **given: object) -> dict:
return {"type": "assistant", "message": {"id": f"msg_{at}", "content": [{"type": "tool_use", "id": f"toolu_{at}", "name": name, "input": given}], "usage": {}}}
def result(at: str, text: str, failed: bool = False) -> dict:
return {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"toolu_{at}", "content": text, "is_error": failed}]}}
def stream(*events: dict) -> bytes:
return "\n".join(map(json.dumps, events)).encode()
def reading(*paths: str) -> bytes:
return stream(*(call("Read", str(at), file_path=f"/work/{path}") for at, path in enumerate(paths)))
class TraceTest(unittest.TestCase):
def test_a_read_names_the_file_in_the_project_and_the_lines_asked_for(self):
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120, limit=61))), ["Read pick.go:120-180"])
self.assertEqual(trace(stream(call("Read", file_path="/work/cmd/pick.go"))), ["Read cmd/pick.go"])
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120))), ["Read pick.go:120-"])
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", limit=40))), ["Read pick.go:1-40"])
self.assertEqual(trace(stream(call("Read", file_path="/etc/hosts"))), ["Read /etc/hosts"])
def test_a_search_names_its_pattern_and_how_many_lines_it_found(self):
found = stream(call("Grep", pattern="readName"), result("1", "a.go:3: readName()\nb.go\n\nc.go\nd.go\n"))
self.assertEqual(trace(found), ['Grep "readName" → 4 hits'])
self.assertEqual(trace(stream(call("Grep", pattern="x", path="/work/cmd"), result("1", "No matches found"))), ['Grep "x" in cmd → 0 hits'])
self.assertEqual(trace(stream(call("Glob", pattern="**/*.go"), result("1", "/work/a.go"))), ['Glob "**/*.go" → 1 hit'])
self.assertEqual(trace(stream(call("Glob", pattern="*.rs"), result("1", "No files found"))), ['Glob "*.rs" → 0 hits'])
# A search whose result the stream doesn't hold, as when the run ended there.
self.assertEqual(trace(stream(call("Grep", pattern="x"))), ['Grep "x"'])
def test_a_shell_command_shows_its_first_line_cut_short(self):
self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 200 + "\nls"))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 120 + "\nls"))), [f"Bash {'x' * 120} …"])
self.assertEqual(trace(stream(call("Bash", command="x" * 120))), [f"Bash {'x' * 120}"])
self.assertEqual(trace(stream(call("Bash", command="ls"), result("1", "a.go\nb.go"))), ["Bash ls"])
def test_a_call_without_its_pattern_or_command_shows_none(self):
self.assertEqual(trace(stream(call("Grep"), call("Bash", "2"))), ['Grep ""', "Bash "])
def test_any_other_tool_shows_its_name_and_the_file_it_names(self):
self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"])
def test_a_call_that_failed_says_so(self):
self.assertEqual(trace(stream(call("Read", file_path="/work/gone.go"), result("1", "File does not exist.", failed=True))), ["Read gone.go → failed"])
self.assertEqual(trace(stream(call("Grep", pattern="x"), result("1", "bad pattern", failed=True))), ['Grep "x" → failed'])
def test_the_models_text_between_calls_is_cut_to_three_lines(self):
told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it."))
self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."])
self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"])
self.assertEqual(trace(stream(said("one\ntwo\nthree"))), ["> one", "> two", "> three"])
self.assertEqual(trace(stream(said(f"one\n{'y' * 300}\nthree\nfour"))), ["> one", f"> {'y' * 200} …", "> three …"])
self.assertEqual(trace(stream(said(f"one\ntwo\n{'y' * 300}\nfour"))), ["> one", "> two", f"> {'y' * 200} …"])
def test_a_stream_without_a_call_or_text_has_no_trace(self):
self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), [])
class DecoysTest(unittest.TestCase):
def test_names_the_files_more_than_half_the_failing_runs_read_that_the_fix_leaves(self):
failing = [reading("calc.go", "sum.go", "README.md"), reading("sum.go", "sum.go", "calc.go"), reading("README.md", "calc.go"), reading("go.mod")]
# sum.go and README.md are each read by 2 of 4, which is not more than half; calc.go, the file the fix changes, by 3.
self.assertEqual(decoys(failing, "calc.go", lambda path: True), [])
self.assertEqual(decoys(failing[:3], "calc.go", lambda path: True), ["`README.md`, read by 2 of the 3 failing runs", "`sum.go`, read by 2 of the 3 failing runs"])
def test_counts_what_a_search_or_a_shell_command_read_and_leaves_out_folders_and_files_outside_the_project(self):
failing = [stream(call("Grep", pattern="Add", path="/work/sum.go"), call("Bash", "2", command="cat /work/mul.go /etc/hosts"), call("Grep", "3", pattern="Add", path="/work/cmd"))]
self.assertEqual(decoys(failing, "calc.go", lambda path: path != "cmd"), ["`mul.go`, read by 1 of the 1 failing run", "`sum.go`, read by 1 of the 1 failing run"])
def test_no_failing_run_gives_no_decoy(self):
self.assertEqual(decoys([], "calc.go", lambda path: True), [])
class DigestTest(unittest.TestCase):
def runs(self) -> list[Run]:
return [
Run("bug-1-base-1", False, 900, reading("sum.go")),
Run("bug-1-head-1", True, 500, reading("calc.go")),
Run("bug-1-base-2", False, 1200, reading("sum.go", "calc.go")),
Run("bug-1-head-2", True, 300, reading("README.md", "calc.go")),
Run("bug-1-base-3", True, 800, reading("go.mod", "calc.go")),
Run("bug-1-head-3", True, 700, reading("mul.go", "calc.go")),
]
def test_holds_the_figures_and_per_bug_its_decoys_and_two_traces_picked_by_rule(self):
held = digest(["| Bug | Side |", "|---|---|"], [Bug("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "Add(1, 2) = -1, want 3", self.runs())], lambda path: True)
self.assertEqual(held.splitlines(), [
"# task-bench digest",
"",
"| Bug | Side |",
"|---|---|",
"",
"## Bug 1: calc.go:4:11 ARITHMETIC_BASE",
"",
"The failing test printed, at the base:",
"",
" Add(1, 2) = -1, want 3",
"",
"4 of the 6 runs passed. Decoys, the files more than half the failing runs read that the fix doesn't change:",
"",
"- `sum.go`, read by 2 of the 2 failing runs",
"",
"### bug-1-base-2, the failing run that spent the most tokens, 1200",
"",
"Read sum.go",
"Read calc.go",
"",
# Of the passing runs' 300, 500, 700 and 800, the lower of the two in the middle.
"### bug-1-head-1, the passing run with the median tokens, 500",
"",
"Read calc.go",
])
def test_a_bug_every_run_fixed_has_no_decoys_and_one_trace(self):
held = digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", [Run("bug-2-base-1", True, 10, reading("a.go"))])], lambda path: True)
self.assertIn("The 1 run passed. No run failed, so the bug has no decoys.", held.splitlines())
self.assertNotIn("failing run that", held)
self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines())
def test_the_median_is_by_tokens_whatever_the_order_the_runs_ran_in(self):
runs = [Run("bug-2-base-1", True, 20, b""), Run("bug-2-head-1", True, 10, b""), Run("bug-2-base-2", True, 30, b"")]
self.assertEqual(digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()[-2:], ["### bug-2-base-1, the passing run with the median tokens, 20", ""])
def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self):
held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True)
self.assertEqual(held.splitlines()[10:14], ["0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", "", "### bug-3-base-1, the failing run that spent the most tokens, 10", ""])
self.assertIn(" at line 2", held.splitlines())
# Of two failing runs that spent the same, the first.
self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines())
self.assertNotIn("passing run with", held)
if __name__ == "__main__":
unittest.main()