"""The digest task-bench distils from its runs' streams: a run's trace, a bug's decoys, and the two runs whose traces it shows.""" import json import pathlib import sys import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from task_bench.digest import Bug, decoys, digest, Run, trace def said(text: str) -> dict: return {"type": "assistant", "message": {"id": "msg_text", "content": [{"type": "text", "text": text}], "usage": {}}} def call(name: str, at: str = "1", **given: object) -> dict: return {"type": "assistant", "message": {"id": f"msg_{at}", "content": [{"type": "tool_use", "id": f"toolu_{at}", "name": name, "input": given}], "usage": {}}} def result(at: str, text: str, failed: bool = False) -> dict: return {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"toolu_{at}", "content": text, "is_error": failed}]}} def stream(*events: dict) -> bytes: return "\n".join(map(json.dumps, events)).encode() def reading(*paths: str) -> bytes: return stream(*(call("Read", str(at), file_path=f"/work/{path}") for at, path in enumerate(paths))) class TraceTest(unittest.TestCase): def test_a_read_names_the_file_in_the_project_and_the_lines_asked_for(self): self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120, limit=61))), ["Read pick.go:120-180"]) self.assertEqual(trace(stream(call("Read", file_path="/work/cmd/pick.go"))), ["Read cmd/pick.go"]) self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120))), ["Read pick.go:120-"]) self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", limit=40))), ["Read pick.go:1-40"]) self.assertEqual(trace(stream(call("Read", file_path="/etc/hosts"))), ["Read /etc/hosts"]) def test_a_search_names_its_pattern_and_how_many_lines_it_found(self): found = stream(call("Grep", pattern="readName"), result("1", "a.go:3: readName()\nb.go\n\nc.go\nd.go\n")) self.assertEqual(trace(found), ['Grep "readName" → 4 hits']) self.assertEqual(trace(stream(call("Grep", pattern="x", path="/work/cmd"), result("1", "No matches found"))), ['Grep "x" in cmd → 0 hits']) self.assertEqual(trace(stream(call("Glob", pattern="**/*.go"), result("1", "/work/a.go"))), ['Glob "**/*.go" → 1 hit']) self.assertEqual(trace(stream(call("Glob", pattern="*.rs"), result("1", "No files found"))), ['Glob "*.rs" → 0 hits']) # A search whose result the stream doesn't hold, as when the run ended there. self.assertEqual(trace(stream(call("Grep", pattern="x"))), ['Grep "x"']) def test_a_shell_command_shows_its_first_line_cut_short(self): self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"]) self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"]) def test_any_other_tool_shows_its_name_and_the_file_it_names(self): self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"]) def test_a_call_that_failed_says_so(self): self.assertEqual(trace(stream(call("Read", file_path="/work/gone.go"), result("1", "File does not exist.", failed=True))), ["Read gone.go → failed"]) self.assertEqual(trace(stream(call("Grep", pattern="x"), result("1", "bad pattern", failed=True))), ['Grep "x" → failed']) def test_the_models_text_between_calls_is_cut_to_three_lines(self): told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it.")) self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."]) self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"]) def test_a_stream_without_a_call_or_text_has_no_trace(self): self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), []) class DecoysTest(unittest.TestCase): def test_names_the_files_more_than_half_the_failing_runs_read_that_the_fix_leaves(self): failing = [reading("calc.go", "sum.go", "README.md"), reading("sum.go", "sum.go", "calc.go"), reading("README.md", "calc.go"), reading("go.mod")] # sum.go and README.md are each read by 2 of 4, which is not more than half; calc.go, the file the fix changes, by 3. self.assertEqual(decoys(failing, "calc.go", lambda path: True), []) self.assertEqual(decoys(failing[:3], "calc.go", lambda path: True), ["`README.md`, read by 2 of the 3 failing runs", "`sum.go`, read by 2 of the 3 failing runs"]) def test_counts_what_a_search_or_a_shell_command_read_and_leaves_out_folders_and_files_outside_the_project(self): failing = [stream(call("Grep", pattern="Add", path="/work/sum.go"), call("Bash", "2", command="cat /work/mul.go /etc/hosts"), call("Grep", "3", pattern="Add", path="/work/cmd"))] self.assertEqual(decoys(failing, "calc.go", lambda path: path != "cmd"), ["`mul.go`, read by 1 of the 1 failing run", "`sum.go`, read by 1 of the 1 failing run"]) def test_no_failing_run_gives_no_decoy(self): self.assertEqual(decoys([], "calc.go", lambda path: True), []) class DigestTest(unittest.TestCase): def runs(self) -> list[Run]: return [ Run("bug-1-base-1", False, 900, reading("sum.go")), Run("bug-1-head-1", True, 500, reading("calc.go")), Run("bug-1-base-2", False, 1200, reading("sum.go", "calc.go")), Run("bug-1-head-2", True, 300, reading("README.md", "calc.go")), Run("bug-1-base-3", True, 800, reading("go.mod", "calc.go")), Run("bug-1-head-3", True, 700, reading("mul.go", "calc.go")), ] def test_holds_the_figures_and_per_bug_its_decoys_and_two_traces_picked_by_rule(self): held = digest(["| Bug | Side |", "|---|---|"], [Bug("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "Add(1, 2) = -1, want 3", self.runs())], lambda path: True) self.assertEqual(held.splitlines(), [ "# task-bench digest", "", "| Bug | Side |", "|---|---|", "", "## Bug 1: calc.go:4:11 ARITHMETIC_BASE", "", "The failing test printed, at the base:", "", " Add(1, 2) = -1, want 3", "", "4 of the 6 runs passed. Decoys, the files more than half the failing runs read that the fix doesn't change:", "", "- `sum.go`, read by 2 of the 2 failing runs", "", "### bug-1-base-2, the failing run that spent the most tokens, 1200", "", "Read sum.go", "Read calc.go", "", # Of the passing runs' 300, 500, 700 and 800, the lower of the two in the middle. "### bug-1-head-1, the passing run with the median tokens, 500", "", "Read calc.go", ]) def test_a_bug_every_run_fixed_has_no_decoys_and_one_trace(self): held = digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", [Run("bug-2-base-1", True, 10, reading("a.go"))])], lambda path: True) self.assertIn("The 1 run passed. No run failed, so the bug has no decoys.", held.splitlines()) self.assertNotIn("failing run that", held) self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines()) def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self): held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True) self.assertIn("0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", held.splitlines()) self.assertIn(" at line 2", held.splitlines()) # Of two failing runs that spent the same, the first. self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines()) self.assertNotIn("passing run with", held) if __name__ == "__main__": unittest.main()