141 lines
7.7 KiB
Python
141 lines
7.7 KiB
Python
"""The digest task-bench distils from its runs' streams: a run's trace, a bug's decoys, and the two runs whose traces it shows."""
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from task_bench.digest import Bug, decoys, digest, Run, trace
|
|
|
|
|
|
def said(text: str) -> dict:
|
|
return {"type": "assistant", "message": {"id": "msg_text", "content": [{"type": "text", "text": text}], "usage": {}}}
|
|
|
|
|
|
def call(name: str, at: str = "1", **given: object) -> dict:
|
|
return {"type": "assistant", "message": {"id": f"msg_{at}", "content": [{"type": "tool_use", "id": f"toolu_{at}", "name": name, "input": given}], "usage": {}}}
|
|
|
|
|
|
def result(at: str, text: str, failed: bool = False) -> dict:
|
|
return {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"toolu_{at}", "content": text, "is_error": failed}]}}
|
|
|
|
|
|
def stream(*events: dict) -> bytes:
|
|
return "\n".join(map(json.dumps, events)).encode()
|
|
|
|
|
|
def reading(*paths: str) -> bytes:
|
|
return stream(*(call("Read", str(at), file_path=f"/work/{path}") for at, path in enumerate(paths)))
|
|
|
|
|
|
class TraceTest(unittest.TestCase):
|
|
def test_a_read_names_the_file_in_the_project_and_the_lines_asked_for(self):
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120, limit=61))), ["Read pick.go:120-180"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/cmd/pick.go"))), ["Read cmd/pick.go"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120))), ["Read pick.go:120-"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", limit=40))), ["Read pick.go:1-40"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/etc/hosts"))), ["Read /etc/hosts"])
|
|
|
|
def test_a_search_names_its_pattern_and_how_many_lines_it_found(self):
|
|
found = stream(call("Grep", pattern="readName"), result("1", "a.go:3: readName()\nb.go\n\nc.go\nd.go\n"))
|
|
self.assertEqual(trace(found), ['Grep "readName" → 4 hits'])
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x", path="/work/cmd"), result("1", "No matches found"))), ['Grep "x" in cmd → 0 hits'])
|
|
self.assertEqual(trace(stream(call("Glob", pattern="**/*.go"), result("1", "/work/a.go"))), ['Glob "**/*.go" → 1 hit'])
|
|
self.assertEqual(trace(stream(call("Glob", pattern="*.rs"), result("1", "No files found"))), ['Glob "*.rs" → 0 hits'])
|
|
# A search whose result the stream doesn't hold, as when the run ended there.
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x"))), ['Grep "x"'])
|
|
|
|
def test_a_shell_command_shows_its_first_line_cut_short(self):
|
|
self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"])
|
|
self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"])
|
|
|
|
def test_any_other_tool_shows_its_name_and_the_file_it_names(self):
|
|
self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"])
|
|
|
|
def test_a_call_that_failed_says_so(self):
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/gone.go"), result("1", "File does not exist.", failed=True))), ["Read gone.go → failed"])
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x"), result("1", "bad pattern", failed=True))), ['Grep "x" → failed'])
|
|
|
|
def test_the_models_text_between_calls_is_cut_to_three_lines(self):
|
|
told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it."))
|
|
self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."])
|
|
self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"])
|
|
|
|
def test_a_stream_without_a_call_or_text_has_no_trace(self):
|
|
self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), [])
|
|
|
|
|
|
class DecoysTest(unittest.TestCase):
|
|
def test_names_the_files_more_than_half_the_failing_runs_read_that_the_fix_leaves(self):
|
|
failing = [reading("calc.go", "sum.go", "README.md"), reading("sum.go", "sum.go", "calc.go"), reading("README.md", "calc.go"), reading("go.mod")]
|
|
# sum.go and README.md are each read by 2 of 4, which is not more than half; calc.go, the file the fix changes, by 3.
|
|
self.assertEqual(decoys(failing, "calc.go", lambda path: True), [])
|
|
self.assertEqual(decoys(failing[:3], "calc.go", lambda path: True), ["`README.md`, read by 2 of the 3 failing runs", "`sum.go`, read by 2 of the 3 failing runs"])
|
|
|
|
def test_counts_what_a_search_or_a_shell_command_read_and_leaves_out_folders_and_files_outside_the_project(self):
|
|
failing = [stream(call("Grep", pattern="Add", path="/work/sum.go"), call("Bash", "2", command="cat /work/mul.go /etc/hosts"), call("Grep", "3", pattern="Add", path="/work/cmd"))]
|
|
self.assertEqual(decoys(failing, "calc.go", lambda path: path != "cmd"), ["`mul.go`, read by 1 of the 1 failing run", "`sum.go`, read by 1 of the 1 failing run"])
|
|
|
|
def test_no_failing_run_gives_no_decoy(self):
|
|
self.assertEqual(decoys([], "calc.go", lambda path: True), [])
|
|
|
|
|
|
class DigestTest(unittest.TestCase):
|
|
def runs(self) -> list[Run]:
|
|
return [
|
|
Run("bug-1-base-1", False, 900, reading("sum.go")),
|
|
Run("bug-1-head-1", True, 500, reading("calc.go")),
|
|
Run("bug-1-base-2", False, 1200, reading("sum.go", "calc.go")),
|
|
Run("bug-1-head-2", True, 300, reading("README.md", "calc.go")),
|
|
Run("bug-1-base-3", True, 800, reading("go.mod", "calc.go")),
|
|
Run("bug-1-head-3", True, 700, reading("mul.go", "calc.go")),
|
|
]
|
|
|
|
def test_holds_the_figures_and_per_bug_its_decoys_and_two_traces_picked_by_rule(self):
|
|
held = digest(["| Bug | Side |", "|---|---|"], [Bug("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "Add(1, 2) = -1, want 3", self.runs())], lambda path: True)
|
|
self.assertEqual(held.splitlines(), [
|
|
"# task-bench digest",
|
|
"",
|
|
"| Bug | Side |",
|
|
"|---|---|",
|
|
"",
|
|
"## Bug 1: calc.go:4:11 ARITHMETIC_BASE",
|
|
"",
|
|
"The failing test printed, at the base:",
|
|
"",
|
|
" Add(1, 2) = -1, want 3",
|
|
"",
|
|
"4 of the 6 runs passed. Decoys, the files more than half the failing runs read that the fix doesn't change:",
|
|
"",
|
|
"- `sum.go`, read by 2 of the 2 failing runs",
|
|
"",
|
|
"### bug-1-base-2, the failing run that spent the most tokens, 1200",
|
|
"",
|
|
"Read sum.go",
|
|
"Read calc.go",
|
|
"",
|
|
# Of the passing runs' 300, 500, 700 and 800, the lower of the two in the middle.
|
|
"### bug-1-head-1, the passing run with the median tokens, 500",
|
|
"",
|
|
"Read calc.go",
|
|
])
|
|
|
|
def test_a_bug_every_run_fixed_has_no_decoys_and_one_trace(self):
|
|
held = digest([], [Bug("2: a.go:1:1 X", "a.go", "boom", [Run("bug-2-base-1", True, 10, reading("a.go"))])], lambda path: True)
|
|
self.assertIn("The 1 run passed. No run failed, so the bug has no decoys.", held.splitlines())
|
|
self.assertNotIn("failing run that", held)
|
|
self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10", held.splitlines())
|
|
|
|
def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self):
|
|
held = digest([], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True)
|
|
self.assertIn("0 of the 2 runs passed. No decoys: no file but the one the fix changes was read by more than half the 2 failing runs.", held.splitlines())
|
|
self.assertIn(" at line 2", held.splitlines())
|
|
# Of two failing runs that spent the same, the first.
|
|
self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10", held.splitlines())
|
|
self.assertNotIn("passing run with", held)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|