167 lines
10 KiB
Python
167 lines
10 KiB
Python
"""The digest task-bench distils from its runs' streams: a run's trace, a bug's decoys, and the two runs whose traces it shows."""
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from task_bench.digest import Bug, decoys, digest, Run, trace
|
|
|
|
|
|
def said(text: str) -> dict:
|
|
return {"type": "assistant", "message": {"id": "msg_text", "content": [{"type": "text", "text": text}], "usage": {}}}
|
|
|
|
|
|
def call(name: str, at: str = "1", **given: object) -> dict:
|
|
return {"type": "assistant", "message": {"id": f"msg_{at}", "content": [{"type": "tool_use", "id": f"toolu_{at}", "name": name, "input": given}], "usage": {}}}
|
|
|
|
|
|
def result(at: str, text: str, failed: bool = False) -> dict:
|
|
return {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"toolu_{at}", "content": text, "is_error": failed}]}}
|
|
|
|
|
|
def stream(*events: dict) -> bytes:
|
|
return "\n".join(map(json.dumps, events)).encode()
|
|
|
|
|
|
def reading(*paths: str) -> bytes:
|
|
return stream(*(call("Read", str(at), file_path=f"/work/{path}") for at, path in enumerate(paths)))
|
|
|
|
|
|
SIDES = {"base": "b" * 40, "head": "h" * 40}
|
|
|
|
|
|
class TraceTest(unittest.TestCase):
|
|
def test_a_read_names_the_file_in_the_project_and_the_lines_asked_for(self):
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120, limit=61))), ["Read pick.go:120-180"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/cmd/pick.go"))), ["Read cmd/pick.go"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", offset=120))), ["Read pick.go:120-"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/pick.go", limit=40))), ["Read pick.go:1-40"])
|
|
self.assertEqual(trace(stream(call("Read", file_path="/etc/hosts"))), ["Read /etc/hosts"])
|
|
|
|
def test_a_search_names_its_pattern_and_how_many_lines_it_found(self):
|
|
found = stream(call("Grep", pattern="readName"), result("1", "Found 3 files\na.go\nb.go\nc.go"))
|
|
self.assertEqual(trace(found), ['Grep "readName" → 3 hits'])
|
|
for printed, count in (("a.go:3: readName()\n\nb.go:9: readName()\n[Showing results with pagination = limit: 2]", "2 hits"), ("No matches found\nFound 0 total occurrences across 0 files.", "0 hits"), ("No entries at this offset. [Showing results with pagination = limit: 3]", "0 hits"), ("/work/a.go\n(Showing 1 of 240 matching files; narrow the pattern)", "1 hit")):
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x"), result("1", printed))), [f'Grep "x" → {count}'])
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x", path="/work/cmd"), result("1", "No matches found"))), ['Grep "x" in cmd → 0 hits'])
|
|
self.assertEqual(trace(stream(call("Glob", pattern="**/*.go"), result("1", "/work/a.go"))), ['Glob "**/*.go" → 1 hit'])
|
|
self.assertEqual(trace(stream(call("Glob", pattern="*.rs"), result("1", "No files found"))), ['Glob "*.rs" → 0 hits'])
|
|
# A search whose result the stream doesn't hold, as when the run ended there.
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x"))), ['Grep "x"'])
|
|
|
|
def test_a_shell_command_shows_its_first_line_cut_short(self):
|
|
self.assertEqual(trace(stream(call("Bash", command="go test ./...\necho done"))), ["Bash go test ./... …"])
|
|
self.assertEqual(trace(stream(call("Bash", command="x" * 200))), [f"Bash {'x' * 120} …"])
|
|
self.assertEqual(trace(stream(call("Bash", command="x" * 200 + "\nls"))), [f"Bash {'x' * 120} …"])
|
|
self.assertEqual(trace(stream(call("Bash", command="x" * 120 + "\nls"))), [f"Bash {'x' * 120} …"])
|
|
self.assertEqual(trace(stream(call("Bash", command="x" * 120))), [f"Bash {'x' * 120}"])
|
|
self.assertEqual(trace(stream(call("Bash", command="ls"), result("1", "a.go\nb.go"))), ["Bash ls"])
|
|
|
|
def test_a_call_without_its_pattern_or_command_shows_none(self):
|
|
self.assertEqual(trace(stream(call("Grep"), call("Bash", "2"))), ['Grep ""', "Bash "])
|
|
|
|
def test_any_other_tool_shows_its_name_and_the_file_it_names(self):
|
|
self.assertEqual(trace(stream(call("Edit", file_path="/work/pick.go", old_string="a", new_string="b"), call("TodoWrite", "2", todos=[]))), ["Edit pick.go", "TodoWrite"])
|
|
|
|
def test_a_call_that_failed_says_so(self):
|
|
self.assertEqual(trace(stream(call("Read", file_path="/work/gone.go"), result("1", "File does not exist.", failed=True))), ["Read gone.go → failed"])
|
|
self.assertEqual(trace(stream(call("Grep", pattern="x"), result("1", "bad pattern", failed=True))), ['Grep "x" → failed'])
|
|
|
|
def test_the_models_text_between_calls_is_cut_to_three_lines(self):
|
|
told = stream(said("The test fails in Add.\n\nI will read it.\nThen the caller.\nThen fix it."), call("Read", file_path="/work/calc.go"), said("Found it."))
|
|
self.assertEqual(trace(told), ["> The test fails in Add.", "> I will read it.", "> Then the caller. …", "Read calc.go", "> Found it."])
|
|
self.assertEqual(trace(stream(said("y" * 300))), [f"> {'y' * 200} …"])
|
|
self.assertEqual(trace(stream(said("one\ntwo\nthree"))), ["> one", "> two", "> three"])
|
|
self.assertEqual(trace(stream(said(f"one\n{'y' * 300}\nthree\nfour"))), ["> one", f"> {'y' * 200} …", "> three …"])
|
|
self.assertEqual(trace(stream(said(f"one\ntwo\n{'y' * 300}\nfour"))), ["> one", "> two", f"> {'y' * 200} …"])
|
|
|
|
def test_a_stream_without_a_call_or_text_has_no_trace(self):
|
|
self.assertEqual(trace(b'{"type": "system", "subtype": "init"}\nnot json\n'), [])
|
|
|
|
|
|
class DecoysTest(unittest.TestCase):
|
|
def test_names_the_files_more_than_half_the_failing_runs_read_that_the_fix_leaves(self):
|
|
failing = [reading("calc.go", "sum.go", "README.md"), reading("sum.go", "sum.go", "calc.go"), reading("README.md", "calc.go"), reading("go.mod")]
|
|
# sum.go and README.md are each read by 2 of 4, which is not more than half; calc.go, the file the fix changes, by 3.
|
|
self.assertEqual(decoys(failing, "calc.go", lambda path: True), [])
|
|
self.assertEqual(decoys(failing[:3], "calc.go", lambda path: True), ["`README.md`, read by 2 of the 3 failing runs", "`sum.go`, read by 2 of the 3 failing runs"])
|
|
|
|
def test_counts_what_a_search_or_a_shell_command_read_and_leaves_out_folders_and_files_outside_the_project(self):
|
|
failing = [stream(call("Grep", pattern="Add", path="/work/sum.go"), call("Bash", "2", command="cat /work/mul.go /etc/hosts ~/notes.go"), call("Grep", "3", pattern="Add", path="/work/cmd"))]
|
|
self.assertEqual(decoys(failing, "calc.go", lambda path: path != "cmd"), ["`mul.go`, read by the one failing run", "`sum.go`, read by the one failing run"])
|
|
|
|
def test_no_failing_run_gives_no_decoy(self):
|
|
self.assertEqual(decoys([], "calc.go", lambda path: True), [])
|
|
|
|
|
|
class DigestTest(unittest.TestCase):
|
|
def runs(self) -> list[Run]:
|
|
return [
|
|
Run("bug-1-base-1", False, 900, reading("sum.go")),
|
|
Run("bug-1-head-1", True, 500, reading("calc.go")),
|
|
Run("bug-1-base-2", False, 1200, reading("sum.go", "calc.go")),
|
|
Run("bug-1-head-2", True, 300, reading("README.md", "calc.go")),
|
|
Run("bug-1-base-3", True, 800, reading("go.mod", "calc.go")),
|
|
Run("bug-1-head-3", True, 700, reading("mul.go", "calc.go")),
|
|
]
|
|
|
|
def test_holds_the_figures_and_per_bug_its_decoys_and_two_traces_picked_by_rule(self):
|
|
held = digest(SIDES, ["| Bug | Side |", "|---|---|"], [Bug("1: calc.go:4:11 ARITHMETIC_BASE", "calc.go", "Add(1, 2) = -1, want 3", self.runs())], lambda path: True)
|
|
self.assertEqual(held.splitlines(), [
|
|
"# task-bench digest",
|
|
"",
|
|
f"Base {'b' * 40}, head {'h' * 40}. A bug's name gives its line at the base. Each side's code without the bugs is in `trees/base` and `trees/head` in the bench's folder.",
|
|
"",
|
|
"| Bug | Side |",
|
|
"|---|---|",
|
|
"",
|
|
"## Bug 1: calc.go:4:11 ARITHMETIC_BASE",
|
|
"",
|
|
"The failing test printed, at the base:",
|
|
"",
|
|
" Add(1, 2) = -1, want 3",
|
|
"",
|
|
"4 of the 6 runs passed. Decoys, the files more than half the failing runs read that the bug isn't in:",
|
|
"",
|
|
"- `sum.go`, read by 2 of the 2 failing runs",
|
|
"",
|
|
"### bug-1-base-2, the failing run that spent the most tokens, 1200 tokens",
|
|
"",
|
|
"Read sum.go",
|
|
"Read calc.go",
|
|
"",
|
|
# Of the passing runs' 300, 500, 700 and 800, the lower of the two in the middle.
|
|
"### bug-1-head-1, the passing run with the median tokens, 500 tokens",
|
|
"",
|
|
"Read calc.go",
|
|
])
|
|
|
|
def test_a_bug_every_run_fixed_has_no_decoys_and_one_trace(self):
|
|
held = digest(SIDES, [], [Bug("2: a.go:1:1 X", "a.go", "boom", [Run("bug-2-base-1", True, 10, reading("a.go"))])], lambda path: True)
|
|
self.assertIn("The 1 run passed. No run failed, so the bug has no decoys.", held.splitlines())
|
|
self.assertNotIn("failing run that", held)
|
|
self.assertIn("### bug-2-base-1, the passing run with the median tokens, 10 tokens", held.splitlines())
|
|
|
|
def test_the_median_is_by_tokens_whatever_the_order_the_runs_ran_in(self):
|
|
runs = [Run("bug-2-base-1", True, 20, b""), Run("bug-2-head-1", True, 10, b""), Run("bug-2-base-2", True, 30, b"")]
|
|
self.assertEqual(digest(SIDES, [], [Bug("2: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()[-2:], ["### bug-2-base-1, the passing run with the median tokens, 20 tokens", ""])
|
|
|
|
def test_a_run_that_did_not_finish_is_the_failing_trace_and_is_left_out_of_the_median(self):
|
|
runs = [Run("bug-4-base-1", False, 900, b""), Run("bug-4-head-1", False, None, b""), Run("bug-4-base-2", True, None, b""), Run("bug-4-head-2", True, 60, b"")]
|
|
held = digest(SIDES, [], [Bug("4: a.go:1:1 X", "a.go", "boom", runs)], lambda path: True).splitlines()
|
|
self.assertEqual([line for line in held if line.startswith("###")], ["### bug-4-head-1, a failing run that didn't finish", "### bug-4-head-2, the passing run with the median tokens, 60 tokens"])
|
|
|
|
def test_a_bug_no_run_fixed_shows_the_failing_trace_alone_and_says_where_no_file_misled(self):
|
|
held = digest(SIDES, [], [Bug("3: a.go:1:1 X", "a.go", "boom\nat line 2", [Run("bug-3-base-1", False, 10, reading("a.go")), Run("bug-3-head-1", False, 10, reading("a.go"))])], lambda path: True)
|
|
self.assertEqual(held.splitlines()[12:16], ["0 of the 2 runs passed. No decoys: no file but the one the bug is in was read by more than half the 2 failing runs.", "", "### bug-3-base-1, the failing run that spent the most tokens, 10 tokens", ""])
|
|
self.assertIn(" at line 2", held.splitlines())
|
|
# Of two failing runs that spent the same, the first.
|
|
self.assertIn("### bug-3-base-1, the failing run that spent the most tokens, 10 tokens", held.splitlines())
|
|
self.assertNotIn("passing run with", held)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|