Files
claude-config/bin/task_bench/test_gather.py
T

139 lines
7.7 KiB
Python

"""`task-bench gather` in a small repository with claude faked: the claude it starts, what it leaves, and when it refuses."""
import contextlib
import io
import json
import pathlib
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from task_bench.gather import gather, PROMPT
from test_env import isolated_env
from task_bench import main
class GatherTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.home = pathlib.Path(folder.name).resolve() / "home"
self.repo = self.home / "code/calc"
self.repo.mkdir(parents=True)
self.env = {**isolated_env(), "HOME": str(self.home)}
self.git("init", "--quiet", "--initial-branch", "main")
self.git("commit", "--quiet", "--allow-empty", "--message", "Base")
self.models = self.home / ".claude/loop-claude-models.json"
self.models.parent.mkdir()
self.models.write_text(json.dumps({"calc": "claude-opus-5-5"}))
self.bench = self.home / ".local/state/larv-review/code/calc/task-bench"
(self.bench / "runs").mkdir(parents=True)
(self.bench / "digest.md").write_text("# task-bench digest\n\n## Bug 1: calc.go:4:11 ARITHMETIC_BASE\n")
self.calls: list[tuple[list[str], str, str]] = []
self.reply: dict = {"type": "result", "subtype": "success", "is_error": False, "result": "512: **Rename `sum` to say it rounds.**\n"}
self.code = 0
self.enterContext(contextlib.chdir(self.repo))
def git(self, *args: str, cwd: pathlib.Path | None = None) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=cwd or self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
def claude(self, args: list[str], cwd: str, prompt: str) -> subprocess.CompletedProcess[str]:
self.calls.append((args, cwd, prompt))
return subprocess.CompletedProcess(args, self.code, json.dumps(self.reply) + "\n", "claude broke\n" if self.code else "")
def test_a_claude_on_the_projects_rung_reads_the_prompt_and_the_digest_and_then_the_benchs_folder_goes(self):
self.assertEqual(gather(self.env, self.home, self.claude), ["512: **Rename `sum` to say it rounds.**", "", f"Removed the bench's folder, {self.bench}."])
(args, cwd, prompt), = self.calls
self.assertEqual(args[:6], ["-p", "--model", "claude-opus-5-5", "--effort", "medium", "--strict-mcp-config"])
self.assertEqual(cwd, str(self.repo))
self.assertEqual(prompt, f"{PROMPT.read_text().strip()}\n\nThe bench's folder: {self.bench}\n\n# task-bench digest\n\n## Bug 1: calc.go:4:11 ARITHMETIC_BASE\n")
self.assertFalse(self.bench.exists())
def test_the_prompt_asks_for_plan_items_filed_with_todo_item(self):
self.assertIn("~/.claude/bin/todo-item add", PROMPT.read_text())
def test_a_worktrees_project_is_its_main_checkouts(self):
worktree = self.home / "code/calc-item-7"
self.git("worktree", "add", "--quiet", str(worktree), "-b", "item-7")
bench = self.home / ".local/state/larv-review/code/calc-item-7/task-bench"
bench.mkdir(parents=True)
(bench / "digest.md").write_text("worktree's\n")
with contextlib.chdir(worktree):
self.assertNotIsInstance(gather(self.env, self.home, self.claude), Failure)
self.assertEqual((self.calls[0][0][2], self.calls[0][1], self.calls[0][2].endswith("\n\nworktree's\n")), ("claude-opus-5-5", str(worktree), True))
self.assertEqual((bench.exists(), self.bench.exists()), (False, True))
def test_refuses_without_a_digest_and_starts_no_claude(self):
(self.bench / "digest.md").unlink()
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"{self.bench} holds no digest: run task-bench in this checkout first.", []))
def test_a_claude_that_fails_or_hits_a_limit_keeps_the_folder_for_another_try(self):
self.code = 1
found = gather(self.env, self.home, self.claude)
self.assertEqual(found.reason if isinstance(found, Failure) else found, f"claude broke\nThe digest stays in {self.bench} until the next bench: run `task-bench gather` again.")
self.reply = {"type": "result", "is_error": True, "api_error_status": 429, "result": "You've hit your session limit"}
found = gather(self.env, self.home, self.claude)
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "claude hit a usage limit: You've hit your session limit")
self.assertTrue((self.bench / "digest.md").exists())
def test_a_digest_that_cannot_be_read_is_refused_with_the_reason(self):
(self.bench / "digest.md").unlink()
(self.bench / "digest.md").mkdir()
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"could not read {self.bench / 'digest.md'}: Is a directory", []))
def test_says_so_where_claude_ends_without_an_answer(self):
self.reply["result"] = ""
self.assertEqual(gather(self.env, self.home, self.claude)[0], "The gathering claude didn't say anything.")
def test_refuses_where_the_project_has_no_model_on_its_ladder(self):
for saved, said in (({}, f"{self.models} names no model for calc: run loop-claude in the project once, which asks for it."), ({"calc": "claude-haiku-5-5"}, "calc runs on claude-haiku-5-5, which is not on its routing ladder.")):
self.models.write_text(json.dumps(saved))
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (said, []))
self.models.write_text("{")
self.assertIsInstance(gather(self.env, self.home, self.claude), Failure)
def test_a_claude_that_fails_without_a_word_is_named(self):
self.code = 1
found = gather(self.env, self.home, lambda args, cwd, prompt: subprocess.CompletedProcess(args, 1, "", ""))
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "the gathering claude exited 1")
def test_a_claude_that_ends_in_an_error_keeps_the_folder(self):
self.reply = {"type": "result", "subtype": "error_during_execution", "is_error": True, "result": "Prompt is too long"}
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason.splitlines()[0] if isinstance(found, Failure) else found, self.bench.exists()), ("the gathering claude ended in an error: Prompt is too long", True))
def test_a_gather_with_arguments_is_refused(self):
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["gather", "--help"], self.env, claude=self.claude), 1)
self.assertEqual((errors.getvalue().split(";")[0], self.calls), ("gather takes no arguments", []))
def test_git_runs_in_the_environment_given(self):
env = {**self.env, "GIT_DIR": str(self.home / "none")}
self.assertIsInstance(gather(env, self.home, self.claude), Failure)
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(main.main(["gather"], env, claude=self.claude), 1)
self.assertEqual(self.calls, [])
def test_refuses_outside_a_checkout(self):
with contextlib.chdir(self.home):
self.assertIsInstance(gather(self.env, self.home, self.claude), Failure)
self.assertEqual(self.calls, [])
def test_the_command_line_gathers_without_a_token_and_prints_what_claude_answered(self):
with contextlib.redirect_stdout(io.StringIO()) as out:
self.assertEqual(main.main(["gather"], self.env, claude=self.claude), 0)
self.assertEqual(out.getvalue(), f"512: **Rename `sum` to say it rounds.**\n\nRemoved the bench's folder, {self.bench}.\n")
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["gather"], self.env, claude=self.claude), 1)
self.assertEqual(errors.getvalue(), f"{self.bench} holds no digest: run task-bench in this checkout first.\n")
if __name__ == "__main__":
unittest.main()