Files
claude-config/bin/task_bench/test_gather.py
T

145 lines
8.0 KiB
Python

"""`task-bench gather` in a small repository with claude faked: the claude it starts, what it leaves, and when it refuses."""
import contextlib
import io
import json
import pathlib
import shutil
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from task_bench.gather import gather, PROMPT
from test_env import isolated_env
from task_bench import main
class GatherTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.home = pathlib.Path(folder.name).resolve() / "home"
self.repo = self.home / "code/calc"
self.repo.mkdir(parents=True)
self.env = {**isolated_env(), "HOME": str(self.home)}
self.git("init", "--quiet", "--initial-branch", "main")
self.git("commit", "--quiet", "--allow-empty", "--message", "Base")
self.models = self.home / ".claude/loop-claude-models.json"
self.models.parent.mkdir()
self.models.write_text(json.dumps({"calc": "claude-opus-5-5"}))
self.bench = self.home / ".local/state/larv-review/code/calc/task-bench"
(self.bench / "runs").mkdir(parents=True)
(self.bench / "digest.md").write_text("# task-bench digest\n\n## Bug 1: calc.go:4:11 ARITHMETIC_BASE\n")
self.calls: list[tuple[list[str], str, str]] = []
self.reply: dict = {"type": "result", "subtype": "success", "is_error": False, "result": "512: **Rename `sum` to say it rounds.**\n"}
self.code = 0
self.enterContext(contextlib.chdir(self.repo))
def git(self, *args: str, cwd: pathlib.Path | None = None) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=cwd or self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
def claude(self, args: list[str], cwd: str, prompt: str) -> subprocess.CompletedProcess[str]:
self.calls.append((args, cwd, prompt))
return subprocess.CompletedProcess(args, self.code, json.dumps(self.reply) + "\n", "claude broke\n" if self.code else "")
def test_a_claude_on_the_projects_rung_reads_the_prompt_and_the_digest_and_then_the_benchs_folder_goes(self):
self.assertEqual(gather(self.env, self.home, self.claude), ["512: **Rename `sum` to say it rounds.**", "", f"Removed the bench's folder, {self.bench}."])
(args, cwd, prompt), = self.calls
self.assertEqual(args[:6], ["-p", "--model", "claude-opus-5-5", "--effort", "medium", "--strict-mcp-config"])
self.assertEqual(cwd, str(self.repo))
self.assertEqual(prompt, f"{PROMPT.read_text().strip()}\n\nThe bench's folder: {self.bench}\n\n# task-bench digest\n\n## Bug 1: calc.go:4:11 ARITHMETIC_BASE\n")
self.assertFalse(self.bench.exists())
def test_the_prompt_asks_for_plan_items_filed_with_todo_item(self):
self.assertIn("~/.claude/bin/todo-item add", PROMPT.read_text())
def test_a_worktrees_project_is_its_main_checkouts(self):
worktree = self.home / "code/calc-item-7"
self.git("worktree", "add", "--quiet", str(worktree), "-b", "item-7")
bench = self.home / ".local/state/larv-review/code/calc-item-7/task-bench"
bench.mkdir(parents=True)
(bench / "digest.md").write_text("worktree's\n")
with contextlib.chdir(worktree):
self.assertNotIsInstance(gather(self.env, self.home, self.claude), Failure)
self.assertEqual((self.calls[0][0][2], self.calls[0][1], self.calls[0][2].endswith("\n\nworktree's\n")), ("claude-opus-5-5", str(worktree), True))
self.assertEqual((bench.exists(), self.bench.exists()), (False, True))
def test_refuses_without_a_digest_and_starts_no_claude(self):
(self.bench / "digest.md").unlink()
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"{self.bench} holds no digest: run task-bench in this checkout first.", []))
def test_a_gather_before_any_review_makes_the_review_folders_private(self):
shutil.rmtree(self.home / ".local")
self.assertIsInstance(gather(self.env, self.home, self.claude), Failure)
self.assertEqual((self.home / ".local/state/larv-review").stat().st_mode & 0o777, 0o700)
def test_a_claude_that_fails_or_hits_a_limit_keeps_the_folder_for_another_try(self):
self.code = 1
found = gather(self.env, self.home, self.claude)
self.assertEqual(found.reason if isinstance(found, Failure) else found, f"claude broke\nThe digest stays in {self.bench} until the next bench: run `task-bench gather` again.")
self.reply = {"type": "result", "is_error": True, "api_error_status": 429, "result": "You've hit your session limit"}
found = gather(self.env, self.home, self.claude)
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "claude hit a usage limit: You've hit your session limit")
self.assertTrue((self.bench / "digest.md").exists())
def test_a_digest_that_cannot_be_read_is_refused_with_the_reason(self):
(self.bench / "digest.md").unlink()
(self.bench / "digest.md").mkdir()
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (f"could not read {self.bench / 'digest.md'}: Is a directory", []))
def test_says_so_where_claude_ends_without_an_answer(self):
self.reply["result"] = ""
self.assertEqual(gather(self.env, self.home, self.claude)[0], "The gathering claude didn't say anything.")
def test_refuses_where_the_project_has_no_model_on_its_ladder(self):
for saved, said in (({}, f"{self.models} names no model for calc: run loop-claude in the project once, which asks for it."), ({"calc": "claude-haiku-5-5"}, "calc runs on claude-haiku-5-5, which is not on its routing ladder.")):
self.models.write_text(json.dumps(saved))
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason if isinstance(found, Failure) else found, self.calls), (said, []))
self.models.write_text("{")
self.assertIsInstance(gather(self.env, self.home, self.claude), Failure)
def test_a_claude_that_fails_without_a_word_is_named(self):
self.code = 1
found = gather(self.env, self.home, lambda args, cwd, prompt: subprocess.CompletedProcess(args, 1, "", ""))
self.assertEqual(found.reason.splitlines()[0] if isinstance(found, Failure) else found, "the gathering claude exited 1")
def test_a_claude_that_ends_in_an_error_keeps_the_folder(self):
self.reply = {"type": "result", "subtype": "error_during_execution", "is_error": True, "result": "Prompt is too long"}
found = gather(self.env, self.home, self.claude)
self.assertEqual((found.reason.splitlines()[0] if isinstance(found, Failure) else found, self.bench.exists()), ("the gathering claude ended in an error: Prompt is too long", True))
def test_a_gather_with_arguments_is_refused(self):
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["gather", "--help"], self.env, claude=self.claude), 1)
self.assertEqual((errors.getvalue().split(";")[0], self.calls), ("gather takes no arguments", []))
def test_git_runs_in_the_environment_given(self):
env = {**self.env, "GIT_DIR": str(self.home / "none")}
self.assertIsInstance(gather(env, self.home, self.claude), Failure)
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(main.main(["gather"], env, claude=self.claude), 1)
self.assertEqual(self.calls, [])
def test_refuses_outside_a_checkout(self):
with contextlib.chdir(self.home):
self.assertIsInstance(gather(self.env, self.home, self.claude), Failure)
self.assertEqual(self.calls, [])
def test_the_command_line_gathers_without_a_token_and_prints_what_claude_answered(self):
with contextlib.redirect_stdout(io.StringIO()) as out:
self.assertEqual(main.main(["gather"], self.env, claude=self.claude), 0)
self.assertEqual(out.getvalue(), f"512: **Rename `sum` to say it rounds.**\n\nRemoved the bench's folder, {self.bench}.\n")
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["gather"], self.env, claude=self.claude), 1)
self.assertEqual(errors.getvalue(), f"{self.bench} holds no digest: run task-bench in this checkout first.\n")
if __name__ == "__main__":
unittest.main()