Files
claude-config/bin/do_epic/test_split.py
T
Lilleman 700ac3e2ce
CI / tests (pull_request) Successful in 3m45s
Load no MCP server in a headless claude call that is denied tools
2026-11-20 17:15:42 +01:00

243 lines
14 KiB
Python

import dataclasses
import json
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from do_epic.test_repo_pick import Claude, says
from do_epic.backend.protocol import EpicIssue
from failure import Failure
from do_epic.split import finished, is_split, Split, split, SPLIT_LABEL, SPLIT_SCHEMA, task_summary
from claude_code.reply import Limited
from do_epic.backend.fake import MemoryEpic
from model_routing.ladder import Rung
RULE = "A label equal to a repo's name names that repo."
RUNG = Rung("claude-sonnet-5-5", "medium")
TWO = {"parts": [{"repo": "api", "change": "Serve the count."}, {"repo": "web", "change": "Show the count."}], "comment": "Delat per repo: api och web."}
class Flaky(MemoryEpic):
"""An epic that fails on demand: while jammed is set it files no sub-task after the first under an issue, and fails_files and fails_comments fail every filing and comment."""
jammed = False
fails_files = False
fails_comments = False
def file(self, parent: str, summary: str, detail: str, labels: tuple[str, ...]) -> str | Failure:
return Failure("Jira answered 500") if self.fails_files or self.jammed and any(issue.parent == parent for issue in self.issues) else super().file(parent, summary, detail, labels)
def comment(self, key: str, text: str) -> None | Failure:
return Failure("Jira answered 500") if self.fails_comments else super().comment(key, text)
class Split_(unittest.TestCase):
def setUp(self) -> None:
self.store = Flaky("ABC-12")
key = self.store.file("ABC-12", "Count the runs.", "Why it counts.", ("api",))
assert isinstance(key, str)
self.key = key
def issue(self) -> EpicIssue:
return next(issue for issue in self.issues() if issue.key == self.key)
def issues(self) -> list[EpicIssue]:
read = self.store.read()
assert isinstance(read, list)
return read
def tasks(self) -> list[EpicIssue]:
return [issue for issue in self.issues() if issue.parent == self.key]
def run_split(self, claude: Claude, by_person: bool = False): # type: ignore[no-untyped-def]
return split(self.store, self.issue(), {}, by_person, RULE, "Swedish", ("/repos/api", "/repos/web"), RUNG, claude, "/rules", None)
def test_files_a_sub_task_per_repo_labelled_for_it_and_marks_the_parent(self) -> None:
done = self.run_split(says(TWO))
tasks = self.tasks()
self.assertEqual(done, Split(("api", "web"), tuple(task.key for task in tasks)))
self.assertEqual([(task.summary, task.detail, task.labels) for task in tasks], [("[api] Count the runs.", "Serve the count.", ("repo:api",)), ("[web] Count the runs.", "Show the count.", ("repo:web",))])
self.assertEqual(self.issue().labels, ("api", SPLIT_LABEL))
self.assertEqual(self.store.comments, {})
def test_files_each_sub_task_as_the_teams_template_and_label_say(self) -> None:
team = {"twu-4": "web", "core-4": "api"}
done = split(self.store, self.issue(), team, False, RULE, "Swedish", (), RUNG, says(TWO), "/rules", "{label}: {Summary}")
self.assertEqual((done, [(task.summary, task.labels) for task in self.tasks()]), (Split(("api", "web"), tuple(task.key for task in self.tasks())), [("core-4: Count the runs.", ("core-4",)), ("twu-4: Count the runs.", ("twu-4",))]))
def test_a_template_names_the_repo_where_the_table_does_not(self) -> None:
self.assertEqual(task_summary("api", "bygg stöd", {}, "{label}: {Summary}"), "api: Bygg stöd")
def test_a_template_names_the_teams_label_where_the_table_names_the_repo(self) -> None:
self.assertEqual(task_summary("api", "bygg stöd", {"core": "api"}, "{Summary} ({label})"), "Bygg stöd (core)")
def test_a_template_leaves_an_empty_summary_empty(self) -> None:
self.assertEqual(task_summary("api", "", {}, "{Summary}"), "")
def test_without_a_template_the_summary_is_the_repo_in_brackets_and_the_item_summary(self) -> None:
self.assertEqual(task_summary("api", "bygg stöd", {"core": "api"}, None), "[api] bygg stöd")
def test_an_item_a_person_filed_a_sub_task_under_is_left_alone_without_asking_claude(self) -> None:
self.store.issues.append(EpicIssue("ABC-90", "twu-4: Count them.", "", ("twu-4",), None, "To Do", False, self.key))
claude = says(TWO)
self.assertEqual((self.run_split(claude), claude.calls, len(self.tasks())), (Split((), ()), [], 1))
def test_a_sub_task_with_the_teams_label_counts_as_its_repos_when_a_failed_run_is_taken_up(self) -> None:
self.store.issues.append(EpicIssue("ABC-90", "Serve it.", "", ("backend",), None, "To Do", False, self.key, by_loop=True))
done = split(self.store, self.issue(), {"backend": "api"}, False, RULE, "Swedish", (), RUNG, says(TWO), "/rules", None)
self.assertEqual((done, [task.labels for task in self.tasks()]), (Split(("api", "web"), (self.tasks()[1].key,)), [("backend",), ("repo:web",)]))
def test_comments_on_a_persons_issue_for_the_team_once_its_sub_tasks_are_filed(self) -> None:
self.run_split(says(TWO), by_person=True)
self.assertEqual(self.store.comments, {self.key: ["Delat per repo: api och web."]})
self.assertEqual(len(self.tasks()), 2)
def test_a_persons_issue_without_a_comment_is_filed_nowhere(self) -> None:
done = self.run_split(says({"parts": TWO["parts"]}), by_person=True)
self.assertEqual(done, Failure("the repo split gave no comment for the team on a person's issue"))
self.assertEqual((self.tasks(), self.issue().labels), ([], ("api",)))
def test_a_first_filing_that_fails_says_the_run_filed_no_sub_task(self) -> None:
self.store.fails_files = True
self.assertEqual(self.run_split(says(TWO)), Failure(f"cannot file the sub-task for api under {self.key}: Jira answered 500; this run filed no sub-task; run do-epic again to file the rest"))
def test_a_comment_that_fails_leaves_the_item_unmarked_and_the_next_run_files_nothing_again(self) -> None:
self.store.fails_comments = True
failed = self.run_split(says(TWO), by_person=True)
first, second = (task.key for task in self.tasks())
self.assertEqual(failed, Failure(f"cannot comment on {self.key}: Jira answered 500; this run filed {first}, {second}; run do-epic again to post it"))
self.assertEqual(self.issue().labels, ("api",))
self.store.fails_comments = False
done = self.run_split(says(TWO), by_person=True)
self.assertEqual((done, len(self.tasks()), self.store.comments[self.key], self.issue().labels), (Split(("api", "web"), ()), 2, ["Delat per repo: api och web."], ("api", SPLIT_LABEL)))
def test_a_person_who_filed_a_sub_task_for_one_repo_still_gets_the_comment_and_the_rest_are_filed(self) -> None:
self.store.file(self.key, "Own task.", "", ("repo:api",))
done = self.run_split(says(TWO), by_person=True)
self.assertEqual((done.repos, [task.labels for task in self.tasks()], self.store.comments[self.key]), (("api", "web"), [("repo:api",), ("repo:web",)], ["Delat per repo: api och web."]))
def test_a_sub_task_already_filed_counts_towards_the_split_whatever_the_new_prediction_says(self) -> None:
self.store.file(self.key, "Own task.", "", ("repo:api",))
done = self.run_split(says({"parts": [{"repo": "web", "change": "Show the count."}]}))
self.assertEqual((done.repos, [task.labels for task in self.tasks()]), (("api", "web"), [("repo:api",), ("repo:web",)]))
def test_splits_nothing_where_the_item_changes_one_repo_or_none(self) -> None:
for parts in ([], [{"repo": "api", "change": "Serve the count."}], [{"repo": "api", "change": "a"}, {"repo": "api", "change": "b"}]):
with self.subTest(parts=parts):
self.assertEqual(self.run_split(says({"parts": parts}), by_person=True), Split((), ()))
self.assertEqual((self.tasks(), self.issue().labels, self.store.comments), ([], ("api",), {}))
def test_leaves_an_item_marked_split_alone_without_asking_claude(self) -> None:
self.store.add_label(self.key, SPLIT_LABEL)
claude = says(TWO)
self.assertEqual(self.run_split(claude), Split((), ()))
self.assertEqual((claude.calls, self.tasks()), ([], []))
def test_a_filing_that_keeps_failing_posts_no_comment(self) -> None:
self.store.fails_files = True
for _ in range(2):
self.assertIsInstance(self.run_split(says(TWO), by_person=True), Failure)
self.assertEqual(self.store.comments, {})
def test_a_failed_filing_names_what_was_filed_and_a_second_run_files_only_the_rest(self) -> None:
self.store.jammed = True
failed = self.run_split(says(TWO), by_person=True)
first = self.tasks()[0].key
self.assertEqual(failed, Failure(f"cannot file the sub-task for web under {self.key}: Jira answered 500; this run filed {first}; run do-epic again to file the rest"))
self.assertEqual(self.issue().labels, ("api",))
self.store.jammed = False
done = self.run_split(says(TWO), by_person=True)
self.assertEqual([task.labels for task in self.tasks()], [("repo:api",), ("repo:web",)])
self.assertEqual((done.tasks, self.issue().labels, self.store.comments[self.key]), (tuple(task.key for task in self.tasks()[1:]), ("api", SPLIT_LABEL), ["Delat per repo: api och web."]))
def test_asks_claude_for_json_on_the_given_rung_with_read_tools_on_the_repos(self) -> None:
claude = says(TWO)
self.run_split(claude)
args, cwd, prompt = claude.calls[0]
self.assertEqual(cwd, "/rules")
self.assertEqual(args[:13], ["-p", "--json-schema", json.dumps(SPLIT_SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", "Read,Grep,Glob", "--strict-mcp-config", "--add-dir", "/repos/api", "--add-dir"])
self.assertEqual(args[13], "/repos/web")
self.assertIn(RULE, prompt)
self.assertIn("## Item ABC-100: Count the runs.", prompt)
self.assertIn("write each part's change in Swedish.", prompt)
self.assertIn("in Swedish. Give no comment.\n", prompt)
self.assertIn("checked out at: /repos/api, /repos/web. Read them", prompt)
self.assertIn("\nLabels: api\n", prompt)
self.assertTrue(prompt.rstrip().endswith("Why it counts."))
def test_tells_claude_of_an_item_with_no_labels_no_description_and_no_checkouts(self) -> None:
key = self.store.file("ABC-12", "Bare item.", "", ())
assert isinstance(key, str)
bare = next(issue for issue in self.issues() if issue.key == key)
claude = says(TWO)
split(self.store, bare, {}, False, RULE, "Swedish", (), RUNG, claude, "/rules", None)
prompt = claude.calls[0][2]
self.assertIn("checked out at: none. Read them", prompt)
self.assertTrue(prompt.endswith("\nLabels: none\n\n(no description)\n"))
self.assertNotIn("--add-dir", claude.calls[0][0])
def test_lists_each_label_of_the_item_in_the_prompt(self) -> None:
key = self.store.file("ABC-12", "Two labels.", "Why.", ("api", "urgent"))
assert isinstance(key, str)
claude = says(TWO)
split(self.store, next(issue for issue in self.issues() if issue.key == key), {}, False, RULE, "Swedish", (), RUNG, claude, "/rules", None)
self.assertIn("\nLabels: api, urgent\n", claude.calls[0][2])
def test_asks_a_persons_issue_for_a_comment(self) -> None:
claude = says(TWO)
self.run_split(claude, by_person=True)
self.assertIn("The item is a person's issue: also give a comment, in Swedish,", claude.calls[0][2])
def test_fails_on_an_answer_without_parts_or_with_a_repo_name_that_is_a_path(self) -> None:
for output in ({}, "nope", {"parts": "api"}, {"parts": [{"repo": "../x", "change": "c"}]}, {"parts": [{"repo": "api"}]}, {"parts": [3]}, {"parts": [{"repo": "api", "change": " "}]}):
with self.subTest(output=output):
self.assertIsInstance(self.run_split(says(output)), Failure)
self.assertEqual(self.tasks(), [])
def test_an_answer_that_fails_is_quoted_up_to_two_hundred_characters(self) -> None:
output = {"parts": "x" * 300}
failed = self.run_split(says(output))
self.assertEqual(failed, Failure(f"the repo split answered {json.dumps(output)[:200]}; expected parts, each with a repo name of letters, digits, dots, hyphens and underscores and a change that holds text"))
def test_a_label_that_fails_after_the_sub_tasks_names_them_and_leaves_the_item_unmarked(self) -> None:
self.store.add_label = lambda key, label: Failure("Jira answered 500") if label == SPLIT_LABEL else None # type: ignore[method-assign]
failed = self.run_split(says(TWO))
first, second = (task.key for task in self.tasks())
self.assertEqual(failed, Failure(f"cannot mark {self.key} as split: Jira answered 500; this run filed {first}, {second}; run do-epic again to mark it"))
def test_passes_on_a_failed_or_limited_claude(self) -> None:
self.assertEqual(self.run_split(Claude("", 1)), Failure("the repo split exited 1"))
limited = self.run_split(Claude(json.dumps({"type": "result", "is_error": True, "api_error_status": 429, "result": "You've hit your limit"})))
self.assertIsInstance(limited, Limited)
class Finished(unittest.TestCase):
def issues(self, item: tuple[bool, tuple[str, ...]], tasks: tuple[bool, ...]) -> list[EpicIssue]:
parent = EpicIssue("ABC-1", "Count.", "", item[1], None, "Done" if item[0] else "To Do", item[0], "ABC-12")
return [parent, *(EpicIssue(f"ABC-{n + 2}", "Task.", "", (), None, "Done" if done else "To Do", done, "ABC-1") for n, done in enumerate(tasks))]
def test_a_split_item_is_done_once_every_sub_task_is_whatever_its_own_status(self) -> None:
self.assertEqual([finished(self.issues((own, (SPLIT_LABEL,)), tasks), "ABC-1") for own, tasks in ((False, (True, True)), (True, (True, False)), (False, (True, False)))], [True, False, False])
def test_an_item_that_is_not_split_is_done_by_its_own_status_and_an_unknown_one_is_not_done(self) -> None:
self.assertEqual([finished(self.issues((own, ()), ()), "ABC-1") for own in (True, False)], [True, False])
self.assertEqual(finished(self.issues((True, ()), ()), "ABC-9"), False)
def test_an_item_with_a_persons_sub_tasks_is_split_without_the_mark_and_done_only_by_its_own_status(self) -> None:
self.assertEqual([finished(self.issues((done, ()), (True, True)), "ABC-1") for done in (False, True)], [False, True])
self.assertTrue(is_split(*self.issues((False, ()), (False,))[:1], self.issues((False, ()), (False,))))
def test_an_item_whose_only_sub_tasks_do_epic_filed_is_not_split_until_marked(self) -> None:
parent, task = self.issues((False, ()), (True,))
ours = [parent, dataclasses.replace(task, by_loop=True)]
self.assertEqual((is_split(parent, ours), finished(ours, "ABC-1")), (False, False))
self.assertTrue(is_split(dataclasses.replace(parent, labels=(SPLIT_LABEL,)), ours))
def test_an_item_marked_split_whose_sub_tasks_are_not_filed_yet_is_done_by_its_own_status(self) -> None:
self.assertEqual(finished(self.issues((False, (SPLIT_LABEL,)), ()), "ABC-1"), False)
if __name__ == "__main__":
unittest.main()