Answer stability round 1: 0 🔴, 4 🟡 (0 misread); refuse an empty blocker or parent, scope the fake's blocker check to the epic, check the link's direction in the e2e, assert the next-step command only
CI / tests (pull_request) Failing after 4m38s

This commit is contained in:
Lilleman
2026-11-19 16:46:27 +01:00
parent 90352d74c4
commit b832c1e669
8 changed files with 64 additions and 7 deletions
+1 -1
View File
@@ -26,7 +26,7 @@ class MemoryEpic:
return refused
if parent != self.epic_key and parent not in {issue.key for issue in self.issues if issue.parent == self.epic_key}:
return no_parent(self.epic_key, parent)
if blocked_by is not None and blocked_by not in {issue.key for issue in self.issues}:
if blocked_by is not None and blocked_by not in {issue.key for issue in self.read()}:
return no_blocker(self.epic_key, blocked_by)
key = f"{self.epic_key.rpartition('-')[0]}-{len(self.issues) + 100}"
self.issues.append(EpicIssue(key, summary, detail.strip(), tuple(sorted(labels)), None, "To Do", False, parent, blockers=(blocked_by,) if blocked_by else (), by_loop=True))
+2 -2
View File
@@ -61,7 +61,7 @@ class JiraEpic:
authorization = basic_auth(self.env_file)
if isinstance(authorization, Failure):
return authorization
if refused := unfit(summary, labels) or self.not_a_parent(parent, authorization) or (blocked_by and self.not_a_blocker(blocked_by, authorization)):
if refused := unfit(summary, labels) or self.not_a_parent(parent, authorization) or (self.not_a_blocker(blocked_by, authorization) if blocked_by is not None else None):
return refused
fields: dict[str, typing.Any] = {"project": {"key": project_of(parent)}, "parent": {"key": parent}, "summary": summary, "labels": list(labels)}
# Marks the issue as the loop's, as plan_store marks the items it files.
@@ -75,7 +75,7 @@ class JiraEpic:
return Failure(f"cannot convert the detail to a Jira description: {description.reason}")
fields["description"] = description
body: dict[str, typing.Any] = {"fields": fields | {"issuetype": kind}, "properties": properties}
if blocked_by:
if blocked_by is not None:
# Linked as it is created, so no issue is left filed and unblocked. The inward issue blocks the new one.
body["update"] = {"issuelinks": [{"add": {"type": {"name": BLOCKS}, "inwardIssue": {"key": blocked_by}}}]}
created = self.http("POST", f"{self.epic.site}/rest/api/3/issue", authorization, body)
+9
View File
@@ -8,6 +8,7 @@ from do_epic.backend.contract import EpicStoreContract
from failure import Failure
from plan_store import jira_epic
from do_epic.backend.fake import MemoryEpic
from do_epic.backend.protocol import EpicIssue, no_blocker
class MemoryEpicContract(EpicStoreContract, unittest.TestCase):
@@ -44,6 +45,14 @@ class Failing(unittest.TestCase):
self.assertEqual((store.read(), store.file("ABC-12", "Count the runs.", "", ()), store.add_label("ABC-100", "api"), store.comment("ABC-100", "Hello."), store.rank("ABC-100", "ABC-101")), (down,) * 5)
class Blocking(unittest.TestCase):
def test_refuses_a_blocker_that_is_an_issue_of_another_epic(self) -> None:
store = MemoryEpic("ABC-12")
store.issues.append(EpicIssue("ABC-50", "Elsewhere", "", (), None, "To Do", False, "ABC-99"))
self.assertEqual(store.file("ABC-12", "Count more.", "", (), "ABC-50"), no_blocker("ABC-12", "ABC-50"))
self.assertEqual(len(store.issues), 1)
class Ranking(unittest.TestCase):
def test_refuses_to_rank_beside_an_issue_that_isnt_a_child_of_the_epic(self) -> None:
store = MemoryEpic("ABC-12")
+11
View File
@@ -80,6 +80,7 @@ def run(sandbox: Sandbox, epic_key: str, launcher: pathlib.Path, env: typing.Map
return report
items_filed(report, store, epic_key)
finding_filed(report, machine, store, epic_key)
follow_up_filed(report, machine, store, epic_key)
decision_recorded(report, machine, record.section, epic_key)
closed(report, machine, store, epic_key)
return report
@@ -111,6 +112,16 @@ def finding_filed(report: Report, machine: DoEpic, store: EpicStore, epic_key: s
report.check(len(found) == 1 and found[0].labels == (CLAUDE_LABEL,) and found[0].detail == "Why the finding matters." and found[0].by_loop, f"the finding is an issue of the epic with its label, its detail and the loop's mark (found {found})")
def follow_up_filed(report: Report, machine: DoEpic, store: EpicStore, epic_key: str) -> None:
"""Files a follow-up finding that waits for the first finding, and checks that Jira reads the link in that direction."""
first = next((issue.key for issue in read(report, store, "before the follow-up") if issue.summary == "Sandbox finding"), "")
done = machine.do_epic(epic_key, findings.FILE, "Sandbox follow-up finding", findings.LABEL, CLAUDE_LABEL, findings.BLOCKED_BY, first, stdin="Waits for the first finding.\n")
report.check(done.returncode == 0 and f"It is blocked by {first}." in done.stdout, f"`do-epic {findings.FILE} {findings.BLOCKED_BY}` files the follow-up and says it is blocked (exit {done.returncode}: {done.stdout.strip() or done.stderr.strip()})")
issues = read(report, store, "after the follow-up")
waiting = [issue for issue in issues if issue.summary == "Sandbox follow-up finding"]
report.check(len(waiting) == 1 and waiting[0].blockers == (first,) and not any(issue.blockers for issue in issues if issue.key == first), f"the follow-up is blocked by the first finding, not the other way round (found {[(issue.key, issue.blockers) for issue in issues if issue.key == first or issue.summary == 'Sandbox follow-up finding']})")
def decision_recorded(report: Report, machine: DoEpic, section: typing.Callable[[str], str | Failure], epic_key: str) -> None:
done = machine.do_epic(epic_key, findings.RECORD_DECISION, "Sandbox decision", stdin="The text of the decision.\n")
report.check(done.returncode == 0 and "Recorded the decision" in done.stdout, f"`do-epic {findings.RECORD_DECISION}` records the decision and says so (exit {done.returncode}: {done.stdout.strip() or done.stderr.strip()})")
+36
View File
@@ -108,6 +108,42 @@ class FindingFiledTest(unittest.TestCase):
self.assertIn("files the finding and says so", report.broke[0])
class FollowUpFiledTest(unittest.TestCase):
def store(self) -> MemoryEpic:
store = MemoryEpic(EPIC)
store.file(EPIC, "Sandbox finding", "", (CLAUDE_LABEL,))
return store
def following(self, store: MemoryEpic, flipped: bool = False) -> typing.Callable[[], subprocess.CompletedProcess[str]]:
def answer() -> subprocess.CompletedProcess[str]:
key = store.file(EPIC, "Sandbox follow-up finding", "", (CLAUDE_LABEL,), None if flipped else "SBX-100")
if flipped:
store.change("SBX-100", blockers=(str(key),))
return done(0, "Filed SBX-101 in the epic SBX-1. It is blocked by SBX-100.")
return answer
def test_holds_where_the_follow_up_is_blocked_by_the_first_finding(self) -> None:
store = self.store()
machine = Scripted([self.following(store)])
report = scenario.Report()
scenario.follow_up_filed(report, machine, store, EPIC)
self.assertEqual((report.broke, len(report.held)), ([], 2))
self.assertEqual(machine.calls, [((EPIC, findings.FILE, "Sandbox follow-up finding", findings.LABEL, CLAUDE_LABEL, findings.BLOCKED_BY, "SBX-100"), "Waits for the first finding.\n")])
def test_breaks_where_the_link_points_the_wrong_way(self) -> None:
store = self.store()
report = scenario.Report()
scenario.follow_up_filed(report, Scripted([self.following(store, flipped=True)]), store, EPIC)
self.assertEqual(len(report.broke), 1)
self.assertIn("not the other way round", report.broke[0])
def test_breaks_where_do_epic_fails_and_files_nothing(self) -> None:
report = scenario.Report()
scenario.follow_up_filed(report, Scripted([lambda: done(1, err="no")]), self.store(), EPIC)
self.assertEqual(len(report.broke), 2)
self.assertIn("exit 1: no", report.broke[0])
class DecisionRecordedTest(unittest.TestCase):
SECTIONS: typing.ClassVar[dict[str, str | Failure]] = {"Decisions": "### Sandbox decision\nThe text of the decision.", "Goals": "1. Prove do-epic's commands work."}
+1 -1
View File
@@ -85,7 +85,7 @@ def parsed(words: list[str]) -> Write | None:
return None
taken[option].append(rest[at + 1])
rest = rest[:at] + rest[at + 2:]
if len(rest) != 1 or len(taken[BLOCKED_BY]) > 1 or len(taken[PARENT]) > 1 or any(word.startswith("--") for word in (*rest, *sum(taken.values(), []))):
if len(rest) != 1 or len(taken[BLOCKED_BY]) > 1 or len(taken[PARENT]) > 1 or any(word.startswith("--") for word in (*rest, *sum(taken.values(), []))) or any(not word.strip() for option in (BLOCKED_BY, PARENT) for word in taken[option]):
return None
return Write(words[0], rest[0], tuple(taken[LABEL]), *(taken[option][0] if taken[option] else None for option in (BLOCKED_BY, PARENT)))
+1 -1
View File
@@ -132,7 +132,7 @@ class FilingAFinding(FindingsTest):
self.assertIn("config.toml", err)
def test_prints_the_help_for_a_filing_spelt_wrong(self) -> None:
for argv in (["ABC-12", "--file"], ["ABC-12", "--file", "a", "b"], ["ABC-12", "--file", "a", "--label"], ["ABC-12", "--file", "a", "--labels", "x"], ["--file", "ABC-12", "a"], ["ABC-12", "--file", "--label", "x"], ["ABC-12", "--file", "--lable"], ["ABC-12", "--file", "a", "--blocked-by"], ["ABC-12", "--file", "a", "--blocked-by", "ABC-100", "--blocked-by", "ABC-101"], ["ABC-12", "--file", "a", "--parent", "ABC-100", "--parent", "ABC-101"], ["ABC-12", "--file", "a", "--parent", "--label", "x"], ["ABC-12", "--record-decision", "a", "--blocked-by", "ABC-100"], ["ABC-12", "--file", "a", "--label", "--x"], ["--chunk", "ABC-20", "ABC-12", "--file", "a"]):
for argv in (["ABC-12", "--file", "a", "--blocked-by", ""], ["ABC-12", "--file", "a", "--parent", " "], ["ABC-12", "--file"], ["ABC-12", "--file", "a", "b"], ["ABC-12", "--file", "a", "--label"], ["ABC-12", "--file", "a", "--labels", "x"], ["--file", "ABC-12", "a"], ["ABC-12", "--file", "--label", "x"], ["ABC-12", "--file", "--lable"], ["ABC-12", "--file", "a", "--blocked-by"], ["ABC-12", "--file", "a", "--blocked-by", "ABC-100", "--blocked-by", "ABC-101"], ["ABC-12", "--file", "a", "--parent", "ABC-100", "--parent", "ABC-101"], ["ABC-12", "--file", "a", "--parent", "--label", "x"], ["ABC-12", "--record-decision", "a", "--blocked-by", "ABC-100"], ["ABC-12", "--file", "a", "--label", "--x"], ["--chunk", "ABC-20", "ABC-12", "--file", "a"]):
with self.subTest(argv=argv):
code, out, err, _ = self.run_with(argv)
self.assertEqual((code, out, err, len(self.store.issues)), (1, "", main.HELP + "\n", 2))
+3 -2
View File
@@ -346,11 +346,12 @@ class TheFirstPrompt(unittest.TestCase):
f"File a finding about one of the epic's goals in the epic, with `/b/do-epic ABC-12 --file \"<summary>\" --label <label>` and its detail on stdin. The label is {labels}; leave it out where you cannot tell, and the next `do-epic` the human runs picks it. "
"Where filing fails and says the issue may exist, ask the human to look before you file it again. File a finding about a repo's goals by that repo's rules. Where unsure which, ask the human. "
"Record a decision that governs the epic, such as an answer the human typed, with `/b/do-epic ABC-12 --record-decision \"<title>\"` and its text on stdin. "
"Where the repo's rules ask for a change to ship as more than one pull request, finish only this issue's pull request, and file only the next one as an issue that waits for it, with `/b/do-epic ABC-12 --file \"<summary>\" --blocked-by ABC-20` and its detail on stdin. "
"Write the issue and the decision in Swedish, the epic's language."
)
task = chunk_prompt.task(issue("ABC-20"), "ABC-12", "/code/api", pathlib.Path("/b/do-epic"), "Swedish", CONFIG)
self.assertTrue(task.startswith(f"Work the Jira issue ABC-20 of the epic ABC-12 as one chunk, in the checkout /code/api, on the branch ABC-20. {told} "))
self.assertTrue(task.startswith(f"Work the Jira issue ABC-20 of the epic ABC-12 as one chunk, in the checkout /code/api, on the branch ABC-20. {told.removesuffix('Write the issue and the decision in Swedish, the epic\'s language.')}"))
self.assertIn("`/b/do-epic ABC-12 --file \"<summary>\" --blocked-by ABC-20`", task)
self.assertIn(" Write the issue and the decision in Swedish, the epic's language. ", task)
self.assertTrue(task.endswith(" Its summary: Do ABC-20"))
def test_a_sub_task_is_told_to_file_the_next_pull_request_under_its_parent_in_its_repo(self) -> None: