Answer stability round 1: 2 🔴, 5 🟡 (0 misread); fixed the import order and the unused return, tested the scenario steps offline, passed env and timeout in, reused the paged search
This commit is contained in:
@@ -30,7 +30,7 @@ def main(argv: list[str], env: typing.Mapping[str, str], launcher: pathlib.Path)
|
||||
print(f"do-epic-e2e: {key.reason}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"Sandbox epic {key} in {found.jira_url}")
|
||||
report = scenario.run(found, key, launcher)
|
||||
report = scenario.run(found, key, launcher, env)
|
||||
except sandbox.Leftover as failure:
|
||||
left = failure
|
||||
# The checks are printed even where the epic could not be deleted.
|
||||
|
||||
@@ -6,6 +6,7 @@ from atlassian.client import basic_auth, Http, jira_json, ToAdf
|
||||
from failure import Failure
|
||||
from adf_codec.convert import markdown_to_adf
|
||||
from do_epic.e2e.settings import Sandbox
|
||||
from plan_store.jira_epic import searched
|
||||
|
||||
EPIC_TYPE = "Epic"
|
||||
# How a team-managed project's workflow moves an issue to done, as do-epic's `jira-finish-transition` and `jira-finish-status` name it.
|
||||
@@ -61,11 +62,11 @@ def epic_made(sandbox: Sandbox, summary: str, http: Http = jira_json, to_adf: To
|
||||
|
||||
def deleted_with_children(sandbox: Sandbox, key: str, authorization: str, http: Http) -> None | Failure:
|
||||
"""Deletes the epic's child issues, with their sub-tasks, and then the epic: Jira deletes an epic's sub-tasks along with it, but not its child issues."""
|
||||
children = http("POST", api(sandbox, "search/jql"), authorization, {"jql": f"parent = {key}", "fields": ["summary"], "maxResults": 100})
|
||||
found = children.get("issues") if isinstance(children, dict) else None
|
||||
# The first failure is the one to report; the epic is deleted all the same.
|
||||
problem = children if isinstance(children, Failure) else None if isinstance(found, list) else Failure(f"Jira answered the search for {key}'s issues with {children!r}, which holds no issues")
|
||||
for each in [*(item.get("key") for item in found if isinstance(item, dict)), key] if isinstance(found, list) else [key]:
|
||||
found = searched(sandbox.jira_url, f"the issues of {key}", f"parent = {key}", {"fields": ["summary"]}, authorization, http)
|
||||
# The epic is deleted all the same; the first failure is the one to report.
|
||||
problem = found if isinstance(found, Failure) else None
|
||||
children = [] if isinstance(found, Failure) else [item["key"] for item in found if isinstance(item, dict) and isinstance(item.get("key"), str)]
|
||||
for each in [*children, key]:
|
||||
deleted = http("DELETE", api(sandbox, f"issue/{each}?deleteSubtasks=true"), authorization, None)
|
||||
problem = problem or (deleted if isinstance(deleted, Failure) else None)
|
||||
return problem
|
||||
|
||||
+28
-20
@@ -1,15 +1,15 @@
|
||||
"""What the e2e command does to a sandbox epic: runs `do-epic --file`, `--record-decision` and `--close-epic`, each as a process of its own against the real Jira site, and checks what the site holds after each."""
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import typing
|
||||
|
||||
from do_epic.repo_pick import CLAUDE_LABEL, HUMAN_LABEL
|
||||
from project_config.settings import Epic, Finish
|
||||
from do_epic import epic_record, findings, finish
|
||||
from record_store.jira_description import EpicDescription
|
||||
from do_epic.backend.protocol import EpicIssue
|
||||
from do_epic.backend.protocol import EpicIssue, EpicStore
|
||||
from failure import Failure
|
||||
from do_epic.e2e.sandbox import FINISH_STATUS, FINISH_TRANSITION
|
||||
from do_epic.backend.jira import JiraEpic
|
||||
@@ -27,26 +27,34 @@ class Report:
|
||||
held: list[str] = dataclasses.field(default_factory=list)
|
||||
broke: list[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
def check(self, holds: bool, what: str) -> bool:
|
||||
def check(self, holds: bool, what: str) -> None:
|
||||
(self.held if holds else self.broke).append(what)
|
||||
return holds
|
||||
|
||||
|
||||
class DoEpic(typing.Protocol):
|
||||
"""What the steps run do-epic through: a Machine, or a stand-in in a test."""
|
||||
|
||||
def do_epic(self, *args: str, stdin: str = "") -> subprocess.CompletedProcess[str]:
|
||||
"""Runs `do-epic args`, feeding it stdin."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Machine:
|
||||
"""A home folder of its own, holding the do-epic settings that point at the sandbox, so that do-epic writes its state there. The sandbox's env-file stays where it is."""
|
||||
"""A home folder of its own, holding the do-epic settings that point at the sandbox, so that do-epic writes its state there. The sandbox's env-file stays where it is. do-epic runs in env with HOME set to the home folder, and is stopped after timeout seconds."""
|
||||
home: pathlib.Path
|
||||
launcher: pathlib.Path
|
||||
env: typing.Mapping[str, str]
|
||||
timeout: int = TIMEOUT
|
||||
|
||||
def do_epic(self, *args: str, stdin: str = "") -> subprocess.CompletedProcess[str]:
|
||||
env = {**os.environ, "HOME": str(self.home)}
|
||||
env = {**self.env, "HOME": str(self.home)}
|
||||
try:
|
||||
return subprocess.run([str(self.launcher), *args], input=stdin, env=env, capture_output=True, text=True, timeout=TIMEOUT, check=False)
|
||||
return subprocess.run([str(self.launcher), *args], input=stdin, env=env, capture_output=True, text=True, timeout=self.timeout, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
return subprocess.CompletedProcess(args, 124, "", f"do-epic {' '.join(args)} ran over {TIMEOUT} seconds and was stopped")
|
||||
return subprocess.CompletedProcess(args, 124, "", f"do-epic {' '.join(args)} ran over {self.timeout} seconds and was stopped")
|
||||
|
||||
|
||||
def machine_made(home: pathlib.Path, sandbox: Sandbox, launcher: pathlib.Path) -> Machine | Failure:
|
||||
def machine_made(home: pathlib.Path, sandbox: Sandbox, launcher: pathlib.Path, env: typing.Mapping[str, str]) -> Machine | Failure:
|
||||
folder = home / ".claude-projects" / "do-epic"
|
||||
folder.mkdir(parents=True)
|
||||
ladder = read_ladder(SHIPPED)
|
||||
@@ -56,28 +64,28 @@ def machine_made(home: pathlib.Path, sandbox: Sandbox, launcher: pathlib.Path) -
|
||||
f'jira-url = "{sandbox.jira_url}"\nenv-file = "{sandbox.env_file}"\nrepoless-model = "{ladder.rungs[0].model}"\n'
|
||||
f'jira-finish-transition = "{FINISH_TRANSITION}"\njira-finish-status = "{FINISH_STATUS}"\n', encoding="utf-8")
|
||||
(folder / "repo-rule.md").write_text(RULE, encoding="utf-8")
|
||||
return Machine(home, launcher)
|
||||
return Machine(home, launcher, env)
|
||||
|
||||
|
||||
def run(sandbox: Sandbox, epic_key: str, launcher: pathlib.Path) -> Report:
|
||||
def run(sandbox: Sandbox, epic_key: str, launcher: pathlib.Path, env: typing.Mapping[str, str]) -> Report:
|
||||
"""Drives do-epic against the epic epic_key in the sandbox, an epic with no issue yet."""
|
||||
report = Report()
|
||||
epic = Epic(sandbox.jira_url, epic_key)
|
||||
store = JiraEpic(epic, sandbox.env_file, finish=Finish(FINISH_TRANSITION, FINISH_STATUS))
|
||||
record = epic_record.EpicRecord(EpicDescription(epic, sandbox.env_file))
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
machine = machine_made(pathlib.Path(home).resolve(), sandbox, launcher)
|
||||
machine = machine_made(pathlib.Path(home).resolve(), sandbox, launcher, env)
|
||||
if isinstance(machine, Failure):
|
||||
report.check(False, f"set up do-epic's settings: {machine.reason}")
|
||||
return report
|
||||
items_filed(report, store, epic_key)
|
||||
finding_filed(report, machine, store, epic_key)
|
||||
decision_recorded(report, machine, record, epic_key)
|
||||
decision_recorded(report, machine, record.section, epic_key)
|
||||
closed(report, machine, store, epic_key)
|
||||
return report
|
||||
|
||||
|
||||
def read(report: Report, store: JiraEpic, what: str) -> list[EpicIssue]:
|
||||
def read(report: Report, store: EpicStore, what: str) -> list[EpicIssue]:
|
||||
issues = store.read()
|
||||
if isinstance(issues, Failure):
|
||||
report.check(False, f"read the epic's issues {what}: {issues.reason}")
|
||||
@@ -85,7 +93,7 @@ def read(report: Report, store: JiraEpic, what: str) -> list[EpicIssue]:
|
||||
return issues
|
||||
|
||||
|
||||
def items_filed(report: Report, store: JiraEpic, epic_key: str) -> None:
|
||||
def items_filed(report: Report, store: EpicStore, epic_key: str) -> None:
|
||||
"""Files an item, its sub-task and a second item in the epic, and checks that a read answers them in the tracker's order."""
|
||||
parent = store.file(epic_key, "Sandbox item with a sub-task", "The first item.", (HUMAN_LABEL,))
|
||||
second = store.file(epic_key, "Sandbox second item", "", ())
|
||||
@@ -96,23 +104,23 @@ def items_filed(report: Report, store: JiraEpic, epic_key: str) -> None:
|
||||
report.check([issue.summary for issue in issues] == ["Sandbox item with a sub-task", "Sandbox sub-task", "Sandbox second item"], f"read the filed issues back in order, the sub-task after its parent (found {[issue.summary for issue in issues]})")
|
||||
|
||||
|
||||
def finding_filed(report: Report, machine: Machine, store: JiraEpic, epic_key: str) -> None:
|
||||
def finding_filed(report: Report, machine: DoEpic, store: EpicStore, epic_key: str) -> None:
|
||||
done = machine.do_epic(epic_key, findings.FILE, "Sandbox finding", findings.LABEL, CLAUDE_LABEL, stdin="Why the finding matters.\n")
|
||||
report.check(done.returncode == 0 and " Filed " in done.stdout, f"`do-epic {findings.FILE}` files the finding and says so (exit {done.returncode}: {done.stdout.strip() or done.stderr.strip()})")
|
||||
found = [issue for issue in read(report, store, "after the finding") if issue.summary == "Sandbox finding"]
|
||||
report.check(len(found) == 1 and found[0].labels == (CLAUDE_LABEL,) and found[0].detail == "Why the finding matters." and found[0].by_loop, f"the finding is an issue of the epic with its label, its detail and the loop's mark (found {found})")
|
||||
|
||||
|
||||
def decision_recorded(report: Report, machine: Machine, record: epic_record.EpicRecord, epic_key: str) -> None:
|
||||
def decision_recorded(report: Report, machine: DoEpic, section: typing.Callable[[str], str | Failure], epic_key: str) -> None:
|
||||
done = machine.do_epic(epic_key, findings.RECORD_DECISION, "Sandbox decision", stdin="The text of the decision.\n")
|
||||
report.check(done.returncode == 0 and "Recorded the decision" in done.stdout, f"`do-epic {findings.RECORD_DECISION}` records the decision and says so (exit {done.returncode}: {done.stdout.strip() or done.stderr.strip()})")
|
||||
decisions = record.section("Decisions")
|
||||
decisions = section("Decisions")
|
||||
report.check(isinstance(decisions, str) and "Sandbox decision" in decisions and "The text of the decision." in decisions, f"the epic's description holds the decision under ## Decisions (found {decisions!r})")
|
||||
goals = record.section("Goals")
|
||||
goals = section("Goals")
|
||||
report.check(isinstance(goals, str) and "Prove do-epic's commands" in goals, f"the decision leaves the description's ## Goals as it was (found {goals!r})")
|
||||
|
||||
|
||||
def closed(report: Report, machine: Machine, store: JiraEpic, epic_key: str) -> None:
|
||||
def closed(report: Report, machine: DoEpic, store: EpicStore, epic_key: str) -> None:
|
||||
early = machine.do_epic(epic_key, finish.CLOSE, stdin="Too early.\n")
|
||||
still = store.closed()
|
||||
report.check(early.returncode == 1 and " open" in early.stderr and still is False, f"`do-epic {finish.CLOSE}` refuses while issues are open and leaves the epic open (exit {early.returncode}: {early.stderr.strip()}; epic closed: {still!r})")
|
||||
|
||||
@@ -37,18 +37,6 @@ class MainTest(unittest.TestCase):
|
||||
self.assertIn(main.HELP, err)
|
||||
|
||||
|
||||
class MachineTest(unittest.TestCase):
|
||||
def test_stops_a_do_epic_that_runs_over_its_time_and_says_so(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
slow = pathlib.Path(folder) / "slow"
|
||||
slow.write_text("#!/bin/sh\nsleep 30\n", encoding="utf-8")
|
||||
slow.chmod(0o755)
|
||||
with mock.patch.object(scenario, "TIMEOUT", 1):
|
||||
done = scenario.Machine(pathlib.Path(folder), slow).do_epic("ABC-1")
|
||||
self.assertEqual(done.returncode, 124)
|
||||
self.assertIn("ran over 1 seconds", done.stderr)
|
||||
|
||||
|
||||
class ReportedTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
made = tempfile.TemporaryDirectory()
|
||||
@@ -90,7 +78,7 @@ class ReportedTest(unittest.TestCase):
|
||||
sandbox_set = settings.read(self.env)
|
||||
self.assertEqual((code, err), (0, ""))
|
||||
self.assertEqual(self.made_with, (sandbox_set, "do-epic e2e sandbox epic"))
|
||||
self.assertEqual(self.ran_with, (sandbox_set, "SBX-1", pathlib.Path("do-epic")))
|
||||
self.assertEqual(self.ran_with, (sandbox_set, "SBX-1", pathlib.Path("do-epic"), self.env))
|
||||
self.assertEqual(out, "Sandbox epic SBX-1 in https://acme.atlassian.net\nok a check held\n1 held, 0 broke\n")
|
||||
|
||||
def test_fails_without_a_scenario_where_the_epic_cannot_be_made(self) -> None:
|
||||
|
||||
@@ -20,6 +20,7 @@ class JiraStub:
|
||||
self.bodies: list[object] = []
|
||||
self.refuses = refuses
|
||||
self.search_answer: object = {"issues": [{"key": "SBX-2"}, {"key": "SBX-3"}]}
|
||||
self.later_pages: list[object] = []
|
||||
|
||||
def __call__(self, method: str, url: str, authorization: str, body: object) -> object:
|
||||
self.requests.append((method, url.split("/rest/api/3/")[1]))
|
||||
@@ -30,7 +31,7 @@ class JiraStub:
|
||||
if method == "POST" and url.endswith("/issue"):
|
||||
return {"key": "SBX-1"}
|
||||
if method == "POST":
|
||||
return self.search_answer
|
||||
return self.later_pages.pop(0) if self.later_pages and isinstance(body, dict) and "nextPageToken" in body else self.search_answer
|
||||
return None
|
||||
|
||||
|
||||
@@ -56,6 +57,15 @@ class EpicMadeTest(unittest.TestCase):
|
||||
("DELETE", "issue/SBX-1?deleteSubtasks=true"),
|
||||
])
|
||||
|
||||
def test_deletes_the_children_on_every_page_of_the_search(self) -> None:
|
||||
jira = JiraStub()
|
||||
jira.search_answer = {"issues": [{"key": "SBX-2"}], "nextPageToken": "more"}
|
||||
jira.later_pages = [{"issues": [{"key": "SBX-3"}]}]
|
||||
with epic_made(self.sandbox, "An epic", jira, lambda markdown: {"type": "doc"}):
|
||||
pass
|
||||
self.assertEqual(jira.bodies[2], {"jql": "parent = SBX-1", "fields": ["summary"], "maxResults": 100, "nextPageToken": "more"})
|
||||
self.assertEqual([url for method, url in jira.requests if method == "DELETE"], [f"issue/SBX-{n}?deleteSubtasks=true" for n in (2, 3, 1)])
|
||||
|
||||
def test_deletes_them_when_the_block_raises(self) -> None:
|
||||
jira = JiraStub()
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
@@ -75,7 +85,7 @@ class EpicMadeTest(unittest.TestCase):
|
||||
with self.assertRaises(Leftover) as left:
|
||||
with epic_made(self.sandbox, "An epic", jira, lambda markdown: {"type": "doc"}):
|
||||
pass
|
||||
self.assertIn("holds no issues", str(left.exception))
|
||||
self.assertIn("something other than a page of issues", str(left.exception))
|
||||
self.assertEqual(jira.requests[1:], [("POST", "search/jql"), ("DELETE", "issue/SBX-1?deleteSubtasks=true")])
|
||||
|
||||
def test_deletes_the_epic_all_the_same_and_gives_the_reason_when_the_search_for_its_issues_is_refused(self) -> None:
|
||||
@@ -84,7 +94,7 @@ class EpicMadeTest(unittest.TestCase):
|
||||
with epic_made(self.sandbox, "An epic", jira, lambda markdown: {"type": "doc"}):
|
||||
pass
|
||||
self.assertIn("POST https://acme.atlassian.net/rest/api/3/search/jql answered 403", str(left.exception))
|
||||
self.assertNotIn("holds no issues", str(left.exception))
|
||||
self.assertNotIn("something other than a page of issues", str(left.exception))
|
||||
self.assertEqual(jira.requests[1:], [("POST", "search/jql"), ("DELETE", "issue/SBX-1?deleteSubtasks=true")])
|
||||
|
||||
def test_yields_a_failure_and_deletes_nothing_when_the_epic_cannot_be_made(self) -> None:
|
||||
|
||||
@@ -1,24 +1,195 @@
|
||||
import dataclasses
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import typing
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
||||
|
||||
from do_epic.config import load
|
||||
from do_epic.e2e import scenario
|
||||
from do_epic.e2e.settings import Sandbox
|
||||
from do_epic.repo_pick import CLAUDE_LABEL
|
||||
from failure import Failure
|
||||
from do_epic import findings, finish
|
||||
from do_epic.config import load
|
||||
from do_epic.backend.fake import MemoryEpic
|
||||
from do_epic.e2e.settings import Sandbox
|
||||
from do_epic.e2e import scenario
|
||||
|
||||
EPIC = "SBX-1"
|
||||
|
||||
|
||||
def done(code: int, out: str = "", err: str = "") -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.CompletedProcess([], code, out, err)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Scripted:
|
||||
"""A do-epic that answers each call with the next of its answers, each a function that may also change the epic as do-epic would, and notes the call."""
|
||||
answers: list[typing.Callable[[], subprocess.CompletedProcess[str]]]
|
||||
calls: list[tuple[tuple[str, ...], str]] = dataclasses.field(default_factory=list)
|
||||
|
||||
def do_epic(self, *args: str, stdin: str = "") -> subprocess.CompletedProcess[str]:
|
||||
self.calls.append((args, stdin))
|
||||
return self.answers.pop(0)()
|
||||
|
||||
|
||||
class ReportTest(unittest.TestCase):
|
||||
def test_sorts_each_check_into_what_held_and_what_broke(self) -> None:
|
||||
report = scenario.Report()
|
||||
self.assertTrue(report.check(True, "one"))
|
||||
self.assertFalse(report.check(False, "two"))
|
||||
report.check(True, "one")
|
||||
report.check(False, "two")
|
||||
self.assertEqual((report.held, report.broke), (["one"], ["two"]))
|
||||
|
||||
|
||||
class ItemsFiledTest(unittest.TestCase):
|
||||
def test_holds_where_the_epic_answers_the_issues_in_order(self) -> None:
|
||||
report = scenario.Report()
|
||||
scenario.items_filed(report, MemoryEpic(EPIC), EPIC)
|
||||
self.assertEqual(report.broke, [])
|
||||
self.assertEqual(len(report.held), 4)
|
||||
|
||||
def test_breaks_where_the_issues_come_back_in_another_order(self) -> None:
|
||||
class Reversed(MemoryEpic):
|
||||
def read(self) -> list[typing.Any] | Failure:
|
||||
found = super().read()
|
||||
return found if isinstance(found, Failure) else found[::-1]
|
||||
|
||||
report = scenario.Report()
|
||||
scenario.items_filed(report, Reversed(EPIC), EPIC)
|
||||
self.assertEqual(len(report.held), 3)
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
self.assertIn("read the filed issues back in order", report.broke[0])
|
||||
|
||||
def test_breaks_naming_why_where_the_epic_refuses_every_call(self) -> None:
|
||||
report = scenario.Report()
|
||||
scenario.items_filed(report, MemoryEpic(EPIC, fails="Jira is down"), EPIC)
|
||||
self.assertEqual(report.held, [])
|
||||
self.assertEqual(report.broke[:3], [f"file {what} in the epic: Jira is down" for what in ("an item", "a second item", "a sub-task")])
|
||||
self.assertEqual(report.broke[3], "read the epic's issues after filing: Jira is down")
|
||||
self.assertIn("found []", report.broke[4])
|
||||
|
||||
|
||||
class FindingFiledTest(unittest.TestCase):
|
||||
def filing(self, store: MemoryEpic, labels: tuple[str, ...] = (CLAUDE_LABEL,), code: int = 0, out: str = "10:00 Filed SBX-100") -> typing.Callable[[], subprocess.CompletedProcess[str]]:
|
||||
def answer() -> subprocess.CompletedProcess[str]:
|
||||
store.file(EPIC, "Sandbox finding", "Why the finding matters.", labels)
|
||||
return done(code, out)
|
||||
return answer
|
||||
|
||||
def test_holds_where_do_epic_files_the_finding_with_its_label_and_detail(self) -> None:
|
||||
store = MemoryEpic(EPIC)
|
||||
machine = Scripted([self.filing(store)])
|
||||
report = scenario.Report()
|
||||
scenario.finding_filed(report, machine, store, EPIC)
|
||||
self.assertEqual((report.broke, len(report.held)), ([], 2))
|
||||
self.assertEqual(machine.calls, [((EPIC, findings.FILE, "Sandbox finding", findings.LABEL, CLAUDE_LABEL), "Why the finding matters.\n")])
|
||||
|
||||
def test_breaks_where_do_epic_fails_and_files_nothing(self) -> None:
|
||||
report = scenario.Report()
|
||||
scenario.finding_filed(report, Scripted([lambda: done(1, err="no")]), MemoryEpic(EPIC), EPIC)
|
||||
self.assertEqual(report.held, [])
|
||||
self.assertIn("exit 1: no", report.broke[0])
|
||||
self.assertIn("found []", report.broke[1])
|
||||
|
||||
def test_breaks_where_the_finding_lacks_the_label_asked_for(self) -> None:
|
||||
store = MemoryEpic(EPIC)
|
||||
report = scenario.Report()
|
||||
scenario.finding_filed(report, Scripted([self.filing(store, labels=())]), store, EPIC)
|
||||
self.assertEqual(len(report.held), 1)
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
self.assertIn("with its label", report.broke[0])
|
||||
|
||||
def test_breaks_where_do_epic_exits_cleanly_but_says_nothing_of_filing(self) -> None:
|
||||
store = MemoryEpic(EPIC)
|
||||
report = scenario.Report()
|
||||
scenario.finding_filed(report, Scripted([self.filing(store, out="")]), store, EPIC)
|
||||
self.assertEqual(len(report.held), 1)
|
||||
self.assertIn("files the finding and says so", report.broke[0])
|
||||
|
||||
|
||||
class DecisionRecordedTest(unittest.TestCase):
|
||||
SECTIONS: typing.ClassVar[dict[str, str | Failure]] = {"Decisions": "### Sandbox decision\nThe text of the decision.", "Goals": "1. Prove do-epic's commands work."}
|
||||
|
||||
def recorded(self, answer: subprocess.CompletedProcess[str], **sections: str | Failure) -> tuple[scenario.Report, Scripted]:
|
||||
machine = Scripted([lambda: answer])
|
||||
report = scenario.Report()
|
||||
found = {**self.SECTIONS, **sections}
|
||||
scenario.decision_recorded(report, machine, lambda name: found[name], EPIC)
|
||||
return report, machine
|
||||
|
||||
def test_holds_where_the_decision_is_in_the_description_and_the_goals_are_as_they_were(self) -> None:
|
||||
report, machine = self.recorded(done(0, "10:00 Recorded the decision"))
|
||||
self.assertEqual((report.broke, len(report.held)), ([], 3))
|
||||
self.assertEqual(machine.calls, [((EPIC, findings.RECORD_DECISION, "Sandbox decision"), "The text of the decision.\n")])
|
||||
|
||||
def test_breaks_where_the_decision_is_not_recorded(self) -> None:
|
||||
report, _ = self.recorded(done(1, err="no"), Decisions="")
|
||||
self.assertEqual(len(report.held), 1)
|
||||
self.assertIn("exit 1: no", report.broke[0])
|
||||
self.assertIn("holds the decision under ## Decisions", report.broke[1])
|
||||
|
||||
def test_breaks_where_only_half_of_the_decision_is_recorded(self) -> None:
|
||||
report, _ = self.recorded(done(0, "Recorded the decision"), Decisions="### Sandbox decision")
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
|
||||
def test_breaks_where_the_goals_changed(self) -> None:
|
||||
report, _ = self.recorded(done(0, "Recorded the decision"), Goals="")
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
self.assertIn("leaves the description's ## Goals", report.broke[0])
|
||||
|
||||
def test_breaks_without_raising_where_the_description_cannot_be_read(self) -> None:
|
||||
report, _ = self.recorded(done(0, "Recorded the decision"), Decisions=Failure("Jira is down"), Goals=Failure("Jira is down"))
|
||||
self.assertEqual((len(report.held), len(report.broke)), (1, 2))
|
||||
|
||||
|
||||
class ClosedTest(unittest.TestCase):
|
||||
def closing(self, store: MemoryEpic, early: typing.Callable[[], subprocess.CompletedProcess[str]] | None = None, late: typing.Callable[[], subprocess.CompletedProcess[str]] | None = None) -> scenario.Report:
|
||||
def closes() -> subprocess.CompletedProcess[str]:
|
||||
store.close("Everything the sandbox epic held is done.")
|
||||
return done(0, "10:00 Posted the summary and closed it")
|
||||
|
||||
store.file(EPIC, "An open issue", "", ())
|
||||
report = scenario.Report()
|
||||
self.machine = Scripted([early or (lambda: done(1, err="SBX-100 is still open")), late or closes])
|
||||
scenario.closed(report, self.machine, store, EPIC)
|
||||
return report
|
||||
|
||||
def test_holds_where_do_epic_refuses_while_an_issue_is_open_and_closes_the_epic_after(self) -> None:
|
||||
store = MemoryEpic(EPIC)
|
||||
report = self.closing(store)
|
||||
self.assertEqual((report.broke, len(report.held)), ([], 4))
|
||||
self.assertEqual(store.comments[EPIC], ["Everything the sandbox epic held is done."])
|
||||
self.assertEqual(self.machine.calls, [((EPIC, finish.CLOSE), "Too early.\n"), ((EPIC, finish.CLOSE), "Everything the sandbox epic held is done.\n")])
|
||||
self.assertTrue(all(issue.done for issue in store.issues))
|
||||
|
||||
def test_breaks_where_the_early_close_closes_the_epic(self) -> None:
|
||||
store = MemoryEpic(EPIC)
|
||||
|
||||
def closes_early() -> subprocess.CompletedProcess[str]:
|
||||
store.close("Too early.")
|
||||
return done(1, err="SBX-100 is still open")
|
||||
|
||||
report = self.closing(store, early=closes_early)
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
self.assertIn("refuses while issues are open", report.broke[0])
|
||||
|
||||
def test_breaks_where_the_late_close_exits_cleanly_but_leaves_the_epic_open(self) -> None:
|
||||
report = self.closing(MemoryEpic(EPIC), late=lambda: done(0, "Posted the summary and closed it"))
|
||||
self.assertEqual(len(report.broke), 1)
|
||||
self.assertIn("Jira counts the epic as done (found False)", report.broke[0])
|
||||
|
||||
def test_breaks_where_the_late_close_fails(self) -> None:
|
||||
report = self.closing(MemoryEpic(EPIC), late=lambda: done(1, err="refused"))
|
||||
self.assertIn("exit 1: refused", report.broke[0])
|
||||
|
||||
def test_breaks_without_raising_where_the_epic_cannot_be_read(self) -> None:
|
||||
report = self.closing(MemoryEpic(EPIC, fails="Jira is down"))
|
||||
self.assertEqual(len(report.held), 1)
|
||||
self.assertIn("read the epic's issues before finishing them: Jira is down", report.broke)
|
||||
self.assertIn("Jira counts the epic as done (found Failure(reason='Jira is down'))", report.broke)
|
||||
|
||||
|
||||
class MachineTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
made = tempfile.TemporaryDirectory()
|
||||
@@ -29,18 +200,33 @@ class MachineTest(unittest.TestCase):
|
||||
self.env_file.chmod(0o600)
|
||||
self.sandbox = Sandbox("https://acme.atlassian.net", self.env_file, "SBX")
|
||||
|
||||
def script(self, body: str) -> pathlib.Path:
|
||||
launcher = self.folder / "launcher"
|
||||
launcher.write_text(f"#!/bin/sh\n{body}\n", encoding="utf-8")
|
||||
launcher.chmod(0o755)
|
||||
return launcher
|
||||
|
||||
def test_writes_settings_that_do_epic_loads_and_that_point_at_the_sandbox(self) -> None:
|
||||
home = self.folder / "home"
|
||||
machine = scenario.machine_made(home, self.sandbox, pathlib.Path("do-epic"))
|
||||
machine = scenario.machine_made(home, self.sandbox, pathlib.Path("do-epic"), {"A": "b"})
|
||||
assert not isinstance(machine, Failure), machine
|
||||
self.assertEqual((machine.home, machine.launcher, machine.env), (home, pathlib.Path("do-epic"), {"A": "b"}))
|
||||
config = load(home)
|
||||
assert not isinstance(config, Failure), config
|
||||
self.assertEqual((config.jira_url, config.env_file, config.repo_rule), ("https://acme.atlassian.net", self.env_file, scenario.RULE))
|
||||
self.assertEqual((config.finish.transition, config.finish.status) if config.finish else None, ("Done", "Done"))
|
||||
|
||||
def test_runs_do_epic_with_the_machines_home_and_the_words_and_stdin_given(self) -> None:
|
||||
launcher = self.folder / "echo-home"
|
||||
launcher.write_text('#!/bin/sh\necho "$HOME $*"; cat\n', encoding="utf-8")
|
||||
launcher.chmod(0o755)
|
||||
done = scenario.Machine(self.folder, launcher).do_epic("ABC-1", "--file", stdin="detail")
|
||||
self.assertEqual((done.returncode, done.stdout), (0, f"{self.folder} ABC-1 --file\ndetail"))
|
||||
def test_runs_do_epic_in_the_given_environment_with_the_machines_home_and_the_words_and_stdin_given(self) -> None:
|
||||
machine = scenario.Machine(self.folder, self.script('echo "$HOME $KEPT $*"; cat'), {"KEPT": "yes", "HOME": "/elsewhere"})
|
||||
finished = machine.do_epic("ABC-1", "--file", stdin="detail")
|
||||
self.assertEqual((finished.returncode, finished.stdout), (0, f"{self.folder} yes ABC-1 --file\ndetail"))
|
||||
|
||||
def test_answers_a_failing_do_epic_with_its_exit_and_output(self) -> None:
|
||||
finished = scenario.Machine(self.folder, self.script("echo out; echo err >&2; exit 3"), {}).do_epic("ABC-1")
|
||||
self.assertEqual((finished.returncode, finished.stdout, finished.stderr), (3, "out\n", "err\n"))
|
||||
|
||||
def test_stops_a_do_epic_that_runs_over_its_time_and_says_so(self) -> None:
|
||||
finished = scenario.Machine(self.folder, self.script("sleep 30"), {}, timeout=1).do_epic("ABC-1")
|
||||
self.assertEqual(finished.returncode, 124)
|
||||
self.assertIn("ran over 1 seconds", finished.stderr)
|
||||
self.assertIn("ABC-1", finished.stderr)
|
||||
|
||||
@@ -5,7 +5,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
||||
|
||||
from do_epic.e2e.settings import ENV_FILE, JIRA_URL, MEANING, PROJECT, read, Sandbox
|
||||
from do_epic.e2e.settings import ENV_FILE, JIRA_URL, PROJECT, read, Sandbox
|
||||
from failure import Failure
|
||||
|
||||
|
||||
@@ -33,11 +33,13 @@ class SettingsTest(unittest.TestCase):
|
||||
self.assertNotIn(PROJECT, self.failure({k: v for k, v in self.env.items() if k != JIRA_URL}))
|
||||
|
||||
def test_treats_a_blank_setting_as_missing(self) -> None:
|
||||
self.assertEqual(self.failure({**self.env, ENV_FILE: " "}), f"{ENV_FILE} is missing: set it to {MEANING[ENV_FILE]}")
|
||||
self.assertIn(f"{ENV_FILE} is missing", self.failure({**self.env, ENV_FILE: " "}))
|
||||
|
||||
def test_names_every_fault_at_once_separated_by_semicolons(self) -> None:
|
||||
self.env_file.chmod(0o644)
|
||||
self.assertEqual(self.failure({**self.env, PROJECT: "sbx-1"}), f"{PROJECT} is 'sbx-1', not a project key such as SBX; {self.env_file} holds an API token, so only its owner may read it; run chmod 600 {self.env_file}")
|
||||
reason = self.failure({**self.env, PROJECT: "sbx-1"})
|
||||
self.assertTrue(reason.startswith(f"{PROJECT} is"))
|
||||
self.assertIn(f"; {self.env_file} ", reason)
|
||||
|
||||
def test_refuses_a_site_that_is_not_an_https_address_without_a_path(self) -> None:
|
||||
self.assertIn(JIRA_URL, self.failure({**self.env, JIRA_URL: "http://acme.atlassian.net/jira"}))
|
||||
@@ -46,9 +48,10 @@ class SettingsTest(unittest.TestCase):
|
||||
self.assertIn(PROJECT, self.failure({**self.env, PROJECT: "sbx-1"}))
|
||||
|
||||
def test_refuses_an_env_file_that_cannot_be_read_saying_why(self) -> None:
|
||||
self.assertIn("does not exist", self.failure({**self.env, ENV_FILE: str(self.env_file.parent / "gone.env")}))
|
||||
gone = self.env_file.parent / "gone.env"
|
||||
self.assertIn(str(gone), self.failure({**self.env, ENV_FILE: str(gone)}))
|
||||
self.env_file.chmod(0o644)
|
||||
self.assertIn("only its owner may read it", self.failure(self.env))
|
||||
self.assertIn(str(self.env_file), self.failure(self.env))
|
||||
|
||||
def test_refuses_an_env_file_that_is_not_an_absolute_path(self) -> None:
|
||||
self.assertIn(ENV_FILE, self.failure({**self.env, ENV_FILE: "jira.env"}))
|
||||
|
||||
Reference in New Issue
Block a user