Test the scoring call, its answer and the refusals a mutant survived
CI / tests (pull_request) Has been cancelled

This commit is contained in:
Lilleman
2026-11-20 15:45:45 +01:00
parent af81dcdcbc
commit 481e13253a
5 changed files with 113 additions and 0 deletions
+3
View File
@@ -236,6 +236,9 @@ class EpicStoreContract:
refusals = [store.set_scores("ZZZ-1", SCORES, "", "Scored."), store.set_scores("not a key", SCORES, "", "Scored."), store.set_scores(key, SCORES | {"L": 9}, "", "Scored."), store.set_scores(key, {"L": 1}, "", "Scored."), store.set_scores(key, SCORES | {"R": 1}, "", "Scored."), store.set_scores(key, SCORES, "the import by 5", "Scored."), store.set_scores(key, SCORES, "", " \n")]
for refused in refusals:
self.assertIsInstance(refused, Failure)
self.assertEqual(refusals[1], Failure(f"not a key is not one of the child issues of the epic {store.epic_key}, so it cannot be scored"))
assert isinstance(refusals[3], Failure)
self.assertEqual(refusals[3].reason[refusals[3].reason.rindex(key):], f"{key} is given L: set each of L, E, I and S, and nothing else")
[issue] = self.issues(store)
self.assertEqual((issue.scores, self.comments_on(store, key)), ({}, []))
+20
View File
@@ -221,6 +221,26 @@ class Jira(Case):
self.jira.answers[("POST", TRANSITIONS)] = Failure("Jira answered 503")
self.assertEqual(self.new_store().close("Shipped."), Failure("posted the summary on ABC-12, but cannot move ABC-12 by 'Mark as done': Jira answered 503; close the epic in Jira rather than posting the summary again"))
def test_scores_a_sub_task_of_a_child_but_no_issue_of_another_epic(self) -> None:
store = self.new_store()
key = store.file("ABC-12", "Count the runs.", "", ())
assert isinstance(key, str), key
task = store.file(key, "Count them in the API.", "", ())
assert isinstance(task, str), task
scores = {"L": 2, "E": 1, "I": 2, "S": 3}
self.assertIsNone(store.set_scores(task, scores, "", "Scored."))
read = f"{ISSUES}/{key}?fields=parent&properties=plan-item"
self.jira.answers[("GET", read)] = {"fields": {"parent": {"key": "XYZ-1"}}}
refused = store.set_scores(key, scores, "", "Scored.")
assert isinstance(refused, Failure), refused
self.assertTrue(refused.reason.startswith("cannot read XYZ-1 to score its sub-task: "), refused.reason)
for answer in ({"fields": {"parent": {"key": task}}}, {"fields": {"parent": {"key": 12}}}, {"fields": {}}, {"fields": "none"}, []):
with self.subTest(answer=answer):
self.jira.answers[("GET", read)] = answer
self.assertEqual(store.set_scores(key, scores, "", "Scored."), Failure(f"{key} is not one of the child issues of the epic ABC-12, so it cannot be scored"))
self.jira.answers[("GET", read)] = {"fields": {"parent": {"key": "ABC-12"}}, "properties": []}
self.assertIsNone(store.set_scores(key, scores, "", "Scored."))
def test_says_how_far_a_scoring_came_where_jira_fails(self) -> None:
store = self.new_store()
key = store.file("ABC-12", "Count the runs.", "", ())
+76
View File
@@ -0,0 +1,76 @@
import json
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from do_epic.issue_scores import answer_of, asked, prompt, SCHEMA, Scores
from do_epic.test_repo_pick import Claude, says
from do_epic.backend.protocol import EpicIssue
from failure import Failure
from model_routing.ladder import Rung
ANSWER = {"L": 2, "E": 1, "I": 2, "S": 3, "moves": " epic 1 by 5 ", "comment": " Scored to pick what starts first. "}
ISSUE = EpicIssue("ABC-13", "Count the runs", "Count each run once.", ("api",), None, "To Do", False, "ABC-12")
def expected(output: object) -> Failure:
return Failure(f"the scoring answered {json.dumps(output)}; expected a whole number for each of L, E, I, S, the moves as text and a comment")
class TheAnswer(unittest.TestCase):
def test_reads_the_scores_the_moves_and_the_comment(self) -> None:
self.assertEqual(answer_of(ANSWER), Scores({"L": 2, "E": 1, "I": 2, "S": 3}, "epic 1 by 5", "Scored to pick what starts first."))
def test_refuses_an_answer_that_is_not_an_object(self) -> None:
self.assertEqual(answer_of(["L", 2]), expected(["L", 2]))
def test_refuses_a_score_that_is_not_a_whole_number(self) -> None:
for score in ("2", 2.0, True, None):
with self.subTest(score=score):
self.assertEqual(answer_of(ANSWER | {"I": score}), expected(ANSWER | {"I": score}))
def test_refuses_moves_that_are_not_text(self) -> None:
self.assertEqual(answer_of(ANSWER | {"moves": None}), expected(ANSWER | {"moves": None}))
def test_refuses_an_answer_without_a_comment(self) -> None:
for comment in (None, " ", 5):
with self.subTest(comment=comment):
self.assertEqual(answer_of(ANSWER | {"comment": comment}), expected(ANSWER | {"comment": comment}))
def test_quotes_the_first_200_characters_of_an_answer_it_refuses(self) -> None:
long = {"comment": "c" * 300}
self.assertEqual(answer_of(long), Failure(f"the scoring answered {json.dumps(long)[:200]}; expected a whole number for each of L, E, I, S, the moves as text and a comment"))
class TheCall(unittest.TestCase):
def ask(self, claude: Claude, issue: EpicIssue = ISSUE) -> object:
return asked(issue, "L is the chance of a defect.", "An item moves only these: `epic 1`.", "Swedish", Rung("claude-sonnet-5-5", "medium"), claude, "/state/scoring")
def test_asks_claude_for_a_json_reply_on_the_given_rung_and_without_tools(self) -> None:
claude = says(ANSWER)
self.assertEqual(self.ask(claude), answer_of(ANSWER))
args, cwd, _ = claude.calls[0]
self.assertEqual(cwd, "/state/scoring")
self.assertEqual(args[:9], ["-p", "--json-schema", json.dumps(SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", ""])
def test_names_the_scoring_where_claude_fails(self) -> None:
failed = self.ask(Claude("", 1))
assert isinstance(failed, Failure), failed
self.assertIn("the scoring", failed.reason)
def test_puts_the_rule_the_goals_the_language_and_the_item_in_the_prompt(self) -> None:
text = prompt(ISSUE, "L is the chance of a defect.", "An item moves only these: `epic 1`.", "Swedish")
for part in ("L is the chance of a defect.", "An item moves only these: `epic 1`.", "Write the comment in Swedish", "## Item ABC-13: Count the runs", "Labels: api", "Count each run once."):
with self.subTest(part=part):
self.assertIn(part, text)
self.assertNotIn("(no description)", text)
def test_says_where_the_item_has_no_labels_or_description(self) -> None:
text = prompt(EpicIssue("ABC-13", "Count the runs", "", (), None, "To Do", False, "ABC-12"), "Rule.", "Goals.", "English")
self.assertTrue(text.endswith("Labels: none\n\n(no description)\n"), text[-60:])
if __name__ == "__main__":
unittest.main()
+12
View File
@@ -1058,6 +1058,18 @@ class TheDryRun(DoEpicTest):
self.assertIn("would start ABC-21, ABC-20\n", shown)
self.assertEqual(self.stores[0].writes, [])
def test_says_why_the_issues_start_in_backlog_order_and_names_no_order_of_one_issue(self) -> None:
self.issues = [dataclasses.replace(issue(key, "do-epic-claude"), scores={"L": 1, "E": 1, "I": 1, "S": 1}) for key in ("ABC-20", "ABC-21")]
out = io.StringIO()
with contextlib.redirect_stdout(out):
main.chunks_started(self.epic_run(FakeEffects(on_terminal=False)), self.issues, None)
self.assertIn("the issues of ABC-12 start in Jira's backlog order, since they cannot be put in order of score: the epic ABC-12's description states `Score = ...` 0 times under ## Scoring", out.getvalue())
self.description = DESCRIPTION.replace("Ship the import.", "1. **Achieve:** Ship the import.") + "\n## Scoring\n`Score = V`\n"
self.issues = self.issues[:1]
code, out, err = self.run_main(["ABC-12", "--dry-run"], FakeEffects(on_terminal=False))
self.assertEqual((code, err), (0, ""))
self.assertNotIn("by score", out)
def test_clones_no_missing_repo_and_says_so(self) -> None:
shutil.rmtree(self.home / "code/web")
self.description = DESCRIPTION + "\n## Scoring\n`Score = V`\n"
+2
View File
@@ -355,6 +355,8 @@ class Planning(DoEpicTest):
self.rule()
self.description = RANKED
self.issues = [scored("ABC-20", "epic 1 by 2", "do-epic-claude", by_loop=False), issue("ABC-21", "do-epic-claude")]
# An earlier run made the folder the scoring runs in.
(self.home / planning.STATE / "ABC-12").mkdir(parents=True, exist_ok=True)
fake = FakeEffects(claude=[says(ANSWER)])
found, store, out = self.plan(fake)
assert not isinstance(found, Failure | Limited), found