Answer stability round 2: 0 🔴, 2 🟡 (0 misread); name an issue by its key where the ranking words the failure
CI / tests (pull_request) Successful in 3m44s

This commit is contained in:
Lilleman
2026-11-20 16:40:43 +01:00
parent b01b799743
commit 9ac35a4699
4 changed files with 26 additions and 11 deletions
+3 -5
View File
@@ -234,7 +234,7 @@ def scores_set(epic: Planned, issue: EpicIssue, answer: issue_scores.Scores, wei
if isinstance(fields, Failure):
return fields
held = dataclasses.replace(issue, scores=fields.scores, moves=fields.moves)
ranked = ranking.ranked([ranking.EpicItem(item_of(held, {issue.key: number(issue.key)}, ()), repo_of(epic, issue))], weighed.formula, weighed.goals, weighed.repos)
ranked = ranking.ranked([ranking.EpicItem(item_of(held, {issue.key: number(issue.key)}, ()), repo_of(epic, issue), issue.key)], weighed.formula, weighed.goals, weighed.repos)
return ranked if isinstance(ranked, Failure) else epic.store.set_scores(issue.key, answer.scores, answer.moves, answer.comment) or held
@@ -258,11 +258,9 @@ def start_order(epic: Planned, issues: list[EpicIssue]) -> list[str] | Failure:
if isinstance(weighed, Failure):
return weighed
by_key = {issue.key: issue for issue in issues}
items = [ranking.EpicItem(item_of(issue, numbers, before(issue, by_key, issues)), repo_of(epic, issue)) for issue in held]
items = [ranking.EpicItem(item_of(issue, numbers, before(issue, by_key, issues)), repo_of(epic, issue), issue.key) for issue in held]
ranked = ranking.ranked(items, weighed.formula, weighed.goals, weighed.repos)
if isinstance(ranked, Failure):
return Failure(re.sub(r"\bitem (\d+)\b", lambda named: keys[int(named[1])], ranked.reason))
return [keys[item.id] for item, _ in ranked]
return ranked if isinstance(ranked, Failure) else [keys[item.id] for item, _ in ranked]
def before(issue: EpicIssue, by_key: dict[str, EpicIssue], issues: list[EpicIssue]) -> tuple[str, ...]:
+7 -3
View File
@@ -7,7 +7,7 @@ import ast
import dataclasses
import decimal
from plan.blockers import blockers_first
from plan.blockers import blockers_first, blocking_loop
from plan.todo_table import CODE_HEALTH, EPIC_GOAL, Item, REPO_GOAL, SHAPE
from do_epic.backend.protocol import EpicStore
from failure import Failure
@@ -31,9 +31,10 @@ class RepoRanking:
@dataclasses.dataclass(frozen=True)
class EpicItem:
"""One of an epic's open items. An Item names a release and an epic item has none: give every item of one epic the same release, so that no blocker sits in a later release than what it blocks. repo is the name of the repo the item is worked in; None where it has none."""
"""One of an epic's open items. An Item names a release and an epic item has none: give every item of one epic the same release, so that no blocker sits in a later release than what it blocks. repo is the name of the repo the item is worked in; None where it has none. name is what a failure calls the item, such as its issue's key."""
item: Item
repo: str | None
name: str
def ranked(items: list[EpicItem], formula: str, epic_goals: str, repos: dict[str, RepoRanking]) -> list[tuple[Item, float]] | Failure:
@@ -44,7 +45,7 @@ def ranked(items: list[EpicItem], formula: str, epic_goals: str, repos: dict[str
scorings: dict[str | None, ranking.Scoring] = {}
scored = []
for each in items:
name = f"item {each.item.id}"
name = each.name
if each.repo not in scorings:
if each.repo is not None and each.repo not in repos:
return Failure(f"{name} is in the repo {each.repo}, whose goals were not read")
@@ -57,6 +58,9 @@ def ranked(items: list[EpicItem], formula: str, epic_goals: str, repos: dict[str
return value
scored.append((each.item, value))
scored.sort(key=lambda pair: (ranking.FIRST.get(pair[0].exempt, len(ranking.FIRST)), -pair[1], pair[0].id))
names = {each.item.id: each.name for each in items}
if loop := blocking_loop({item.id: (item, value) for item, value in scored}):
return Failure(f"{', '.join(names[id_] for id_ in loop[:-1])} and {names[loop[-1]]} block each other in a loop; drop one of the blockers")
return blockers_first(scored)
+10 -1
View File
@@ -339,6 +339,15 @@ class Planning(DoEpicTest):
self.plan(FakeEffects())
self.assertEqual(planning.start_order(self.epic, self.issues), Failure("ABC-20 names epic 7, which the epic's ## Goals does not number"))
def test_quotes_a_formula_that_names_an_item_as_the_epic_states_it(self) -> None:
self.description = RANKED.replace("Score = V", "Score = item 1")
self.issues = [scored("ABC-30", "epic 1 by 5", "do-epic-claude", by_loop=False), scored("ABC-20", "epic 1 by 5", "do-epic-claude", by_loop=False)]
self.plan(FakeEffects())
failed = planning.start_order(self.epic, self.issues)
assert isinstance(failed, Failure), failed
self.assertIn("item 1", failed.reason)
self.assertNotIn("ABC-30", failed.reason)
def test_fails_where_the_description_cannot_be_read_to_rank_by(self) -> None:
self.description = Failure("Jira answered 503")
self.issues = [scored("ABC-20", "epic 1 by 2", "do-epic-claude"), scored("ABC-21", "epic 1 by 9", "do-epic-claude")]
@@ -412,7 +421,7 @@ class Planning(DoEpicTest):
found, store, out = self.plan(fake)
assert not isinstance(found, Failure | Limited), found
self.assertIn('ABC-20 is not scored: the scoring answered {"L": "big"}; expected a whole number for each of L, E, I, S, the moves as text and a comment\n', out)
self.assertIn("ABC-21 is not scored: item 21 names epic 7, which the epic's ## Goals does not number\n", out)
self.assertIn("ABC-21 is not scored: ABC-21 names epic 7, which the epic's ## Goals does not number\n", out)
self.assertIn("ABC-22 is not scored: ", out)
self.assertEqual([each.key for each in found if each.scores], ["ABC-23"])
self.assertEqual([write.split(" L ")[0] for write in store.writes], ["score ABC-23"])
+6 -2
View File
@@ -24,7 +24,7 @@ TOTAL = 1 + 0.97
def item(id_: int, moves: str, repo: str | None = "billing-api", exempt: str = "", blocked_by: tuple[int, ...] = ()) -> EpicItem:
parsed = parse_moves(f"item {id_}", moves)
assert not isinstance(parsed, Failure), parsed
return EpicItem(Item(id_, (0, 0, 0), exempt, f"**Item {id_}.**", {"L": 1, "E": 1, "I": 1, "S": 1}, parsed, blocked_by), repo) # type: ignore[arg-type]
return EpicItem(Item(id_, (0, 0, 0), exempt, f"**Item {id_}.**", {"L": 1, "E": 1, "I": 1, "S": 1}, parsed, blocked_by), repo, f"item {id_}") # type: ignore[arg-type]
def order(items: list[EpicItem], repos: dict[str, RepoRanking], formula: str = "V") -> list[int] | Failure:
@@ -117,7 +117,7 @@ class RankedTest(unittest.TestCase):
self.assertAlmostEqual(value, expected)
def test_an_item_without_a_score_fails_naming_it(self) -> None:
unscored = EpicItem(dataclasses.replace(item(4, "").item, scores={}), None)
unscored = EpicItem(dataclasses.replace(item(4, "").item, scores={}), None, "item 4")
self.assertEqual(ranked([unscored], "V", EPIC_GOALS, {}), Failure("item 4 has no L: score it on L, E, I and S before it is ranked"))
def test_an_epic_that_numbers_a_goal_twice_fails(self) -> None:
@@ -170,6 +170,10 @@ class RankedTest(unittest.TestCase):
def test_a_blocker_ranks_above_the_item_it_blocks_and_a_tie_goes_to_the_older_item(self) -> None:
self.assertEqual(order([item(1, "epic 1 by 9", blocked_by=(3,)), item(2, "epic 1 by 2"), item(3, "epic 1 by 1"), item(5, "epic 1 by 2")], {"billing-api": VOIDED}), [3, 1, 2, 5])
def test_names_by_their_names_the_items_that_block_each_other_in_a_loop(self) -> None:
looped = [dataclasses.replace(item(1, "epic 1 by 9", blocked_by=(2,)), name="ABC-1"), dataclasses.replace(item(2, "epic 1 by 2", blocked_by=(1,)), name="XYZ-1"), item(3, "epic 1 by 1")]
self.assertEqual(ranked(looped, "V", EPIC_GOALS, {"billing-api": VOIDED}), Failure("ABC-1 and XYZ-1 block each other in a loop; drop one of the blockers"))
def test_a_formula_that_cannot_be_read_fails(self) -> None:
self.assertIsInstance(order([item(1, "epic 1 by 9")], {"billing-api": VOIDED}, "V +"), Failure)