Answer stability round 1: 0 🔴, 1 🟡 (0 misread); split the re-rank into the pure rank_moves, rank only just before, as the item says, and name ranking in the README and the protocol

This commit is contained in:
2026-11-18 19:56:29 +01:00
parent 7bd192735a
commit ddce0f2cb0
10 changed files with 93 additions and 57 deletions
+1 -1
View File
@@ -214,7 +214,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
- `branch_chunks/` cuts a default branch into chunks and finds, for each, the items it shipped and the later chunks that fixed its code, for `score_outcomes` and `routing_figures`. It imports `reading_cost`'s record path.
- `command_lines/` reads a shell command: its simple commands, and a docker call's subcommand and memory limits, for `command_patterns`, `reading_cost`, `routing_figures`, `rule_hook` and `run_heavy`.
- `deploy_release` imports `release_plan`'s `release_standing`.
- `do_epic/` will hold the tool that drives one Jira epic. So far `config.py` reads its machine-wide settings, importing `project_config`'s settings and `model_routing`'s ladder file, and `epic_record.py` reads and writes an epic's description by section and language line, through `record_store`'s `EpicDescription`. `repos.py` finds a repo's checkout and project config by its name, and clones a missing checkout from the host and owner of the epic's other repos, through `forge`'s `url_beside`. `repo_pick.py` asks claude, by `repo-rule.md`, where an epic item is worked, and records the answer as a label on the issue. `ranking.py` orders an epic's items by score over one chain of goals, the epic's, then code health, then the item's repo's, through `plan`'s ranking. `backend/` reads and files the epic's issues and adds those labels.
- `do_epic/` will hold the tool that drives one Jira epic. So far `config.py` reads its machine-wide settings, importing `project_config`'s settings and `model_routing`'s ladder file, and `epic_record.py` reads and writes an epic's description by section and language line, through `record_store`'s `EpicDescription`. `repos.py` finds a repo's checkout and project config by its name, and clones a missing checkout from the host and owner of the epic's other repos, through `forge`'s `url_beside`. `repo_pick.py` asks claude, by `repo-rule.md`, where an epic item is worked, and records the answer as a label on the issue. `ranking.py` orders an epic's items by score over one chain of goals, the epic's, then code health, then the item's repo's, through `plan`'s ranking, and re-ranks the tracker's backlog to that order through `backend/`. `backend/` reads, files and ranks the epic's issues and adds those labels.
- `forge/` tells a GitHub origin from a Gitea one, names a repo beside an origin's and runs that forge's CLI (`commands.py`), and finds a pull request and waits for its checks (`pulls.py`), for `chunk_preflight`, `chunk_pr`, `deploy_release`, `do_epic` and `larv_review`.
- `git_cli/` runs git for every tool (`git_repo.py`), and reads the git calls in a shell command (`git_command.py`).
- `judge_replay` imports `agent_cycle`'s judge and verdicts, and `loop_claude`'s premises, records and `ANSWER_DESTINATIONS`.
+3 -3
View File
@@ -151,11 +151,11 @@ class EpicStoreContract:
read = {issue.key: issue for issue in self.issues(store)}
self.assertEqual([read[key].by_loop for key in (ours, theirs, task)], [True, False, True])
def test_ranks_a_child_issue_just_before_or_just_after_another_and_its_sub_tasks_follow_it(self) -> None:
def test_ranks_a_child_issue_just_before_another_and_its_sub_tasks_follow_it(self) -> None:
store = self.new_store()
first, second, third = (self.filed(store, store.epic_key, f"Item {n}.") for n in range(3))
task = self.filed(store, third, "A task.")
self.assertEqual(store.rank(third, first), None)
self.assertEqual([issue.key for issue in self.issues(store)], [third, task, first, second])
self.assertEqual(store.rank(third, second, after=True), None)
self.assertEqual([issue.key for issue in self.issues(store)], [first, second, third, task])
self.assertEqual(store.rank(first, third), None)
self.assertEqual([issue.key for issue in self.issues(store)], [first, third, task, second])
+5 -5
View File
@@ -7,7 +7,7 @@ from failure import Failure
@dataclasses.dataclass
class MemoryEpic:
"""An epic whose issues rank in the order filed, until one is ranked. Where fails is set, every call fails with it as the reason."""
"""An epic whose issues rank in the order filed, until one is ranked. Where fails is set, every call but change fails with it as the reason."""
epic_key: str
issues: list[EpicIssue] = dataclasses.field(default_factory=list)
fails: str | None = None
@@ -39,14 +39,14 @@ class MemoryEpic:
self.change(key, labels=tuple(sorted({*child.labels, label})))
return None
def rank(self, key: str, other: str, after: bool = False) -> None | Failure:
def rank(self, key: str, before: str) -> None | Failure:
if self.fails:
return Failure(self.fails)
children = {issue.key: issue for issue in self.issues if issue.parent == self.epic_key}
if stray := next((each for each in (key, other) if each not in children), None):
return Failure(f"{stray} is not one of the child issues of the epic {self.epic_key}, so it cannot be ranked")
if stray := next((each for each in (key, before) if each not in children), None):
return Failure(f"{stray} is not one of the child issues of the epic {self.epic_key}, so {key} is not ranked")
self.issues.remove(children[key])
self.issues.insert(self.issues.index(children[other]) + after, children[key])
self.issues.insert(self.issues.index(children[before]), children[key])
return None
def change(self, key: str, **fields: object) -> None:
+2 -2
View File
@@ -92,11 +92,11 @@ class JiraEpic:
edited = self.http("PUT", f"{self.epic.site}/rest/api/3/issue/{key}", authorization, {"update": {"labels": [{"add": label}]}})
return Failure(f"cannot label {key}: {edited.reason}") if isinstance(edited, Failure) else None
def rank(self, key: str, other: str, after: bool = False) -> None | Failure:
def rank(self, key: str, before: str) -> None | Failure:
authorization = basic_auth(self.env_file)
if isinstance(authorization, Failure):
return authorization
refused = rank_refusal(self.epic.site, key, other, after, authorization, self.http)
refused = rank_refusal(self.epic.site, key, before, authorization, self.http)
return Failure(refused) if refused else None
def not_a_parent(self, parent: str, authorization: str) -> Failure | None:
+3 -3
View File
@@ -1,4 +1,4 @@
"""What every backend of an epic's plan reads, files and labels: an EpicIssue, and the EpicStore protocol they all meet. do-epic reads an epic only through EpicStore, so another tracker can stand behind it; contract.py holds the tests every backend passes.
"""What every backend of an epic's plan reads, files, labels and ranks: an EpicIssue, and the EpicStore protocol they all meet. do-epic reads an epic only through EpicStore, so another tracker can stand behind it; contract.py holds the tests every backend passes.
EpicStore is not plan_store's PlanStore. A PlanStore item names a release, but an epic item has none (`docs/decisions.md` → `do-epic: drives one Jira epic …`), and it may be a sub-task."""
import dataclasses
@@ -57,8 +57,8 @@ class EpicStore(typing.Protocol):
def add_label(self, key: str, label: str) -> None | Failure:
"""Adds label to the epic's child issue named key, leaving its other labels as they are; adding one it already has leaves the labels as they are. A sub-task, or an issue outside the epic, is refused."""
def rank(self, key: str, other: str, after: bool = False) -> None | Failure:
"""Ranks the child issue named key just before the child issue named other, or just after it where after is True; its sub-tasks follow it. Both must be child issues of the epic, as read answers them: a backend may not check, and Jira ranks any issue it is given."""
def rank(self, key: str, before: str) -> None | Failure:
"""Ranks the child issue named key just before the child issue named before. The sub-tasks of key follow key. Pass only keys of the epic's child issues, as read answers them: some backends don't check, and Jira ranks any issue it is given."""
def unfit(summary: str, labels: tuple[str, ...]) -> Failure | None:
+1 -1
View File
@@ -44,7 +44,7 @@ class Ranking(unittest.TestCase):
def test_refuses_to_rank_beside_an_issue_that_isnt_a_child_of_the_epic(self) -> None:
store = MemoryEpic("ABC-12")
child = store.file("ABC-12", "Count the runs.", "", ())
self.assertEqual(store.rank(str(child), "ZZZ-1"), Failure("ZZZ-1 is not one of the child issues of the epic ABC-12, so it cannot be ranked"))
self.assertEqual(store.rank(str(child), "ZZZ-1"), Failure(f"ZZZ-1 is not one of the child issues of the epic ABC-12, so {child} is not ranked"))
if __name__ == "__main__":
+3 -3
View File
@@ -91,12 +91,12 @@ class FakeJira:
return {"id": "1", "key": key}
def ranked(self, body: object) -> object:
"""Moves the one issue named just before or just after another, and answers no body, as Jira does for a rank it made."""
"""Moves the issue the body names to just before the other issue it names, and answers no body, as Jira does for a rank it made."""
assert isinstance(body, dict)
[key], other = body["issues"], body.get("rankBeforeIssue") or body["rankAfterIssue"]
[key], before = body["issues"], body["rankBeforeIssue"]
moved = next(issue for issue in self.issues if issue["key"] == key)
self.issues.remove(moved)
self.issues.insert(next(place for place, issue in enumerate(self.issues) if issue["key"] == other) + ("rankAfterIssue" in body), moved)
self.issues.insert(next(place for place, issue in enumerate(self.issues) if issue["key"] == before), moved)
return None
def change(self, key: str, **fields: object) -> None:
+37 -18
View File
@@ -60,35 +60,54 @@ def ranked(items: list[EpicItem], formula: str, epic_goals: str, repos: dict[str
return blockers_first(scored)
def reranked(store: EpicStore, order: list[str]) -> None | Failure:
"""Ranks the epic's child issues named in order, each one the loop filed, so the tracker's backlog lists them in order. Each other child issue, such as one a person filed, keeps its slot; the slots of the named issues take them in order's order. A sub-task holds no slot and follows its parent.
@dataclasses.dataclass(frozen=True)
class Rank:
"""One rank to ask of a tracker: the issue named key just before the issue named before."""
key: str
before: str
The walk goes from the backlog's end: an issue not just before the one that follows it in the target ranks just before that one, so only the loop's issues move. The target's last issue, where it isn't already last, ranks just after the issue that is. A rank the tracker refuses stops the walk, which fails naming it, since a wrongly ranked team backlog takes hours to restore; the ranks made before it stay."""
def reranked(store: EpicStore, order: list[str]) -> None | Failure:
"""Re-ranks the tracker's backlog so that the child issues whose keys order lists come in that sequence. Each key must name a child issue the loop filed, and only once; otherwise the call fails before it ranks any issue. The listed issues change positions only with each other: every other child issue, such as one a person filed, stays at its position in the backlog. A sub-task has no position of its own and follows its parent.
The first rank the tracker refuses ends the re-rank with a Failure that names it, since a wrongly ranked team backlog takes hours to restore; the ranks made before it stay."""
read = store.read()
if isinstance(read, Failure):
return read
children = [issue for issue in read if issue.parent == store.epic_key]
if stray := next((key for key in order if key not in {issue.key for issue in children if issue.by_loop}), None):
ours = {issue.key for issue in children if issue.by_loop}
if stray := next((key for key in order if key not in ours), None):
return Failure(f"{stray} is not one of the child issues of the epic {store.epic_key} that the loop filed, so the backlog was not re-ranked")
if twice := next((key for key in order if order.count(key) > 1), None):
return Failure(f"the order names {twice} twice, so the backlog was not re-ranked")
now = [issue.key for issue in children]
slots = iter(order)
target = [next(slots) if key in order else key for key in now]
for place in reversed(range(len(target))):
key = target[place]
at = now.index(key)
# The slices are empty past the end, so the last issue already in place is skipped too.
if key not in order or now[at + 1:at + 2] == target[place + 1:place + 2]:
continue
other, after = (now[-1], True) if place == len(target) - 1 else (target[place + 1], False)
if refused := store.rank(key, other, after):
return Failure(f"cannot rank {key} just {'after' if after else 'before'} {other}: {refused.reason}; the re-rank stopped there, so rank the loop's issues in {store.epic_key}'s backlog by hand as {', '.join(order)}")
now.remove(key)
now.insert(now.index(other) + after, key)
for rank in rank_moves([issue.key for issue in children], order):
if refused := store.rank(rank.key, rank.before):
return Failure(f"cannot rank {rank.key} just before {rank.before}: {refused.reason}; the re-rank stopped there, so rank the loop's issues in {store.epic_key}'s backlog by hand to come in this sequence: {', '.join(order)}")
return None
def rank_moves(backlog: list[str], order: list[str]) -> list[Rank]:
"""The ranks, in the sequence to make them, that turn backlog, a list of keys, into one where the keys order lists come in that sequence, each at a position one of them held, while every other key keeps its position. order names each key once, and each is in backlog.
Working from the backlog's end, a listed key that isn't already just before the key due after it ranks just before that key. The key due last, where it isn't already last, ranks just before the key that is, which moves away in its own turn, since it is due earlier."""
listed = set(order)
sequence = iter(order)
target = [next(sequence) if key in listed else key for key in backlog]
now = list(backlog)
ranks = []
for position in reversed(range(len(target))):
key = target[position]
due_after = target[position + 1] if position + 1 < len(target) else None
at = now.index(key)
if key not in listed or (now[at + 1] if at + 1 < len(now) else None) == due_after:
continue
rank = Rank(key, now[-1] if due_after is None else due_after)
ranks.append(rank)
now.remove(key)
now.insert(now.index(rank.before), key)
return ranks
def chain_scoring(formula: ast.Expression, epic_goals: str, repo: str | None, own: RepoRanking | None) -> ranking.Scoring | Failure:
"""How an item in repo scores: the chain's goals, and what each achieve goal weighs. The bar is 0, since a bar decides what is filed, and the order holds every item that was.
+34 -17
View File
@@ -1,12 +1,13 @@
import dataclasses
import decimal
import itertools
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from do_epic.ranking import chain, EpicItem, ranked, RepoRanking, reranked
from do_epic.ranking import chain, EpicItem, rank_moves, ranked, RepoRanking, reranked
from failure import Failure
from plan.todo_table import Item, parse_moves
from do_epic.backend.fake import MemoryEpic
@@ -157,23 +158,23 @@ class RankedTest(unittest.TestCase):
class Recording(MemoryEpic):
"""A MemoryEpic that keeps each rank asked of it, and refuses the one to rank the issue named in refused."""
"""A MemoryEpic that keeps each rank asked of it, and refuses every rank of the issue named in refused."""
refused: str | None = None
asked: list[tuple[str, str, bool]]
asked: list[tuple[str, str]]
def rank(self, key: str, other: str, after: bool = False) -> None | Failure:
self.asked.append((key, other, after))
return Failure("Jira answered 403") if key == self.refused else super().rank(key, other, after)
def rank(self, key: str, before: str) -> None | Failure:
self.asked.append((key, before))
return Failure("Jira answered 403") if key == self.refused else super().rank(key, before)
def epic(*people: int, issues: int = 4) -> Recording:
"""An epic of child issues ABC-100 onwards, the loop's apart from those whose places are in people."""
"""An epic whose child issues are ABC-100 onwards. The loop filed each of them, except those whose positions, counted from 0, are in people."""
store = Recording("ABC-12")
store.asked = []
for place in range(issues):
store.file("ABC-12", f"Item {place}.", "", ())
if place in people:
store.change(f"ABC-{100 + place}", by_loop=False)
for position in range(issues):
store.file("ABC-12", f"Item {position}.", "", ())
if position in people:
store.change(f"ABC-{100 + position}", by_loop=False)
return store
@@ -201,8 +202,8 @@ class RerankedTest(unittest.TestCase):
def test_ranks_only_the_issues_out_of_place(self) -> None:
store = epic(issues=5)
self.assertEqual(reranked(store, ["ABC-100", "ABC-101", "ABC-102", "ABC-104", "ABC-103"]), None)
self.assertEqual(store.asked, [("ABC-103", "ABC-104", True)])
self.assertEqual(reranked(store, ["ABC-100", "ABC-102", "ABC-101", "ABC-103", "ABC-104"]), None)
self.assertEqual(store.asked, [("ABC-101", "ABC-103")])
def test_an_issue_the_order_doesnt_name_keeps_its_slot(self) -> None:
store = epic()
@@ -232,13 +233,13 @@ class RerankedTest(unittest.TestCase):
def test_a_rank_the_tracker_refuses_stops_the_walk_and_names_the_order_to_finish_by_hand(self) -> None:
store = epic()
store.refused = "ABC-101"
self.assertEqual(reranked(store, ["ABC-103", "ABC-102", "ABC-101", "ABC-100"]), Failure("cannot rank ABC-101 just before ABC-100: Jira answered 403; the re-rank stopped there, so rank the loop's issues in ABC-12's backlog by hand as ABC-103, ABC-102, ABC-101, ABC-100"))
self.assertEqual(store.asked, [("ABC-100", "ABC-103", True), ("ABC-101", "ABC-100", False)])
self.assertEqual(reranked(store, ["ABC-103", "ABC-102", "ABC-101", "ABC-100"]), Failure("cannot rank ABC-101 just before ABC-100: Jira answered 403; the re-rank stopped there, so rank the loop's issues in ABC-12's backlog by hand to come in this sequence: ABC-103, ABC-102, ABC-101, ABC-100"))
self.assertEqual(store.asked, [("ABC-100", "ABC-103"), ("ABC-101", "ABC-100")])
def test_a_refused_rank_after_the_last_issue_names_it(self) -> None:
def test_a_refused_rank_of_the_issue_due_last_names_it(self) -> None:
store = epic()
store.refused = "ABC-100"
self.assertEqual(reranked(store, ["ABC-103", "ABC-102", "ABC-101", "ABC-100"]), Failure("cannot rank ABC-100 just after ABC-103: Jira answered 403; the re-rank stopped there, so rank the loop's issues in ABC-12's backlog by hand as ABC-103, ABC-102, ABC-101, ABC-100"))
self.assertEqual(reranked(store, ["ABC-103", "ABC-102", "ABC-101", "ABC-100"]), Failure("cannot rank ABC-100 just before ABC-103: Jira answered 403; the re-rank stopped there, so rank the loop's issues in ABC-12's backlog by hand to come in this sequence: ABC-103, ABC-102, ABC-101, ABC-100"))
def test_a_read_that_fails_ranks_nothing(self) -> None:
store = epic()
@@ -247,5 +248,21 @@ class RerankedTest(unittest.TestCase):
self.assertEqual(store.asked, [])
class RankMovesTest(unittest.TestCase):
def test_every_backlog_of_up_to_six_issues_reaches_its_target_moving_only_the_listed_keys(self) -> None:
for size in range(7):
backlog = [f"K-{n}" for n in range(size)]
for listed in itertools.chain.from_iterable(itertools.combinations(backlog, count) for count in range(size + 1)):
for order in itertools.permutations(listed):
sequence = iter(order)
target = [next(sequence) if key in listed else key for key in backlog]
now = list(backlog)
for rank in rank_moves(backlog, list(order)):
self.assertIn(rank.key, listed)
now.remove(rank.key)
now.insert(now.index(rank.before), rank.key)
self.assertEqual(now, target, (backlog, order))
if __name__ == "__main__":
unittest.main()
+4 -4
View File
@@ -156,7 +156,7 @@ class JiraPlan:
notes.append(f"filed {key}, but cannot link it as blocked by {blocker}: {linked.reason}; link it in Jira")
if isinstance(before, Failure):
notes.append(f"filed {key}, but cannot rank it: {before.reason}; rank it in Jira")
elif before is not None and (refused := rank_refusal(self.epic.site, key, before, False, authorization, self.http)):
elif before is not None and (refused := rank_refusal(self.epic.site, key, before, authorization, self.http)):
notes.append(f"filed {key}, but cannot rank it before {before}: {refused}; rank it in Jira")
return tuple(notes)
@@ -442,9 +442,9 @@ def children(epic: Epic, authorization: str, http: Http) -> list[object] | Failu
return searched(epic.site, f"{epic.key}'s children", f'parent = "{epic.key}" AND statusCategory != Done ORDER BY Rank ASC', {"fields": ["summary", "description", "fixVersions", "issuelinks"], "properties": [PROPERTY]}, authorization, http)
def rank_refusal(site: str, key: str, other: str, after: bool, authorization: str, http: Http) -> str | None:
"""Ranks the issue named key just before the issue named other, or just after it where after is True; why Jira refused, or None where it didn't. Jira answers a rank it made with no body, and one it refused with an error, or with an entry per issue that holds the errors."""
answer = http("PUT", f"{site}/rest/agile/1.0/issue/rank", authorization, {"issues": [key], "rankAfterIssue" if after else "rankBeforeIssue": other})
def rank_refusal(site: str, key: str, before: str, authorization: str, http: Http) -> str | None:
"""Ranks the issue named key just before the issue named before; why Jira refused, or None where it didn't. Jira answers a rank it made with no body, and one it refused with an error, or with an entry per issue that holds the errors."""
answer = http("PUT", f"{site}/rest/agile/1.0/issue/rank", authorization, {"issues": [key], "rankBeforeIssue": before})
if isinstance(answer, Failure):
return answer.reason
entries = answer.get("entries") if isinstance(answer, dict) else None