From daa3a54b90de57814208ddb16799890d6b11c3e4 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 11 Nov 2026 17:42:10 +0100 Subject: [PATCH] Have loop-claude read the items a chunk takes from the worker's Chunk items: line, and keep the judge's escalation and release-ready fields, which code misread in replays --- .gitignore | 3 + CHANGELOG.md | 1 + README.md | 2 +- bin/judge_replay/test_judge_replay.py | 2 +- bin/loop_claude/cycle/chunk_items.py | 35 +++++++ bin/loop_claude/cycle/judge.py | 100 +++++++----------- bin/loop_claude/cycle/prompts.py | 8 +- bin/loop_claude/cycle/run.py | 7 +- bin/loop_claude/loop/chunk_start.py | 7 +- bin/loop_claude/loop/main.py | 7 +- bin/loop_claude/test_black_box.py | 22 ++-- bin/loop_claude/test_loop_claude.py | 139 +++++++++++++------------ bin/loop_claude/verdicts.py | 4 +- bin/model_routing/route.py | 4 +- bin/repo_checks/test_trials_imports.py | 9 +- docs/decisions.md | 21 +++- judge-field-trials/README.md | 28 +++++ judge-field-trials/cases.py | 41 ++++++++ judge-field-trials/escalations.py | 20 ++++ judge-field-trials/items.py | 19 ++++ judge-field-trials/release_ready.py | 53 ++++++++++ todo.md | 18 ++-- 22 files changed, 374 insertions(+), 176 deletions(-) create mode 100644 bin/loop_claude/cycle/chunk_items.py create mode 100644 judge-field-trials/README.md create mode 100644 judge-field-trials/cases.py create mode 100644 judge-field-trials/escalations.py create mode 100644 judge-field-trials/items.py create mode 100644 judge-field-trials/release_ready.py diff --git a/.gitignore b/.gitignore index f684d7d8..e39da701 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,9 @@ !bin/transcript_archive/*.py !docs/ !docs/*.md +!judge-field-trials/ +!judge-field-trials/*.py +!judge-field-trials/README.md !presentation/ !presentation/README.md !presentation/build-pdf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd1bef4..292c4938 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- loop-claude now asks each worker to name the items its chunk takes on a line of its own, as in `Chunk items: 12 (L 3, E + I 5), 14 (L 2, E + I 4)`, and scores them from `todo.md` on the default branch: that line, not the judge, now sets the rung an extra item lifts the worker and its reviewers to, and the items loop-inbox lists. The judge still reports escalations and whether a release is ready to cut. - `~/.claude` holds no `AGENTS.md` any more, so each session in your projects starts about 3,600 tokens smaller, and a session in `~/code/claude-config` loads its `CLAUDE.md` once instead of twice. - A project's `AGENTS.md` may group decision titles into project principles, and each new decision is checked against them: a principle covers it, it breaks one, or it suggests a new one for you to approve. - The prose reviewer now holds prose to the negation rule: write `doesn't cost anything`, never `costs nothing`, after any verb or its preposition. diff --git a/README.md b/README.md index dbf0ffaf..83b9e17d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Claude Code configuration -My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. +My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in loop-claude's judge's place, with their results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. ## Goals diff --git a/bin/judge_replay/test_judge_replay.py b/bin/judge_replay/test_judge_replay.py index 9023ccae..f9094b08 100644 --- a/bin/judge_replay/test_judge_replay.py +++ b/bin/judge_replay/test_judge_replay.py @@ -123,7 +123,7 @@ class Main(unittest.TestCase): @staticmethod def verdict(interrupt, evidence): - output = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "evidence": evidence, "finished": not interrupt, "interrupt": interrupt, "question": "Which?" if interrupt else "", "answered_before": "", "left_open": "", "release_ready": False} + output = {"answer": "", "escalation": "", "evidence": evidence, "finished": not interrupt, "interrupt": interrupt, "question": "Which?" if interrupt else "", "answered_before": "", "left_open": "", "release_ready": False} return {"type": "result", "structured_output": output} def test_replays_the_models_projects_on_the_reference_then_from_the_projects_rung_down_and_prints_the_lowest_that_returns_all(self): diff --git a/bin/loop_claude/cycle/chunk_items.py b/bin/loop_claude/cycle/chunk_items.py new file mode 100644 index 00000000..8604602c --- /dev/null +++ b/bin/loop_claude/cycle/chunk_items.py @@ -0,0 +1,35 @@ +"""The todo.md items a chunk takes: those the worker names on its latest `Chunk items:` line, scored as todo.md scores them (docs/decisions.md → `routing: loop-claude reads the items a chunk takes`).""" +import re + +from claude_code.entries import AssistantTurn, Entry, Text +from failure import Failure +from release_plan.todo_table import items +from claude_code.transcript import plain +from loop_claude.verdicts import TakenIn + +# The label opening the line; prompts.NAME_THE_ITEMS asks the worker for it. +CHUNK_ITEMS = "Chunk items:" +# A line the worker opens with the label, after any list or quote marker; what follows names the items. +LINE = re.compile(rf"[\s>`-]*{re.escape(CHUNK_ITEMS)}(.*)") +# What the worker says of an item in parentheses, such as its L and E + I. +SAID_OF_AN_ITEM = re.compile(r"\([^)]*\)") +# One entry of the line, which commas or "and" separate: an item's ID, which may open with "item" or "#". +ITEM_ID = re.compile(r"\s*(?:item\s+)?#?(\d+)", re.IGNORECASE) + + +def taken_in(entries: list[Entry], todo: str | Failure) -> TakenIn: + """The items named, with the highest L and E + I todo.md, as the cycle read it, scores among them. An item todo.md doesn't hold, or a todo.md that can't be read, counts at no level, so the route holds.""" + ids = named(entries) + found = items(todo) if isinstance(todo, str) else todo + scored = [item for item in found if item.id in ids] if isinstance(found, list) else [] + return TakenIn(ids, max((item.likelihood for item in scored), default=0), max((item.escape_and_impact for item in scored), default=0)) + + +def named(entries: list[Entry]) -> tuple[int, ...]: + """The IDs on the worker's latest Chunk items: line, each once, in its order; empty where it wrote none. Only the worker's own text counts: a loop-claude prompt quotes the line's example, and a tool call may write the label into a file.""" + lines = [line for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, Text) for line in part.text.splitlines()] + for line in reversed(lines): + if found := LINE.match(plain(line)): + each = re.split(r",|\band\b", SAID_OF_AN_ITEM.sub("", found[1])) + return tuple(dict.fromkeys(int(item[1]) for named_item in each if (item := ITEM_ID.match(named_item)))) + return () diff --git a/bin/loop_claude/cycle/judge.py b/bin/loop_claude/cycle/judge.py index 137a8d03..aec3687c 100644 --- a/bin/loop_claude/cycle/judge.py +++ b/bin/loop_claude/cycle/judge.py @@ -9,9 +9,7 @@ from loop_effects.effects import Effects from claude_code.entries import Entry, structured_output from loop_claude import escalation_mark from failure import Failure -from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, TakenIn, Unfinished, Verdict -from release_plan.todo_table import LEVELS -from model_routing.route import on_its_scale +from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict from claude_code.transcript import plain, verbatim from loop_claude.cycle.retry import retried from model_routing.ladder import Rung @@ -69,12 +67,6 @@ JUDGE_RULES = "\n\n".join((INTERRUPT_RULES, QUOTE_RULES, FINISHED_RULES)) @dataclasses.dataclass(frozen=True) class Reply: answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the records that answers the worker's question, quoted verbatim, without elisions. Empty otherwise."}) - # Routes the next cycle's worker. Counting the item the chunk was routed on is harmless: loop-claude never steps the worker down. - highest_likelihood_taken_in: int = dataclasses.field(metadata={"description": "The highest L the worker states for a todo.md item it took into this chunk; 0 when it states none."}) - # Routes the next cycle's judge and reviewers, as highest_likelihood_taken_in routes the worker. - highest_escape_and_impact_taken_in: int = dataclasses.field(metadata={"description": "The highest E + I the worker states for a todo.md item it took into this chunk; 0 when it states none."}) - # Lists the chunk's items in loop-inbox: running_loops.py. - items_taken_in: list[int] = dataclasses.field(metadata={"description": "The IDs of the todo.md items the worker states this chunk takes, the item the chunk was routed on included, never an item it only filed in todo.md for later; empty when it states none."}) # Routes the worker one rung up, as the judge interrupts or quotes an answer, or on the next cycle: loop_claude/escalation_mark.py. escalation: str = dataclasses.field(metadata={"description": "Empty unless the chunk is unfinished, interrupt is true or answer quotes a passage, and the worker's run ends asking loop-claude for a higher rung, citing an escalation trigger. Then: a verbatim quote, without elisions, of the transcript passage where the trigger fired, such as a review round's commit title or the worker's report of the finding or failing test. " + escalation_mark.QUOTE_AFTER_STEP + " Never a later mention of the trigger, and never the request itself."}) evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the worker's question; else what shows the chunk finished or the work remaining."}) @@ -88,8 +80,7 @@ class Reply: REPLY_TYPES = typing.get_type_hints(Reply) -JSON_TYPES: dict[object, dict[str, typing.Any]] = {bool: {"type": "boolean"}, int: {"type": "integer"}, str: {"type": "string"}, list[int]: {"type": "array", "items": {"type": "integer"}}} -HIGHEST_L = LEVELS["L"] +JSON_TYPES: dict[object, dict[str, typing.Any]] = {bool: {"type": "boolean"}, int: {"type": "integer"}, str: {"type": "string"}} JUDGE_SCHEMA = { "properties": {field.name: {**field.metadata, **JSON_TYPES[REPLY_TYPES[field.name]]} for field in dataclasses.fields(Reply)}, "required": list(REPLY_TYPES), @@ -128,43 +119,25 @@ def verdict_from(output: object) -> Verdict | Failure: return Failure("the judge returned no reply") for name, kind in REPLY_TYPES.items(): if not fits(output.get(name), kind): - return Failure(f"the judge's reply has no {name} of type {type_name(JSON_TYPES[kind])}") - taken_in = taken_in_from(output) + return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]['type']}") if output["interrupt"]: if not output["question"].strip(): return Failure("the judge interrupted without a question") - return Interrupt(output["evidence"], output["question"], escalation=output["escalation"], taken_in=taken_in, release_ready=output["release_ready"], answered_before=output["answered_before"], left_open=output["left_open"]) + return Interrupt(output["evidence"], output["question"], escalation=output["escalation"], release_ready=output["release_ready"], answered_before=output["answered_before"], left_open=output["left_open"]) if output["answer"].strip(): if not output["question"].strip(): return Failure("the judge quoted an answer without a question") - return QuotedAnswer(output["evidence"], output["question"], output["answer"], escalation=output["escalation"], taken_in=taken_in) + return QuotedAnswer(output["evidence"], output["question"], output["answer"], escalation=output["escalation"]) if output["finished"]: return Finished(output["evidence"]) - return Unfinished(output["evidence"], escalation=output["escalation"], taken_in=taken_in) - - -def type_name(schema: dict[str, typing.Any]) -> str: - return f"array of {schema['items']['type']}s" if schema["type"] == "array" else schema["type"] + return Unfinished(output["evidence"], escalation=output["escalation"]) def fits(value: object, kind: object) -> bool: """Whether value, read from the judge's JSON, is of kind. type(), not isinstance(): JSON true is no integer.""" - if typing.get_origin(kind) is list: - return type(value) is list and all(fits(each, typing.get_args(kind)[0]) for each in value) return type(value) is kind -def taken_in_from(output: dict) -> TakenIn: - """The reply's readings of the items taken in. An ID below 1 is left out, and an L or E + I off its scale counts as 0. Neither fails the reply: three failures would stop an unattended loop, and holding the rung is safe.""" - likelihood = output["highest_likelihood_taken_in"] - escape_and_impact = output["highest_escape_and_impact_taken_in"] - return TakenIn( - tuple(item for item in output["items_taken_in"] if item > 0), - likelihood if 0 <= likelihood <= HIGHEST_L else 0, - escape_and_impact if on_its_scale(escape_and_impact) else 0, - ) - - def judge(root: str, brief: Brief, rung: Rung, effects: Effects, keep: typing.Callable[[Rejected], Stop | None]) -> Verdict | Stop: return retried("judge", lambda: judge_once(root, brief, rung, effects), effects, lambda failure: keep(failure) if isinstance(failure, Rejected) else None) @@ -211,15 +184,15 @@ def cost_of(reply: dict) -> float: def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure: """The verdict with each of its quotes checked against the text it must appear in, as written: - | Quote | Must be in | Where it isn't | - |---|---|---| - | the evidence | grounds.shown, the evidence's fragments in order | a Failure: run_judge rejects the reply, and judge retries it | - | an interrupt's answered_before | grounds.shown | the interrupt stands, as if no earlier answer was found | - | its left_open | answered_before and the question | the interrupt becomes an Unfinished that quotes answered_before back to the worker | - | a quoted answer | grounds.quotable | the question goes to the human as an interrupt, which keeps the escalation the answer carried | - | the escalation | grounds.shown and grounds.after_mark | the escalation is dropped, so the worker's rung holds, as escalation_mark.checked says | + | Quote | Must be in | Where it isn't | Checked by | + |---|---|---|---| + | the evidence | grounds.shown, the evidence's fragments in order | a Failure: run_judge rejects the reply, and judge retries it | grounded | + | an interrupt's answered_before | grounds.shown | the interrupt stands, as if no earlier answer was found | re_ask_checked | + | its left_open | answered_before and the question | the interrupt becomes an Unfinished that quotes answered_before back to the worker | re_ask_checked | + | a quoted answer | grounds.quotable | the question goes to the human as an interrupt, which keeps the escalation the answer carried | answer_checked | + | the escalation | grounds.shown and grounds.after_mark | the escalation is dropped, so the worker's rung holds | escalation_mark.checked | - Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead. + Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead. The checks below follow the table's order; escalation_mark.checked sits beside the mark it moves. """ if not grounded(verdict.evidence, grounds.shown): return misquoted_evidence(verdict) @@ -228,13 +201,18 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure: return escalation_mark.checked(verdict, after=grounds.after_mark, shown=grounds.shown) -def re_ask_checked(verdict: Verdict, shown: str) -> Verdict: - """The verdict, or an Unfinished quoting the human's earlier answer back to the worker, where it is an Interrupt that re-asks what that answer settles: its left_open isn't in both the answer and the question.""" - if not isinstance(verdict, Interrupt) or not verbatim(verdict.answered_before, shown): - return verdict - if verbatim(verdict.left_open, verdict.answered_before) and verbatim(verdict.left_open, verdict.question): - return verdict - return Unfinished(verdict.answered_before, escalation=verdict.escalation, taken_in=verdict.taken_in) +def grounded(evidence: str, rendered_transcript: str) -> bool: + """Whether the transcript holds the evidence word for word, once emphasis marks are stripped and whitespace is reflowed in both. The judge joins passages with an ellipsis though told to quote one, so each fragment between ellipses must come after the one before it.""" + transcript = plain(rendered_transcript) + fragments = [fragment.strip() for fragment in re.split(r"\.\.\.|…", plain(evidence)) if fragment.strip()] + after = 0 + for fragment in fragments: + at = transcript.find(fragment, after) + if at < 0: + return False + after = at + len(fragment) + return bool(fragments) + def misquoted_evidence(verdict: Verdict) -> Failure: @@ -242,10 +220,19 @@ def misquoted_evidence(verdict: Verdict) -> Failure: return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the transcript{asked}") +def re_ask_checked(verdict: Verdict, shown: str) -> Verdict: + """The verdict, or an Unfinished quoting the human's earlier answer back to the worker, where it is an Interrupt that re-asks what that answer settles: its left_open isn't in both the answer and the question.""" + if not isinstance(verdict, Interrupt) or not verbatim(verdict.answered_before, shown): + return verdict + if verbatim(verdict.left_open, verdict.answered_before) and verbatim(verdict.left_open, verdict.question): + return verdict + return Unfinished(verdict.answered_before, escalation=verdict.escalation) + + def answer_checked(verdict: Verdict, quotable: str) -> Verdict: """The verdict; where it is a quoted answer that README.md and the records don't hold, an Interrupt on the same question instead.""" if isinstance(verdict, QuotedAnswer) and not verbatim(verdict.answer, quotable): - return Interrupt(verdict.evidence, verdict.question, escalation=verdict.escalation, taken_in=verdict.taken_in) + return Interrupt(verdict.evidence, verdict.question, escalation=verdict.escalation) return verdict @@ -266,17 +253,4 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]: after_mark=escalation_mark.after_mark(brief.entries, brief.escalated_at), quotable=f"{brief.premises}\n\n{brief.records}", ) - return f"{head}{shown}\n\n{JUDGE_RULES}", grounds - - -def grounded(evidence: str, rendered_transcript: str) -> bool: - """Whether the transcript holds the evidence word for word, once emphasis marks are stripped and whitespace is reflowed in both. The judge joins passages with an ellipsis though told to quote one, so each fragment between ellipses must come after the one before it.""" - transcript = plain(rendered_transcript) - fragments = [fragment.strip() for fragment in re.split(r"\.\.\.|…", plain(evidence)) if fragment.strip()] - after = 0 - for fragment in fragments: - at = transcript.find(fragment, after) - if at < 0: - return False - after = at + len(fragment) - return bool(fragments) + return f"{head}{shown}\n\n{JUDGE_RULES}", grounds \ No newline at end of file diff --git a/bin/loop_claude/cycle/prompts.py b/bin/loop_claude/cycle/prompts.py index 8aba873f..cdb3da4b 100644 --- a/bin/loop_claude/cycle/prompts.py +++ b/bin/loop_claude/cycle/prompts.py @@ -1,5 +1,6 @@ """What loop-claude says to the worker each cycle. model_routing.review_choice chooses where the reviews run. review_launches and reruns put that choice into sentences, and escalation says where escalated subagents run; rung_sentences returns the sentences of all three. In a chunk's first prompt, routed_by places them after its routing sentences where an item routed the chunk, and fresh places them alone otherwise. rungs_held repeats them at the end of a prompt after a step-up or an unfinished verdict.""" from model_routing.review_choice import chosen, ReviewChoice +from loop_claude.cycle.chunk_items import CHUNK_ITEMS from model_routing.ladder import climbed, LADDER, position, Rung from model_routing.route import ESCALATION_MARKER, Route from loop_claude.routing import escalation_rungs @@ -21,6 +22,8 @@ THE_SECURITY_PASS = "the security pass's subagent" # Workers only, not CLAUDE.md, which reviewers load too: a reviewer under a like rule missed findings. READING_RULE = "Read in small pieces: never read more than 80 lines of a file at once, nor take a tool output over 3,000 characters into your context." +# chunk_items reads the latest such line; the bench picked the label (docs/decisions.md → `routing: loop-claude reads the items a chunk takes`). +NAME_THE_ITEMS = f"Name each todo.md item this chunk takes on a line of its own, as in `{CHUNK_ITEMS} 297 (L 3, E + I 5), 298 (L 3, E + I 7)`, and write that line again whenever the set changes." AFTER_CODE_HEALTH = 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless README.md voids it' @@ -30,13 +33,12 @@ def fresh(task: str, premises: str, route: Route, project: Rung | None, facts: s """A chunk's first prompt; facts is preflight_facts' or preflight_failed's text.""" routed = f"{routed_by(route, project)} {recount(route)}" if route.item else " ".join(rung_sentences(route, project)) opening = f"{routed}\n\n" if routed else "" - return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one. {READING_RULE}\n\n{opening}{facts}\n\nThe chunk's todo.md items, or else the task above, set its scope. The goals below, {AFTER_CODE_HEALTH}, choose between ways to deliver that scope, and never widen it.\n\nFrom README.md:\n{premises}" + return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one. {READING_RULE} {NAME_THE_ITEMS}\n\n{opening}{facts}\n\nThe chunk's todo.md items, or else the task above, set its scope. The goals below, {AFTER_CODE_HEALTH}, choose between ways to deliver that scope, and never widen it.\n\nFrom README.md:\n{premises}" def routed_by(route: Route, project: Rung | None) -> str: return " ".join(( f"loop-claude runs you on {route.rung}, routed by item {route.item}'s L of {route.likelihood}.", - f"Name each todo.md item this chunk takes, with its L and E + I, whether or not it is item {route.item}.", *rung_sentences(route, project), )) @@ -97,7 +99,7 @@ def shared_launches(choice: ReviewChoice, route: Route, project: Rung) -> list[s def taken_in_launches(choice: ReviewChoice) -> list[str]: - """The sentence naming the reviewers' rung each higher E + I picks, so the worker reviews an item it takes in on that rung in the same run, since the judge reports a taken-in item's E + I only as the cycle ends, and loop-claude routes on it from the next cycle. None where no higher E + I picks a higher rung. + """The sentence naming the reviewers' rung each higher E + I picks, so the worker reviews an item it takes in on that rung in the same run, since loop-claude reads a taken-in item's E + I only as the cycle ends, and loop-claude routes on it from the next cycle. None where no higher E + I picks a higher rung. For a worker on Opus 5.5 at medium, on an Opus 5.5 project, routed by an item of E + I 6, it reads: `Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.` """ diff --git a/bin/loop_claude/cycle/run.py b/bin/loop_claude/cycle/run.py index e5703776..1340bfa9 100644 --- a/bin/loop_claude/cycle/run.py +++ b/bin/loop_claude/cycle/run.py @@ -6,6 +6,7 @@ from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, Hu from transcript_archive.write import archive from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args from loop_claude.cycle.judge import Brief, judge, Rejected +from loop_claude.cycle import chunk_items from claude_code.session import delete_session, Session, session_open, transcript_file from loop_effects.effects import Effects, say from failure import Failure @@ -42,9 +43,11 @@ class Cycle: root: str route: Route session: Session - # Lists in loop-inbox the items the chunk takes, given the IDs an unfinished ruling found taken in. + # Lists in loop-inbox the items the chunk takes, given the IDs a ruling found the worker names. show_taken_in: typing.Callable[[tuple[int, ...]], None] task: str + # todo.md on the default branch as the cycle starts, which scores the items the worker names. + todo: str | Failure # Empty when the previous cycle's verdict was not Unfinished. unfinished_evidence: str @@ -151,6 +154,8 @@ def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Sto verdict = judge(cycle.root, Brief(cycle.task, cycle.premises, cycle.records, entries, cycle.route.escalated_at), judge_of(cycle), cycle.effects, lambda rejected: keep_rejection(cycle, transcript_path, rejected)) if isinstance(verdict, Stop): return verdict + if not isinstance(verdict, Finished): + verdict = dataclasses.replace(verdict, taken_in=chunk_items.taken_in(entries, cycle.todo)) if not isinstance(verdict, Interrupt | QuotedAnswer): return Ruled(verdict) stepped = ruling_step_up(cycle, verdict) diff --git a/bin/loop_claude/loop/chunk_start.py b/bin/loop_claude/loop/chunk_start.py index 3ac6ee14..64d8143c 100644 --- a/bin/loop_claude/loop/chunk_start.py +++ b/bin/loop_claude/loop/chunk_start.py @@ -20,14 +20,13 @@ from loop_state.running_loops import taken_items ORIGIN_TIMEOUT = 60 -def shown_items(loop: Loop, routed: int) -> typing.Callable[[tuple[int, ...]], None]: - """A function that lists in loop-inbox the items a chunk takes, given the IDs the judge found taken in. routed is the item the chunk was routed on, 0 for none. shown_items reads todo.md once, as the cycle starts, and the function shows those taglines.""" - todo = fetched_todo(loop) +def shown_items(loop: Loop, todo: str | Failure, routed: int) -> typing.Callable[[tuple[int, ...]], None]: + """A function that lists in loop-inbox the items a chunk takes, given the IDs the worker names, with their taglines in todo, as fetched_todo read it as the cycle started. routed is the item the chunk was routed on, 0 for none.""" return lambda taken_in: loop.listed.show(taken_items(todo, routed, taken_in)) def fetched_todo(loop: Loop) -> str | Failure: - """todo.md on the default branch, as last fetched. fetched_plan reads it too, but also the release tag, CHANGELOG.md and each todo.md since the tag, which the listing doesn't need.""" + """todo.md on the default branch, as last fetched, which lists and scores the items a chunk takes. fetched_plan reads it too, but also the release tag, CHANGELOG.md and each todo.md since the tag, which neither needs.""" ref = default_ref(loop.root, ORIGIN_TIMEOUT, run=loop.effects.git) return ref if isinstance(ref, Failure) else file_at(ref, "todo.md", loop.root, loop.effects.git) diff --git a/bin/loop_claude/loop/main.py b/bin/loop_claude/loop/main.py index d2bdfb4c..7c002163 100644 --- a/bin/loop_claude/loop/main.py +++ b/bin/loop_claude/loop/main.py @@ -3,7 +3,7 @@ import pathlib import typing from transcript_archive.location import archive_root, project_archive -from loop_claude.loop.chunk_start import archived_route, chunk_start, routed, shown_items +from loop_claude.loop.chunk_start import archived_route, chunk_start, fetched_todo, routed, shown_items from loop_claude.loop.routing_check import check_routing from loop_claude.project.checkout import checkout_root from loop_claude.loop.close import closed_cycle @@ -80,9 +80,10 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop: project = project_rung(models.worker) route = routed(loop_run.route, project, models.worker, lambda: archived_route(archive_folder, loop), item) say(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", loop.effects, stdout=True) - show_taken_in = shown_items(loop, route.item) + todo = fetched_todo(loop) + show_taken_in = shown_items(loop, todo, route.item) show_taken_in(loop_run.items_taken_in) - return Cycle(effects=loop.effects, facts=facts, judge_model=models.judge, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence) + return Cycle(effects=loop.effects, facts=facts, judge_model=models.judge, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, todo=todo, unfinished_evidence=loop_run.unfinished_evidence) def loop_run_from(argv: list[str], cwd: pathlib.Path, effects: Effects) -> LoopRun | Stop: diff --git a/bin/loop_claude/test_black_box.py b/bin/loop_claude/test_black_box.py index 021f9e6e..12c44078 100644 --- a/bin/loop_claude/test_black_box.py +++ b/bin/loop_claude/test_black_box.py @@ -137,6 +137,9 @@ PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n" TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n" # Item 4's L of 2 steps one rung down. CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |") +# Item 9 follows CHEAP's item 4, and a worker routed by item 4 may take it in: at L 4 it picks a higher rung for the worker, and at E + I 6 for the judge and the reviewers. At E + I 3, or at L 2, it picks the rung item 4 picks. +TAKEN_IN_AT_L_4 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 4 | 1 | 2 | 5 | 2 | 1 by 9 | 20 |\n" +TAKEN_IN_AT_E_PLUS_I_6 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 2 | 3 | 3 | 6 | 2 | 1 by 9 | 20 |\n" # Waits for the interrupt's prompt before typing the answer, since loop-claude drops what was typed before it. @@ -147,8 +150,8 @@ def worker(**step): return {"role": "worker", **step} -def judge(interrupt=False, finished=False, evidence="working", question="", items_taken_in=(), highest_likelihood_taken_in=0, highest_escape_and_impact_taken_in=0, escalation="", release_ready=False, **step): - verdict = {"items_taken_in": list(items_taken_in), "answer": "", "highest_likelihood_taken_in": highest_likelihood_taken_in, "highest_escape_and_impact_taken_in": highest_escape_and_impact_taken_in, "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question, "answered_before": "", "left_open": "", "release_ready": release_ready} +def judge(interrupt=False, finished=False, evidence="working", question="", escalation="", release_ready=False, **step): + verdict = {"answer": "", "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question, "answered_before": "", "left_open": "", "release_ready": release_ready} return {"role": "judge", "verdict": verdict, **step} @@ -427,11 +430,11 @@ class Cycles(unittest.TestCase): def test_routes_the_worker_by_its_items_l_and_steps_it_up_within_the_chunk(self): checkout = Checkout(self) checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5" - checkout.commit_plan(CHEAP) + checkout.commit_plan(TAKEN_IN_AT_L_4) run = checkout.run([ worker(say="one"), judge(evidence="one"), worker(say="two"), judge(evidence="two"), - worker(say="three"), judge(evidence="three", highest_likelihood_taken_in=4), + worker(say="three\nChunk items: 4 (L 2, E + I 3), 9 (L 4, E + I 5)"), judge(evidence="three"), worker(say="four"), judge(finished=True, evidence="four"), worker(say="next chunk"), ]) @@ -445,7 +448,8 @@ class Cycles(unittest.TestCase): ("claude-sonnet-5-5", "high"), ]) self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)}) - self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2. Name each todo.md item this chunk takes, with its L and E + I, whether or not it is item 4.", argument(workers[0], "-p")) + self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2.", argument(workers[0], "-p")) + self.assertIn("Name each todo.md item this chunk takes on a line of its own, as in `Chunk items: 297 (L 3, E + I 5), 298 (L 3, E + I 7)`, and write that line again whenever the set changes.", argument(workers[0], "-p")) outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values()) self.assertIn(' Verdict: unfinished\n\nEvidence:\n> three\n\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n Stepped up\n\nWorker: {"model": "claude-opus-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes) self.assertIn(' Verdict: finished\n\nEvidence:\n> four\n\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes) @@ -474,11 +478,11 @@ class Cycles(unittest.TestCase): self.assertIn("item 6, L 4, E + I 5", prompt) self.assertTrue((checkout.home / "code" / "proj-item-6").is_dir()) - def test_lists_itself_in_loop_inbox_with_its_routed_item_then_those_the_judge_finds_taken_in_until_the_chunk_finishes(self): + def test_lists_itself_in_loop_inbox_with_its_routed_item_then_those_the_worker_names_on_its_chunk_items_line_until_the_chunk_finishes(self): checkout = Checkout(self) checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5" checkout.commit_plan(TODO) - run = checkout.run([worker(), judge(items_taken_in=[7, 4]), worker(), judge(finished=True), worker()]) + run = checkout.run([worker(say="working\nChunk items: 7, 4"), judge(), worker(), judge(finished=True), worker()]) self.assertEqual(run.code, 130) routed, taken_in = {"id": 4, "tagline": "Do it."}, {"id": 7, "tagline": ""} self.assertEqual([call["running"] for call in run.calls], [[[routed]], [[routed]], [[routed, taken_in]], [[routed, taken_in]], [[routed]], [[routed]]]) @@ -540,9 +544,9 @@ class Cycles(unittest.TestCase): def test_routes_the_judge_and_the_reviewers_on_the_highest_e_plus_i_the_worker_takes_in(self): checkout = Checkout(self) checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5" - checkout.commit_plan(CHEAP) + checkout.commit_plan(TAKEN_IN_AT_E_PLUS_I_6) run = checkout.run([ - worker(say="one"), judge(evidence="one", highest_escape_and_impact_taken_in=6), + worker(say="one\nChunk items: 4, 9"), judge(evidence="one"), worker(say="two"), judge(finished=True, evidence="two"), worker(say="next chunk"), ]) diff --git a/bin/loop_claude/test_loop_claude.py b/bin/loop_claude/test_loop_claude.py index 17dbbe1a..4321128c 100644 --- a/bin/loop_claude/test_loop_claude.py +++ b/bin/loop_claude/test_loop_claude.py @@ -20,6 +20,7 @@ from routing_figures import archive, state from loop_state.waiting.ask import Ask, Asking, OpenInInbox, OpenOnTerminal from routing_figures.calls import Call from loop_claude.project import checkout, model_choice, records +from loop_claude.cycle import chunk_items from loop_claude.loop import chunk_start, close from loop_state.waiting.answers import ClaudeEnded, ClaudeOpening, send, Typed from release_plan.release_standing import DefaultPlan, LastCut @@ -161,8 +162,8 @@ def ran(stdout="", code=0, stderr=""): return subprocess.CompletedProcess([], code, stdout, stderr) -def judged(verdict, evidence="working", answer="", escalation="", release_ready=False, items_taken_in=(), cost=None): - reply = {"items_taken_in": list(items_taken_in), "answer": answer, "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": escalation, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else "", "answered_before": "", "left_open": "", "release_ready": release_ready} +def judged(verdict, evidence="working", answer="", escalation="", release_ready=False, cost=None): + reply = {"answer": answer, "escalation": escalation, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else "", "answered_before": "", "left_open": "", "release_ready": release_ready} return ran(json.dumps({"structured_output": reply, **({} if cost is None else {"total_cost_usd": cost})})) @@ -268,7 +269,7 @@ class Cycles(unittest.TestCase): self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {}) def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."): - return run.Cycle(effects=fake.effects(), facts=facts, judge_model="judge-model", premises="## Goals", project=None, project_archive=self.archive, questions=self.paths.waiting, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, show_taken_in=lambda items: None, task="Do it.", unfinished_evidence=unfinished_evidence) + return run.Cycle(effects=fake.effects(), facts=facts, judge_model="judge-model", premises="## Goals", project=None, project_archive=self.archive, questions=self.paths.waiting, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, show_taken_in=lambda items: None, task="Do it.", todo="", unfinished_evidence=unfinished_evidence) def archived_outcomes(self): return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md")) @@ -284,16 +285,16 @@ class Cycles(unittest.TestCase): self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nJudge cost: 0.2500\nWorker: {OFF_LADDER}\n"]) self.assertIsNone(session.transcript_file(self.session)) - def test_shows_the_items_each_ruling_finds_the_chunk_takes_as_the_judge_gave_them(self): - worker = Worker(self.paths.claude_code_transcripts.parent) - fake = FakeEffects(streams=[worker], claude=[judged("unfinished", items_taken_in=(14, 3))]) + def test_shows_the_items_each_ruling_finds_the_worker_names_on_its_chunk_items_line(self): + worker = Worker(self.paths.claude_code_transcripts.parent, text="working\nChunk items: 14 (L 2, E + I 3), 3 (L 1, E + I 2)") + fake = FakeEffects(streams=[worker], claude=[judged("unfinished")]) shown = [] run.run_cycle(dataclasses.replace(self.cycle(fake), show_taken_in=shown.append)) self.assertEqual(shown, [(14, 3)]) def test_shows_no_items_for_a_finished_chunk(self): - worker = Worker(self.paths.claude_code_transcripts.parent) - fake = FakeEffects(streams=[worker], claude=[judged("finished", items_taken_in=(3,))]) + worker = Worker(self.paths.claude_code_transcripts.parent, text="working\nChunk items: 3") + fake = FakeEffects(streams=[worker], claude=[judged("finished")]) shown = [] run.run_cycle(dataclasses.replace(self.cycle(fake), show_taken_in=shown.append)) self.assertEqual(shown, []) @@ -1340,9 +1341,9 @@ class WorkerArgs(unittest.TestCase): def test_names_the_route_in_a_fresh_prompt_only_where_an_item_routed_it(self): routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS) - self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4. Name each todo.md item this chunk takes, with its L and E + I, whether or not it is item 3.", routed) + self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4.", routed) unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS) - self.assertIn(f"before creating a new one. {prompts.READING_RULE}\n\n{self.FACTS}\n\nThe chunk's", unrouted) + self.assertIn(f"before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\n{self.FACTS}\n\nThe chunk's", unrouted) self.assertNotIn("loop-claude runs you on", unrouted) def test_has_the_routed_worker_count_its_items_size_again_and_puts_the_facts_before_the_scope(self): @@ -1372,21 +1373,21 @@ class WorkerArgs(unittest.TestCase): def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self): opus = Rung("claude-opus-5-5", "medium") below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), opus, self.FACTS) - self.assertIn(f"whether or not it is item 3. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 ` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger", below) + self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 ` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger", below) self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self): opus = Rung("claude-opus-5-5", "medium") level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), opus, self.FACTS) - self.assertIn(f"whether or not it is item 3. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", level) + self.assertIn(f"item 3's L of 3. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", level) below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), opus, self.FACTS) - self.assertIn(f"whether or not it is item 3. Launch each reviewer with the Agent tool set to model opus, effort high, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below) + self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model opus, effort high, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below) self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 6), opus)) def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self): opus = Rung("claude-opus-5-5", "medium") taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8." - self.assertIn(f"whether or not it is item 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) + self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), opus)) self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 4), opus)) self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), opus)) @@ -1404,12 +1405,12 @@ class WorkerArgs(unittest.TestCase): self.assertIn(f"{prompts.ANSWERS_RECORDED}. loop-claude runs you on claude-opus-5-5 at high effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up) self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), opus)) self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), None)) - self.assertIn(f"whether or not it is item 3.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", above) + self.assertIn(f"item 3's L of 4.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", above) def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self): opus = Rung("claude-opus-5-5", "medium") fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS) - self.assertIn("whether or not it is item 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", fresh) + self.assertIn("item 3's L of 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", fresh) self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 3, 4), opus)) self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model opus, effort max`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), opus)) self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-1", "max"), 3, 4, 8), opus)) @@ -1417,7 +1418,7 @@ class WorkerArgs(unittest.TestCase): self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort max`, one rung above yours;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), opus)) self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), None)) self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), opus)) - self.assertIn(f"{prompts.READING_RULE}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model opus, effort xhigh`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS)) + self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model opus, effort xhigh`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS)) def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self): opus = Rung("claude-opus-5-5", "medium") @@ -1566,65 +1567,67 @@ class Grounded(unittest.TestCase): self.assertTrue(judge.grounded("a ... b ... c", "x a b a c")) +# Item 297 scores the higher L, item 298 the higher E + I. +SCORED = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 297 | 0.8.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n| 298 | 0.8.0 | | **Do more.** | 3 | 3 | 4 | 8 | 2 | 1 by 9 | 20 |\n" + + +def said(*texts): + return entries.AssistantTurn(parts=tuple(entries.Text(text) for text in texts)) + + +class ChunkItems(unittest.TestCase): + def test_names_the_items_on_the_workers_latest_chunk_items_line(self): + earlier = said("Chunk items: 297 (L 3, E + I 5)") + latest = said("Starting.\n**Chunk items:** 297 (L 4, E + I 5), item 298 (L 3, E + I 7), #12, 298\nOn to the tests.") + self.assertEqual(chunk_items.named([earlier, latest]), (297, 298, 12)) + self.assertEqual(chunk_items.named([latest, earlier]), (297,)) + self.assertEqual(chunk_items.named([earlier, said("Chunk items: none")]), ()) + + def test_reads_only_a_line_the_worker_opens_with_the_label_in_its_own_text(self): + prompt = entries.LoopPrompt(text="Chunk items: 297 (L 3, E + I 5)") + written = entries.AssistantTurn(parts=(entries.ToolCall("t", "Write", {"content": "Chunk items: 9"}),)) + mentioned = said("Once I know them, I write `Chunk items: 5`.") + self.assertEqual(chunk_items.named([prompt, written, mentioned]), ()) + + def test_takes_the_highest_l_and_e_plus_i_todo_md_scores_among_the_named_items(self): + self.assertEqual(chunk_items.taken_in([said("Chunk items: 297 (L 1, E + I 2), 298, 400")], SCORED), TakenIn((297, 298, 400), 4, 7)) + self.assertEqual(chunk_items.taken_in([said("Chunk items: 400")], SCORED), TakenIn((400,))) + self.assertEqual(chunk_items.taken_in([], SCORED), TakenIn()) + + def test_names_the_items_at_no_level_where_todo_md_cannot_be_read(self): + self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], Failure("no todo.md")), TakenIn((297,))) + self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], "no items table"), TakenIn((297,))) + + class Verdict(unittest.TestCase): def test_needs_every_field_the_schema_requires(self): - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Finished("e")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "release_ready": False}), Unfinished("e")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True)) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answered_before": "", "left_open": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A.")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge quoted an answer without a question")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string")) - self.assertEqual(verdict_from({"items_taken_in": [], "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge interrupted without a question")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Finished("e")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "release_ready": False}), Unfinished("e")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True)) + self.assertEqual(verdict_from({"escalation": "", "answered_before": "", "left_open": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean")) + self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A.")) + self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?")) + self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge quoted an answer without a question")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string")) + self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge interrupted without a question")) self.assertEqual(verdict_from(None), Failure("the judge returned no reply")) def test_carries_the_earlier_answer_an_interrupt_names(self): - reply = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?"} + reply = {"answer": "", "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?"} self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", answered_before="Grant one more round.", left_open="one more")) self.assertEqual(verdict_from({**reply, "interrupt": False, "finished": True}), Finished("e")) - def test_takes_the_highest_l_taken_in_on_its_scale_and_counts_any_other_as_none(self): - reply = {"items_taken_in": [], "answer": "", "highest_escape_and_impact_taken_in": 0, "escalation": "", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False} - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": 4}), Unfinished("e", taken_in=TakenIn(highest_likelihood=4))) - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": True}), Failure("the judge's reply has no highest_likelihood_taken_in of type integer")) - for highest_likelihood_taken_in in (5, 13, -1): - with self.subTest(highest_likelihood_taken_in=highest_likelihood_taken_in): - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": highest_likelihood_taken_in}), Unfinished("e")) - - - def test_takes_the_highest_e_plus_i_taken_in_on_its_scale_and_counts_any_other_as_none(self): - reply = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "escalation": "", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False} - self.assertEqual(verdict_from({**reply, "highest_escape_and_impact_taken_in": 6}), Unfinished("e", taken_in=TakenIn(highest_escape_and_impact=6))) - self.assertEqual(verdict_from({**reply, "highest_escape_and_impact_taken_in": 2}), Unfinished("e", taken_in=TakenIn(highest_escape_and_impact=2))) - self.assertEqual(verdict_from({**reply, "highest_escape_and_impact_taken_in": False}), Failure("the judge's reply has no highest_escape_and_impact_taken_in of type integer")) - for escape_and_impact in (1, 9, -1): - with self.subTest(escape_and_impact=escape_and_impact): - self.assertEqual(verdict_from({**reply, "highest_escape_and_impact_taken_in": escape_and_impact}), Unfinished("e")) - - def test_takes_the_items_taken_in_as_the_judge_gives_them_leaving_out_ids_below_1(self): - reply = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False} - self.assertEqual(verdict_from({**reply, "items_taken_in": [12, 14, 12]}), Unfinished("e", taken_in=TakenIn((12, 14, 12)))) - self.assertEqual(verdict_from({**reply, "items_taken_in": [7, 0]}), Unfinished("e", taken_in=TakenIn((7,)))) - self.assertEqual(verdict_from({**reply, "items_taken_in": [3], "interrupt": True, "question": "Which one?"}), Interrupt("e", "Which one?", taken_in=TakenIn((3,)))) - self.assertEqual(verdict_from({**reply, "items_taken_in": [3], "answer": "A.", "question": "Which one?"}), QuotedAnswer("e", "Which one?", "A.", taken_in=TakenIn((3,)))) - self.assertEqual(verdict_from({**reply, "items_taken_in": [3], "finished": True}), Finished("e")) - self.assertEqual(verdict_from({**reply, "items_taken_in": "3"}), Failure("the judge's reply has no items_taken_in of type array of integers")) - self.assertEqual(verdict_from({**reply, "items_taken_in": ["3"]}), Failure("the judge's reply has no items_taken_in of type array of integers")) - def test_takes_an_escalation_quote_on_an_unfinished_verdict(self): - reply = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False} + reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False} self.assertEqual(verdict_from(reply), Unfinished("e", "round 2: 🔴")) self.assertEqual(verdict_from({**reply, "finished": True}), Finished("e")) self.assertEqual(verdict_from({**reply, "escalation": None}), Failure("the judge's reply has no escalation of type string")) def test_takes_an_escalation_quote_on_an_interrupt(self): - reply = {"items_taken_in": [], "answer": "", "highest_likelihood_taken_in": 0, "highest_escape_and_impact_taken_in": 0, "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False} + reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False} self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", "round 2: 🔴")) - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": 4}), Interrupt("e", "Which one?", "round 2: 🔴", TakenIn(highest_likelihood=4))) - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": 5}), Interrupt("e", "Which one?", "round 2: 🔴")) def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_an_interrupt_or_an_unfinished_verdict(self): shown = "The agent: round 2: 🔴. Which one?" @@ -1639,8 +1642,8 @@ class Verdict(unittest.TestCase): asking = "Grant a third round? Your answer left open: a 🔴 survives it" kept = Interrupt("[step] Round 2", asking, answered_before=answer, left_open="a 🔴 survives it") self.assertEqual(judge.quotes_checked(kept, grounds), kept) - re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", TakenIn((3,)), answered_before=answer, left_open="a 🔴 survives it") - self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴", TakenIn((3,)))) + re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", answered_before=answer, left_open="a 🔴 survives it") + self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴")) self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open=""), grounds), Unfinished(answer)) self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open="a 🔴 survived"), grounds), Unfinished(answer)) invented = dataclasses.replace(re_asked, answered_before="Grant two more rounds.") @@ -1648,10 +1651,9 @@ class Verdict(unittest.TestCase): unnamed = Interrupt("[step] Round 2", "Grant a third round?") self.assertEqual(judge.quotes_checked(unnamed, grounds), unnamed) - def test_takes_an_escalation_quote_and_the_highest_l_taken_in_on_a_quoted_answer(self): - reply = {"items_taken_in": [], "answer": "A.", "highest_likelihood_taken_in": 4, "highest_escape_and_impact_taken_in": 0, "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False} - self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴", TakenIn(highest_likelihood=4))) - self.assertEqual(verdict_from({**reply, "highest_likelihood_taken_in": 5}), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴")) + def test_takes_an_escalation_quote_on_a_quoted_answer(self): + reply = {"answer": "A.", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False} + self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴")) def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_a_quoted_answer(self): shown = "The agent: round 2: 🔴. Which one?" @@ -1659,8 +1661,7 @@ class Verdict(unittest.TestCase): self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: … 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A.")) def test_puts_a_quoted_answer_the_records_do_not_hold_to_the_human_with_the_escalation_it_carried(self): - self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there", taken_in=TakenIn((3,))), judge.Grounds("shown", "shown", "A.")), Interrupt("s", "q", taken_in=TakenIn((3,)))) - self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴", TakenIn(highest_likelihood=4)), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4))) + self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴"), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴")) def test_puts_a_quoted_answer_the_records_do_not_hold_word_for_word_to_the_human(self): self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there"), judge.Grounds("shown", "shown", "the records")), Interrupt("s", "q")) diff --git a/bin/loop_claude/verdicts.py b/bin/loop_claude/verdicts.py index 7cd8c9da..f0ba9ff2 100644 --- a/bin/loop_claude/verdicts.py +++ b/bin/loop_claude/verdicts.py @@ -1,10 +1,10 @@ -"""The verdicts cycle.judge.verdict_from builds from the judge's reply. They sit outside cycle/, since routing reads them and cycle imports routing.""" +"""The verdicts cycle.judge.verdict_from builds from the judge's reply, which the cycle then stamps with the items the chunk takes. They sit outside cycle/, since routing reads them and cycle imports routing.""" import dataclasses @dataclasses.dataclass(frozen=True) class TakenIn: - """What the judge read in the transcript about the todo.md items the worker took into the chunk, the item the chunk was routed on included.""" + """The todo.md items the worker names on its latest `Chunk items:` line, which cycle.chunk_items reads, the item the chunk was routed on among them where the worker names it.""" # Their IDs; loop-inbox lists them. items: tuple[int, ...] = () # The highest L among them, 0 for none: an escalation steps up from the rung that L picks, and an unfinished verdict also raises the worker to it. diff --git a/bin/model_routing/route.py b/bin/model_routing/route.py index 55472328..3d3960cc 100644 --- a/bin/model_routing/route.py +++ b/bin/model_routing/route.py @@ -43,7 +43,7 @@ class Route: likelihood: int = 0 escape_and_impact: int = 0 unfinished: int = 0 - # The highest E + I the judge saw the worker take in, kept only where it picks a higher review step than the item's; 0 otherwise. + # The highest E + I among the items the worker names on its `Chunk items:` line, kept only where it picks a higher review step than the item's; 0 otherwise. escape_and_impact_taken_in: int = 0 # The escalation mark, as loop_claude/escalation_mark.py says: how many transcript entries the session held when an escalation last stepped the worker up, 0 before any. escalated_at: int = 0 @@ -59,7 +59,7 @@ class Route: return levels_fit(fields.get("item", 0), fields.get("likelihood", 0), fields.get("escape_and_impact", 0), fields.get("escape_and_impact_taken_in", 0)) def taking_in(self, escape_and_impact: int) -> "Route": - """The route after the judge saw the worker take in an item of this E + I, 0 for none: it keeps the E + I only where REVIEW_STEPS steps it further than reviewed_on, so the archive records a step-up only where REVIEW_STEPS steps the review floor further.""" + """The route after the worker takes in an item of this E + I, 0 for none: it keeps the E + I only where REVIEW_STEPS steps it further than reviewed_on, so the archive records a step-up only where REVIEW_STEPS steps the review floor further.""" return dataclasses.replace(self, escape_and_impact_taken_in=escape_and_impact) if REVIEW_STEPS.get(escape_and_impact, 0) > REVIEW_STEPS.get(self.reviewed_on(), 0) else self def reviewed_on(self) -> int: diff --git a/bin/repo_checks/test_trials_imports.py b/bin/repo_checks/test_trials_imports.py index db3e1887..0364ffb5 100644 --- a/bin/repo_checks/test_trials_imports.py +++ b/bin/repo_checks/test_trials_imports.py @@ -9,7 +9,8 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from repo_checks import ROOT -TRIALS = ROOT / "reviewer-merge-trials" +# The trials folders whose scripts import from bin/. +TRIALS = [ROOT / "reviewer-merge-trials", ROOT / "judge-field-trials"] def bin_imports(source: str) -> list[tuple[str, str | None]]: @@ -39,11 +40,11 @@ def in_bin(module: str) -> bool: class TrialsImports(unittest.TestCase): - """docs/decisions.md says to rerun the trials' scripts when a new model joins the ladder or a reviewer's definition changes, so a refactor of bin/ must not break what they import.""" + """docs/decisions.md says when to rerun each trials folder's scripts, so a refactor of bin/ must not break what they import.""" def test_each_name_a_trials_script_imports_from_bin_exists(self): - scripts = sorted(TRIALS.glob("*.py")) - self.assertTrue(scripts) + scripts = sorted(script for folder in TRIALS for script in folder.glob("*.py")) + self.assertTrue(all(any(folder.glob("*.py")) for folder in TRIALS)) for script in scripts: for module, name in bin_imports(script.read_text(encoding="utf-8")): with self.subTest(script=script.name, module=module, name=name): diff --git a/docs/decisions.md b/docs/decisions.md index 4c7bd2b8..7aa710e9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -27,7 +27,8 @@ Each project principle states one rule that the decisions below apply; their ent - routing: a worker delegates a task only where it needs just the task's conclusion, and each subagent runs on the rung its task's L picks from the project's rung, never below the project's rung when it files or scores todo.md items - routing: a worker hands a task one rung up on each escalation trigger, up to two rungs above the rung its item picks unless the human agrees, and opens the subagent's prompt with the trigger so the routing figures count it - routing: a worker whose run ends asking for a higher rung, citing an escalation trigger the judge finds in the transcript, steps one rung up as the judge interrupts or quotes an answer, or next cycle, up to two rungs above the rung its item picks -- routing: the worker changes rung only as a chunk starts, after every second unfinished verdict, or when the judge reports a taken-in item that picks a higher rung or an escalation trigger the worker cites +- routing: loop-claude reads the items a chunk takes from the worker's latest `Chunk items:` line and scores them by `todo.md` as the default branch held it when the cycle started; the judge still reports the escalation and whether a release is ready to cut +- routing: the worker changes rung only as a chunk starts, after every second unfinished verdict, or when it names a chunk item that picks a higher rung or cites an escalation trigger the judge finds - tests: each test earns its keep as the technical principles' test bullet says, and the worker takes a test's expected values from its item, never from the code - the repo may name bender, bender-infra and ai-sandbox - todo.md: one table of items, scored by the formula it states, held to its bar and ranked by `bin/rank_todo.py`, which ranks each blocker above the items it blocks; each achieve goal weighs by its position, code health first and four times as heavy while the comprehension panel's overall is under 7.0 or unmeasured, then README.md's goals, and a maintain goal weighs 0.5 per point of the confidence an item's test, gate leg or probe adds to it; the table's order is the plan's only order @@ -171,7 +172,7 @@ Valid while cache reads cost a twentieth of input and Claude Code's compaction k - loop-inbox runs the claude command the loop wrote, `bypassPermissions` included, so that claude runs in the same sandbox as the loop. Since loop-inbox runs a command read from a file, the folder is private to the user. - Files, read by polling, need no daemon and only the standard library, and a loop with no loop-inbox running is answered on its terminal as before. - loop-inbox lists a question only while its loop holds the session's lock file with flock, and removes a question whose lock is free. The loop holds the lock for its whole run. The kernel frees the lock however the loop ends, killed included. A process id could not tell a running loop from a killed one: on 2026-10-29 this machine's kernel handed out its 32,768 ids again every 30 to 75 minutes, so a killed loop's question came back whenever another process got its id. The lock file stays once the loop ends, one per project: removing it while another process has it open could let loop-inbox lock a file no loop holds. PR #396. -- A loop also lists itself as running, with the items its chunk takes, in `.running`, under the same lock: one lock tells loop-inbox both that the loop runs and that its question stands. The judge names the items the worker took in, as an array of IDs in its reply's `items_taken_in`. PR #397. +- A loop also lists itself as running, with the items its chunk takes, in `.running`, under the same lock: one lock tells loop-inbox both that the loop runs and that its question stands. The items are those the worker names on its latest `Chunk items:` line (`routing: loop-claude reads the items a chunk takes`). A bench of three readers seated as the README's audience picked the name `loop-inbox`, 3–0. @@ -407,7 +408,7 @@ A run that asks for a higher rung may also ask a question. The judge then interr The judge may quote a different passage for the trigger that already stepped the rung, and loop-claude cannot tell from the quote alone whether the trigger is new. So it goes by where the quoted passage sits in the transcript. When an escalation steps the worker, the route marks how many transcript entries the session held: the mark. loop-claude then accepts an escalation quote only from the worker's own lines after the mark, never from its own prompts or the human's lines, which repeat earlier passages. The judge reads a `[step]` line at the mark saying the triggers before it are spent, so it quotes a passage the check accepts. The session's transcript only grows, so the mark holds across cycles. The trigger that bought a step, and any rewording of it before the mark, cannot buy another. A trigger that fires after the step steps the worker again, in the same cycle or a later one. One risk remains: a worker on the stepped rung may restate an old trigger after the mark, and the judge may quote that. The cap two rungs above the rung the item picks limits what this costs. Before the mark, once an interrupt or a quoted answer had taken an escalation, no later ruling of that cycle carried one, so a trigger that fired after the answer was lost. lilleman chose the mark on 2026-10-18 over two alternatives: lifting that rule when a worker run ended would have let one trigger buy two steps, and telling the worker its request was dropped would still have lost the step. -The step is one rung up from the rung the worker would otherwise hold: its own, or the rung an item the judge saw it take in picks, whichever is higher. It resets the count of unfinished verdicts, so the step every second one takes counts afresh; when both steps fall on one verdict, the worker steps once. It stops at the limit escalated subagents stop at, two rungs above the rung the chunk started on. Past that limit the rung holds: loop-claude takes no yes from the human between cycles, so a worker there asks the human for an escalated subagent instead. The judge keeps its rung, as `routing: on a model judge-replay covered` says. +The step is one rung up from the rung the worker would otherwise hold: its own, or the rung an item it names on its `Chunk items:` line picks, whichever is higher. It resets the count of unfinished verdicts, so the step every second one takes counts afresh; when both steps fall on one verdict, the worker steps once. It stops at the limit escalated subagents stop at, two rungs above the rung the chunk started on. Past that limit the rung holds: loop-claude takes no yes from the human between cycles, so a worker there asks the human for an escalated subagent instead. The judge keeps its rung, as `routing: on a model judge-replay covered` says. The archive quotes the judge's passage under `Escalation:` in the unfinished verdict, the interrupt or the quoted answer. The step an unfinished verdict causes is archived after it. The step an interrupt or a quoted answer causes is archived with it, at once: the interrupt or answer names the rung the worker ran on until then, and a step-up archived beside it names the rung it runs on from then. So the routing figures charge each stretch to its own rung, and a loop that dies while the human answers resumes on the stepped rung. Each archived worker line names the mark once there is one, so a resumed loop keeps it. `bin/routing_figures/` counts none of these steps as an escalation by trigger, which it counts only from subagent prompts. @@ -421,6 +422,16 @@ The cost of a cheaper worker shows as extra review rounds and unfinished verdict Since 2026-10-16 the reviewers may run above the worker's rung, on the one the highest E + I among their chunk's items picks, so chunks whose worker ran on a lower rung may take more rounds from stronger reviewers, and a call could blame the worker's rung for them; `todo.md` item 171 groups the rounds by the reviewers' rung too. Valid while loop-claude's archive holds every finished chunk's review rounds and unfinished verdicts. +## routing: loop-claude reads the items a chunk takes from the worker's latest `Chunk items:` line and scores them by `todo.md` as the default branch held it when the cycle started; the judge still reports the escalation and whether a release is ready to cut + +2026-11-11, claude. Serves goal 1.4 and code health. The judge's reply held three facts beside its verdict: the items the worker took into the chunk, with their highest L and E + I; whether a release was ready to cut; and any escalation the worker asked for. Routing lifts the worker's, the reviewers' and the judge's rungs by the items' levels. Code now reads the items: each fresh prompt asks the worker for a line opening `Chunk items:`, `bin/loop_claude/cycle/chunk_items.py` reads the latest such line in the worker's own text, and `release_plan.todo_table` scores the items. So the judge doesn't score anything, its reply is smaller, and one module scores every item. + +The judge field trials (`judge-field-trials/`) read each fact by code over the 709 archived cases. Code was wrong in each of the 39 cases where it and the judge disagreed on an escalation, and read a release ready to cut in 3 of the 31 interrupts that asked about a cut, so the judge keeps both. Workers stated an item with its L and E + I in 413 cases, but also stated items they filed, finished or only mentioned, so code reads a line of its own. Three fresh readers picked its label `Chunk items:`, 3 to 0, because it says the items are the chunk's. + +An item the chunk re-scores on its branch routes on its planned levels until the branch merges; where the new L picks a higher rung, the worker cites escalation trigger 1, which the judge still finds. A worker that writes no line takes in no item, so the route holds. Rerun `judge-field-trials/release_ready.py` when the way `chunk-preflight` detects a release review changes. + +Valid while workers write the line when asked, and while `todo.md` on the default branch scores each item a chunk takes. + ## routing: on a model judge-replay covered, the judge runs on the rung a review would get from the highest E + I among its chunk's items, never below the project's rung 2026-10-15, lilleman's plan, settled by the replay below. Serves goals 1.2 and 1.4, under the ladder and steps of `routing: one ladder`. @@ -453,11 +464,11 @@ A project's rung is its model's cheapest rung, which for Opus 5.5 is its default Valid while the index ranks these models, and while no newer model joins the catalog. -## routing: the worker changes rung only as a chunk starts, after every second unfinished verdict, or when the judge reports a taken-in item that picks a higher rung or an escalation trigger the worker cites +## routing: the worker changes rung only as a chunk starts, after every second unfinished verdict, or when it names a chunk item that picks a higher rung or cites an escalation trigger the judge finds 2026-10-14, lilleman. Serves goal 1.4. -Because a model switch inside a session discards its prompt cache, the worker changes rung only as the chunk starts, after every second unfinished verdict, when the judge reports a taken-in item whose L picks a higher rung, or when the judge finds the escalation trigger a worker cites in asking for a higher rung (`routing: a worker whose run ends asking for a higher rung`). loop-claude routes on the first row of the next release in `todo.md`'s table that does not wait: an item waits while it is marked `question` or a blocker of it is still in the table. Since `todo.md` holds no order but its table's (`todo.md: one table`), that row is the item the worker takes. An item the worker takes in beyond that row lifts the worker's rung once the judge reports it, after the first cycle. +Because a model switch inside a session discards its prompt cache, the worker changes rung only as the chunk starts, after every second unfinished verdict, when the worker names an item on its `Chunk items:` line whose L picks a higher rung (`routing: loop-claude reads the items a chunk takes`), or when the judge finds the escalation trigger a worker cites in asking for a higher rung (`routing: a worker whose run ends asking for a higher rung`). loop-claude routes on the first row of the next release in `todo.md`'s table that does not wait: an item waits while it is marked `question` or a blocker of it is still in the table. Since `todo.md` holds no order but its table's (`todo.md: one table`), that row is the item the worker takes. An item the worker takes in beyond that row lifts the worker's rung once the cycle that names it ends. Valid while a model switch inside a session discards its prompt cache. diff --git a/judge-field-trials/README.md b/judge-field-trials/README.md new file mode 100644 index 00000000..d405fe4c --- /dev/null +++ b/judge-field-trials/README.md @@ -0,0 +1,28 @@ +# Judge field trials + +These trials ask one question: which of the facts loop-claude's judge reports could code read instead? +The judge is the headless claude that rules on each worker run; its reply named the items the chunk took, whether a release was ready to cut, and any escalation the worker asked for. +They ran on 2026-11-11, over the 709 archived cases. `docs/decisions.md` → `routing: loop-claude reads the items a chunk takes` records what was decided from them. + +## Terms + +- **Case:** an outcomes file in `~/.claude/loop-claude-transcripts/`, its first verdict, and the transcript the judge read for it. +- **Final message:** the worker's text after the last prompt loop-claude or the human gave it. + +## Results + +| Field | What code tried | Result | +|---|---|---| +| Escalation | "higher rung", "rung up" or "one rung above" in the final message | Code is wrong in each of the 39 disagreements: 37 where only code finds a request, 2 where only the judge does. Code matched a worker asking the human's yes at the top rung and a review report quoting "rerun one rung up", and missed requests made before the final message. The judge keeps the field. | +| Release ready to cut | The plan as the default branch and tags stood at the verdict, read by `release_plan.release_standing` | Of 240 interrupts, 31 ask about a cut, and code reads ReadyToCut for 3. Two misses are code's: in two projects a release review had merged without the `Release review of ` commit that code looks for. The rest asked under older rules or after the cut. The judge keeps the field, and `todo.md` holds the fix. | +| Items the chunk takes | An item stated with its L and E + I anywhere in the worker's text | 413 of 709 cases state one, 77 of them more than one, and workers also state items they file, finish or only mention. Code needs a line of its own, which no archived transcript holds, so it can't be measured here. | + +## Running + +Each script reads the archive on this machine and prints its counts last; run it from the checkout's root, as in `python3 judge-field-trials/escalations.py`. +The lines above the counts quote transcripts and name projects, so they stay on this machine. + +- `cases.py` loads the cases, for the other three. +- `escalations.py` prints each escalation disagreement with the words around it. +- `release_ready.py` prints each interrupt's question beside whether code reads the release ready to cut. +- `items.py` prints a few cases stating more than one item. diff --git a/judge-field-trials/cases.py b/judge-field-trials/cases.py new file mode 100644 index 00000000..6de7026c --- /dev/null +++ b/judge-field-trials/cases.py @@ -0,0 +1,41 @@ +"""Each archived case: an outcomes file's first verdict, with the transcript copy the judge read for it.""" +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin")) + +from transcript_archive.outcomes import ANY_VERDICT_AT, OUTCOME_AT, task_of +from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text +from claude_code.transcript import read_transcript +from transcript_archive.location import transcript_of + +ARCHIVE = pathlib.Path.home() / ".claude/loop-claude-transcripts" + + +def first_block(outcomes: str) -> tuple[str, str, str] | None: + """The first verdict's stamp, its heading, such as `Verdict: interrupt`, and its whole block, up to the next outcome.""" + found = ANY_VERDICT_AT.search(outcomes) + if not found: + return None + ends = [start.start() for start in OUTCOME_AT.finditer(outcomes) if start.start() > found.start()] + return found[1], found[2], outcomes[found.start():ends[0] if ends else len(outcomes)] + + +def cases(): + """Each outcomes file in the archive whose task and first verdict can be read, with that verdict and its transcript's entries.""" + for outcomes in sorted(ARCHIVE.glob("*/*/*.outcomes.md")): + text = outcomes.read_text(encoding="utf-8") + block = first_block(text) + entries = read_transcript(str(transcript_of(outcomes))) + if block is not None and task_of(text) is not None and isinstance(entries, list): + yield outcomes, block, entries + + +def worker_texts(entries) -> list[str]: + return [part.text for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, Text)] + + +def last_run_text(entries) -> list[str]: + """The worker's text after the last prompt loop-claude or the human gave it: what the judge rules on.""" + at = max((index for index, entry in enumerate(entries) if isinstance(entry, LoopPrompt | HumanTurn)), default=-1) + return worker_texts(entries[at + 1:]) diff --git a/judge-field-trials/escalations.py b/judge-field-trials/escalations.py new file mode 100644 index 00000000..dbf09575 --- /dev/null +++ b/judge-field-trials/escalations.py @@ -0,0 +1,20 @@ +"""Could code find an escalation request where the judge did? Counts, per verdict kind, whether the worker's final message asks for a higher rung and whether the judge's verdict carried an escalation, and prints each disagreement with the words around it.""" +import collections +import re + +from cases import cases, last_run_text + +ASKS = re.compile(r"higher rung|rung up|one rung above", re.IGNORECASE) + +table = collections.Counter() +for outcomes, (stamp, kind, block), entries in cases(): + texts = last_run_text(entries) + final = texts[-1] if texts else "" + found = ASKS.search(final) + escalated = "\nEscalation:\n" in block + table[(kind, bool(found), escalated)] += 1 + if bool(found) != escalated: + around = final[max(0, found.start() - 200):found.end() + 150] if found else final[-300:] + print(f"## {outcomes.parent.parent.name} {stamp} {kind} code={bool(found)} judge={escalated}\n {around!r}\n") +for (kind, code, judge), count in sorted(table.items()): + print(f"{kind}: code {code}, judge {judge}: {count}") diff --git a/judge-field-trials/items.py b/judge-field-trials/items.py new file mode 100644 index 00000000..717b817a --- /dev/null +++ b/judge-field-trials/items.py @@ -0,0 +1,19 @@ +"""How do workers name the items a chunk takes, unasked? Counts, per verdict kind, the cases whose worker states an item with its L and E + I, and those stating more than one, and prints a few of the latter.""" +import collections +import re + +from cases import cases, worker_texts + +STATED = re.compile(r"item (\d+)\b[^.\n]{0,60}?\bL (\d)\b[^.\n]{0,40}?E \+ I (\d+)", re.IGNORECASE) + +table = collections.Counter() +shown = 0 +for outcomes, (stamp, kind, block), entries in cases(): + found = [match for text in worker_texts(entries) for match in STATED.finditer(text)] + ids = sorted({int(match[1]) for match in found}) + table[(kind, bool(found), len(ids) > 1)] += 1 + if len(ids) > 1 and shown < 12: + shown += 1 + print(outcomes.parent.parent.name, stamp, ids, [match[0][:120] for match in found[:4]]) +for (kind, stated, several), count in sorted(table.items()): + print(f"{kind}: states an item {stated}, more than one {several}: {count}") diff --git a/judge-field-trials/release_ready.py b/judge-field-trials/release_ready.py new file mode 100644 index 00000000..645c04c5 --- /dev/null +++ b/judge-field-trials/release_ready.py @@ -0,0 +1,53 @@ +"""Could code find a release ready to cut where the judge did? For each archived interrupt, prints whether the plan, as the project's default branch and tags stood at the verdict's stamp, read ReadyToCut, and whether the judge's question is about a cut.""" +import collections +import datetime +import pathlib +import re + +from cases import cases +from transcript_archive.project_key import checkout_of +from release_plan.release_standing import chunk_item, default_plan, ReadyToCut +from git_cli.git_repo import default_ref, git +from release_plan.todo_table import items + +ABOUT_CUT = re.compile(r"\bcut\b|ready to (?:cut|release|ship)|tag (?:it|v\d)|version bump", re.IGNORECASE) + + +def as_of(stamp: str): + """A git runner that lists only the tags created by stamp, so a later cut doesn't count.""" + def run(*args, cwd=None, **kwargs): + out = git(*args, cwd=cwd, **kwargs) + if "tag" in args and "--list" in args and isinstance(out, str): + dates = git("for-each-ref", "--format=%(refname:short) %(creatordate:iso-strict)", "refs/tags", cwd=cwd) + created = dict(line.split(" ", 1) for line in dates.splitlines()) if isinstance(dates, str) else {} + out = "\n".join(tag for tag in out.split() if utc(created.get(tag, "")) <= stamp) + return out + return run + + +def utc(iso: str) -> str: + """iso in the verdict stamp's form, or "" for no date, which sorts before every stamp.""" + return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") if iso else "" + + +table = collections.Counter() +for outcomes, (stamp, kind, block), entries in cases(): + if kind != "Verdict: interrupt": + continue + project = outcomes.parent.parent.name + root = str(checkout_of(project, pathlib.Path.home())) + question = block.split("\n\n", 2)[1] if block.count("\n\n") >= 2 else "" + about = bool(ABOUT_CUT.search(question)) + ref = default_ref(root, 10) if pathlib.Path(root).is_dir() else None + commit = git("rev-list", "-1", f"--before={stamp}", ref, cwd=root) if isinstance(ref, str) else None + if not isinstance(commit, str) or not commit.strip(): + table[("no checkout or commit", about)] += 1 + continue + plan = default_plan(root, commit.strip(), as_of(stamp)) + found = items(plan.todo) if isinstance(plan.todo, str) else plan.todo + pick = chunk_item(found, plan.cut) if isinstance(found, list) else found + ready = isinstance(pick, ReadyToCut) + table[(ready, about)] += 1 + print(f"{'ready' if ready else 'not '} {'cut?' if about else ' '} {project} {stamp} {type(pick).__name__}: {question[:220]!r}") +for (ready, about), count in sorted(table.items(), key=str): + print(f"ReadyToCut {ready}, asks about a cut {about}: {count}") diff --git a/todo.md b/todo.md index 6a97f906..9600eb2d 100644 --- a/todo.md +++ b/todo.md @@ -6,13 +6,12 @@ `Bar = 0.1` -`Next ID = 323` +`Next ID = 324` ## Items | ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score | |---|---|---|---|---|---|---|---|---|---|---|---| -| 297 | 0.8.0 | | | **Have loop-claude compute the judge's taken-in items, release-ready and escalation fields from `chunk-preflight`, git and the worker's structured lines, and leave the judge only the verdict kind.** | 4 | 2 | 3 | 7 | 4 | 1.4 by 2, code health by 2 | 3.7 | | 293 | 0.8.0 | | | **Run mutation tests on a chunk's changed lines in the worker's pre-review gate, through one command that picks the tool by language, and hand stability-reviewer the surviving mutants.** | 3 | 2 | 3 | 6 | 5 | 1.1 by 5, 1.4 by 4, 1.5 by 2 | 2.8 | | 284 | 0.8.0 | | owner | **Script each step that 0.7.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.** | 1 | 2 | 1 | 2 | 2 | 1.4 by 3, 1.5 by 3 | 2.6 | | 270 | 0.8.0 | | | **Move each check a review pass repeats across chunks, other than the mutation tests item 293 moves, into the worker's pre-review gate, starting with the passes most often taken to a second round.** | 2 | 2 | 3 | 5 | 3 | 1.1 by 3, 1.4 by 3 | 2.5 | @@ -30,7 +29,7 @@ | 311 | 0.8.0 | | | **Hand each worker's claude a Bash output ceiling that saves longer output to a file, and refuse a Read of a session transcript under `~/.claude/projects`.** | 2 | 2 | 3 | 5 | 3 | 1.4 by 3 | 1.2 | | 316 | 0.8.0 | | | **Launch comprehension seats as an agent type of their own under `agents/`, without `CLAUDE.md` and the skill and agent listings a general-purpose subagent carries.** | 2 | 2 | 2 | 4 | 3 | 1.4 by 3 | 1.2 | | 310 | 0.8.0 | | | **Compact a waiting worker's session shortly before its prompt cache expires, only when the human has not answered an interrupt by then.** | 3 | 2 | 3 | 6 | 4 | 1.4 by 3 | 1.0 | -| 298 | 0.8.0 | 297 | | **Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind.** | 3 | 3 | 3 | 7 | 4 | 1.4 by 2, 1.5 by 1 | 1.0 | +| 298 | 0.8.0 | | | **Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind.** | 3 | 3 | 3 | 7 | 4 | 1.4 by 2, 1.5 by 1 | 1.0 | | 313 | 0.8.0 | | | **Launch the first agent of each type in a parallel wave one message before the others of that type, so they read its prompt cache.** | 1 | 2 | 2 | 3 | 2 | 1.4 by 2 | 0.9 | | 317 | 0.8.0 | | | **Cut the descriptions of stability-reviewer, product-owner-reviewer and systems-architect to one line each.** | 1 | 1 | 1 | 1 | 1 | 1.4 by 1 | 0.5 | | 303 | 0.9.0 | | defect | **Archive each judge run's prompt and json reply beside the verdict it gave, so judge cost, replays and backups cover the judge as they cover the worker.** | 2 | 2 | 3 | 5 | 3 | 1.4 by 3, 3 by 2 | 4.8 | @@ -46,6 +45,7 @@ | 171 | 0.9.0 | | decision | **Group the routing figures' review rounds by the reviewers' rung as well as the worker's, so a call blames a rung only for rounds its own reviews raised.** | 3 | 2 | 2 | 5 | 4 | 1.2 by 3, 1.4 by 3 | 2.0 | | 300 | 0.9.0 | | owner | **Restructure the places the comprehension seats found hardest to follow, so the project reads at 7.0 or above on every dimension, Locality first: today the comprehension floor run at 52da15e scored Locality 6.00, Shape 6.25, Self-sufficiency 6.38 and Overall 6.38.** | 4 | 2 | 3 | 7 | 8 | code health by 8 | 2.0 | | 264 | 0.9.0 | | | **Report the comprehension seats and the escalated subagents as subagent types of their own in `routing-figures.md`, with their dollars.** | 2 | 2 | 1 | 3 | 3 | 1.4 by 5 | 2.0 | +| 323 | 0.9.0 | | | **Count a release review as landed once its pull request has merged, whatever its commits are titled.** | 3 | 2 | 3 | 6 | 3 | 1.4 by 3, 1.5 by 2 | 2.0 | | 149 | 0.9.0 | | decision | **Fill each release by priority up to a capacity it states, in place of the bar, and move what does not fit to a later release.** | 4 | 2 | 3 | 7 | 5 | 1.1 by 7 | 1.8 | | 165 | 0.9.0 | | | **Make this repo's tests assert the facts a prompt or help text carries in place of its sentences, and drop the tests that scan source or assert a constant.** | 3 | 2 | 1 | 4 | 5 | 1.1 by 4, 1.4 by 3 | 1.8 | | 151 | 0.9.0 | | decision | **Keep 10 to 20 settled decisions per project as its goals' test set, and rerun them after a goal changes.** | 3 | 2 | 2 | 5 | 5 | 1.2 by 7 | 1.8 | @@ -74,10 +74,6 @@ ## Details -### 297. Have loop-claude compute the judge's taken-in items, release-ready and escalation fields from `chunk-preflight`, git and the worker's structured lines, and leave the judge only the verdict kind. - -`bin/loop_claude/cycle/judge.py` already checks quotes in code (`quotes_checked`, `grounded`, `re_ask_checked`). The `Reply` fields `items_taken_in`, `release_ready` and `escalation` state facts the plan and git already hold, so code can derive them with certainty, and the judge's prompt shrinks to what needs judgement. Before switching, recompute each field for the 668 archived verdicts offline and count where code and judge disagree; read each disagreement and keep the judge for a field where code is wrong. Seats found `quotes_checked` hard to read (PR #397's scoring run: the misquote rule is split across `judge.py` and `escalation_mark.py`); make it easier to read where the change touches it. - ### 293. Run mutation tests on a chunk's changed lines in the worker's pre-review gate, through one command that picks the tool by language, and hand stability-reviewer the surviving mutants. Of stability-reviewer's 428 round-1 findings from 2026-11-02 to 2026-11-10, 223 (52%) were test gaps, such as a surviving mutant or an unpinned boundary, and 31 of its 50 round 2s held one. 133 of its 144 reports ran mutation tests by hand: today the reviewer picks and runs mutmut, Stryker, cargo-mutants or Infection itself, per `agents/stability-reviewer.md` step 5. A command run once before review, on the lines the branch changed, finds the same survivors deterministically, so the worker kills them before review and the reviewer judges only what remains. This takes over the stability half of item 270. Ship it alone: the nearest finished items, 227, 210, 180 and 73, changed 261 to 402 lines and took 3 to 8 rounds past the first. @@ -158,7 +154,7 @@ Decide once `agreement` has its 10 commits scored twice. Keep four seats on any ### 299. Replay the archived judge verdicts through code rules and a classifier trained locally on the archive, and file an item for each verdict kind one of them decides with no missed interrupt. -The judge's verdict kind is the one judge decision that item 297 leaves an LLM. No single rule found so far separates interrupts: a "?" in the worker's final message appears in 48 interrupts and misses 66. A smaller LLM already failed the replay (`docs/decisions.md` → `routing: on a model judge-replay covered, …`: Sonnet returned another kind on 3 interrupts, Haiku missed 7). Untested are combined code rules, a decision table, and a small classifier trained and run on this machine on the archived verdicts (372 finished, 234 interrupt, 42 unfinished, 20 quoted). Run each offline, nothing sent to an outside service (goal 4), and count disagreements by kind; the bar is zero missed interrupts. A hosted decision model is out of scope, since it would send transcripts to an outside service. +The judge's verdict kind, its escalation and whether a release is ready to cut are what an LLM still decides (`docs/decisions.md` → `routing: loop-claude reads the items a chunk takes …`); this item replays the verdict kind. No single rule found so far separates interrupts: a "?" in the worker's final message appears in 48 interrupts and misses 66. A smaller LLM already failed the replay (`docs/decisions.md` → `routing: on a model judge-replay covered, …`: Sonnet returned another kind on 3 interrupts, Haiku missed 7). Untested are combined code rules, a decision table, and a small classifier trained and run on this machine on the archived verdicts (372 finished, 234 interrupt, 42 unfinished, 20 quoted). Run each offline, nothing sent to an outside service (goal 4), and count disagreements by kind; the bar is zero missed interrupts. A hosted decision model is out of scope, since it would send transcripts to an outside service. ### 311. Hand each worker's claude a Bash output ceiling that saves longer output to a file, and refuse a Read of a session transcript under `~/.claude/projects`. @@ -180,7 +176,7 @@ Probe first that `claude -p "/compact" --resume ` compacts a session: the Ag ### 298. Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind. -153 `Judge cost:` entries total $109.76, a median of about $0.76 a verdict at every rung, and the cost follows the size of the records, not the transcript: about 76k characters in log and 188k in claude-config. In 224 of 251 recent verdicts (89%) the evidence sits in the worker's final message. A missed interrupt is the costly error, so the bar is zero changed interrupts and zero new rejections across the replay. It waits for item 297, which moves the judge's factual fields into code, since that shrinks the prompt first and changes what this cut measures. The judge is about 4.5% of dollars, so the saving is small; each cycle also gets faster. +153 `Judge cost:` entries total $109.76, a median of about $0.76 a verdict at every rung, and the cost follows the size of the records, not the transcript: about 76k characters in log and 188k in claude-config. In 224 of 251 recent verdicts (89%) the evidence sits in the worker's final message. A missed interrupt is the costly error, so the bar is zero changed interrupts and zero new rejections across the replay. The judge is about 4.5% of dollars, so the saving is small; each cycle also gets faster. ### 313. Launch the first agent of each type in a parallel wave one message before the others of that type, so they read its prompt cache. @@ -315,6 +311,10 @@ Scored L 4 because the refactor touches several components that hold known defec Seats launch as `general-purpose`, so they hide inside that type: $301.24 of the $962.17 the figures of 2026-11-04 count, from 221 subagents. By 2026-11-10 general-purpose was the second-largest spend, $585 of about $2,385, and by transcript size about two thirds of it were seats (291 launches), then 177 security passes. Escalations are counted by trigger only, never in dollars. `bin/routing_figures/chunk.py` groups subagents by their `subagent_type`; a seat is told apart by its prompt, and an escalated subagent by the escalation marker its prompt opens with. +### 323. Count a release review as landed once its pull request has merged, whatever its commits are titled. + +`release_reviewed` in `bin/release_plan/release_standing.py` counts a release review only by a commit titled `Release review of ` on the default branch. In a replay of 240 archived interrupts (`judge-field-trials/`), 31 asked about a cut, and the plan's standing at the time read ReadyToCut for only 3 of them. In two projects whose release review had merged, the default branch held no such commit, so `chunk-preflight` and routing read ReviewDue, and the next loop would run the release review again. `skills/larv-review/SKILL.md` opens the review's branch with an empty commit of that title, but a worker may skip it, and a rebase or squash merge, which `chunk-pr` uses where the forge allows no merge commit, may drop it. Detect the review by something every merge method keeps, and check the fix against both projects' histories. Until then the judge keeps `release_ready` (docs/decisions.md → `routing: loop-claude reads the items a chunk takes …`). + ### 149. Fill each release by priority up to a capacity it states, in place of the bar, and move what does not fit to a later release. A release's capacity is a sum of its items' cost, as the formula counts it. Each project states its own; the human sets the first value per project. Nothing is deleted by score.