Give the judge's contract one home in cycle/verdicts.py
This commit is contained in:
@@ -1,12 +1,10 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import re
|
||||
import typing
|
||||
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
|
||||
from loop_claude.claude_code.entries import hit_limit, limit_said, parse_entry, structured_output
|
||||
from loop_claude.cycle.verdicts import Interrupt, JUDGE_RULES, JUDGE_SCHEMA, QuotedAnswer, Verdict, verdict_from
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from loop_claude.cycle.retry import retried
|
||||
from loop_claude.stop import Stop
|
||||
@@ -14,51 +12,6 @@ from loop_claude.stop import Stop
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
JUDGE_PROMPT_CHARS = 400000
|
||||
|
||||
JUDGE_RULES = (
|
||||
"Decide whether the human must now be interrupted. interrupt=true when any of these appear:\n"
|
||||
"- the agent asks the human anything, seeks approval, or says a decision is the human's, and neither "
|
||||
"README.md nor the project's records settle the matter, or it asks again a question a [loop] line already answered\n"
|
||||
"- it carried on under an assumption that decides the deliverable, that the human could "
|
||||
"reasonably reject and that neither README.md nor the project's records settle; a caveat about "
|
||||
"transient state it then handled is not one\n"
|
||||
"- it is blocked on something only the human can supply: a credential, a version bump, a choice "
|
||||
"outside the repo, or a release ready to cut, which only the human may publish\n"
|
||||
"- it flags that the goals, the plan in todo.md or the task itself is unclear, missing or in need of change\n"
|
||||
"- it chose between alternatives that neither README.md nor the project's records discriminate "
|
||||
"between - neither rules the rejected one out - and the choice materially changes the deliverable\n"
|
||||
"interrupt=false otherwise. Bookkeeping the agent settled silently - what to document, wording, "
|
||||
"where a note goes - is not on its own a reason to interrupt. The human may have replied inside the transcript, on a [human] line: judge only what "
|
||||
"is still open, never a question they already answered there. A [loop] line is what loop-claude told "
|
||||
"the agent, such as an answer quoted from README.md or the project's records. A question a [loop] "
|
||||
"line answered is settled. If the agent asks it again, interrupt with that question.\n"
|
||||
"Where the agent asked or flagged something itself, the question is that one, in the agent's own "
|
||||
"wording; never construct another from facts it stated.\n"
|
||||
"Where README.md or the project's records answer the agent's own question and no [loop] line has "
|
||||
"quoted that answer yet, set interrupt=false, put that question in question and quote the passage that answers it in answer, verbatim; the agent is "
|
||||
"then given that passage in place of the human's reply.\n"
|
||||
"The chunk is the piece of work the agent is on, shipped as one pull request. "
|
||||
"finished=true when the chunk is complete: merged, or nothing remains that the agent could do on "
|
||||
"its own. finished=false when work remains that a further run can continue: an open PR, a review "
|
||||
"not yet run, CI not yet reported."
|
||||
)
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Reply:
|
||||
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the project's records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."})
|
||||
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."})
|
||||
finished: bool
|
||||
interrupt: bool
|
||||
question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the agent's question that the answer field answers. Empty otherwise."})
|
||||
|
||||
|
||||
REPLY_TYPES = typing.get_type_hints(Reply)
|
||||
JSON_TYPES = {bool: "boolean", str: "string"}
|
||||
JUDGE_SCHEMA = {
|
||||
"properties": {field.name: {**field.metadata, "type": JSON_TYPES[REPLY_TYPES[field.name]]} for field in dataclasses.fields(Reply)},
|
||||
"required": list(REPLY_TYPES),
|
||||
"type": "object",
|
||||
}
|
||||
|
||||
|
||||
def judge(root: str, task: str, premises: str, records: str, rendered_transcript: str, model: str, effects: Effects) -> Verdict | Stop:
|
||||
# --json-schema has claude check its reply against the schema and return it as the json result's structured_output.
|
||||
@@ -110,24 +63,6 @@ def judge_prompt(task: str, premises: str, records: str, rendered_transcript: st
|
||||
return f"{head}{rendered_transcript[max(0, len(rendered_transcript) - room):]}\n\n{JUDGE_RULES}"
|
||||
|
||||
|
||||
def verdict_from(output: object) -> Verdict | Failure:
|
||||
if not isinstance(output, dict):
|
||||
return Failure("the judge returned no reply")
|
||||
for name, kind in REPLY_TYPES.items():
|
||||
if not isinstance(output.get(name), kind):
|
||||
return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]}")
|
||||
# An interrupt outranks a quoted answer and finished: the human answers before the chunk can close.
|
||||
if output["interrupt"]:
|
||||
if not output["question"].strip():
|
||||
return Failure("the judge interrupted without a question")
|
||||
return Interrupt(output["evidence"], output["question"])
|
||||
if output["answer"].strip():
|
||||
if not output["question"].strip():
|
||||
return Failure("the judge quoted an answer without a question")
|
||||
return QuotedAnswer(output["evidence"], output["question"], output["answer"])
|
||||
return Finished(output["evidence"]) if output["finished"] else Unfinished(output["evidence"])
|
||||
|
||||
|
||||
def grounded(evidence: str, rendered_transcript: str) -> bool:
|
||||
# The judge joins passages with an ellipsis though told to quote one.
|
||||
fragments = [fragment.strip() for fragment in re.split(r"\.\.\.|…", plain(evidence)) if fragment.strip()]
|
||||
|
||||
@@ -6,7 +6,7 @@ from loop_claude.cycle.archive import archive
|
||||
from loop_claude.claude_code.session import delete_session, Session, session_open, transcript_file
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from loop_claude.cycle.verdicts import Finished, QuotedAnswer, Unfinished
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
||||
from loop_claude.cycle.interrupt import interrupt
|
||||
from loop_claude.cycle.judge import judge
|
||||
from loop_claude.project.model_choice import Models
|
||||
@@ -57,8 +57,8 @@ def step(cycle: Cycle, state: Active) -> Event | Stop:
|
||||
return work(cycle, answered)
|
||||
case Judging(transcript_path, _):
|
||||
return rule(cycle, transcript_path)
|
||||
case Interrupting(transcript_path, question, _):
|
||||
return ask(cycle, transcript_path, question)
|
||||
case Interrupting(transcript_path, ruling, quoted_answers):
|
||||
return ask(cycle, transcript_path, ruling, quoted_answers)
|
||||
case Closing(transcript_path, verdict):
|
||||
return close(cycle, transcript_path, verdict)
|
||||
case _:
|
||||
@@ -83,7 +83,8 @@ def rule(cycle: Cycle, transcript_path: str) -> Ruled | Stop:
|
||||
return verdict if isinstance(verdict, Stop) else Ruled(verdict)
|
||||
|
||||
|
||||
def ask(cycle: Cycle, transcript_path: str, question: str) -> HumanAnswered | Stop:
|
||||
def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer, quoted_answers: int) -> HumanAnswered | Stop:
|
||||
question = ruling.question if isinstance(ruling, Interrupt) else f"{ruling.question}\n\n{QUOTE_SOURCES} says:\n{ruling.answer}\n\nThose files have answered {quoted_answers} questions this cycle, the most allowed, so this one comes to you."
|
||||
# Archived now, so the transcript the question was asked on is kept with it; the human's answer then grows the transcript.
|
||||
if failure := archive(cycle.project_key, transcript_path, cycle.task, f"Verdict: interrupt\n\n{question}", cycle.paths):
|
||||
return Stop(1, failure.reason)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import dataclasses
|
||||
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
|
||||
from loop_claude.project.records import QUOTE_SOURCES
|
||||
|
||||
# A backstop for what the judge misses, such as a reworded repeat, so the worker cannot loop unattended on answers quoted from README.md or the records.
|
||||
QUOTED_ANSWERS_PER_CYCLE = 3
|
||||
@@ -23,8 +22,9 @@ class Judging:
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Interrupting:
|
||||
"""The human answers; a QuotedAnswer here is one past the cycle's QUOTED_ANSWERS_PER_CYCLE."""
|
||||
transcript_path: str
|
||||
question: str
|
||||
ruling: Interrupt | QuotedAnswer
|
||||
quoted_answers: int = 0
|
||||
|
||||
|
||||
@@ -72,10 +72,8 @@ def transition(state: State, event: Event) -> State:
|
||||
return Judging(transcript_path, quoted_answers)
|
||||
case Judging(_, quoted_answers), Ruled(QuotedAnswer() as answered) if quoted_answers < QUOTED_ANSWERS_PER_CYCLE:
|
||||
return Working(answered, quoted_answers + 1)
|
||||
case Judging(transcript_path, quoted_answers), Ruled(QuotedAnswer(question=question, answer=answer)):
|
||||
return Interrupting(transcript_path, f"{question}\n\n{QUOTE_SOURCES} says:\n{answer}\n\nThose files have answered {quoted_answers} questions this cycle, the most allowed, so this one comes to you.", quoted_answers)
|
||||
case Judging(transcript_path, quoted_answers), Ruled(Interrupt(question=question)):
|
||||
return Interrupting(transcript_path, question, quoted_answers)
|
||||
case Judging(transcript_path, quoted_answers), Ruled(Interrupt() | QuotedAnswer() as ruling):
|
||||
return Interrupting(transcript_path, ruling, quoted_answers)
|
||||
case Judging(transcript_path, _), Ruled(Finished() | Unfinished() as verdict):
|
||||
return Closing(transcript_path, verdict)
|
||||
# The human works in the interrupt's claude session, so the judge rules again on the transcript it leaves.
|
||||
|
||||
@@ -1,4 +1,54 @@
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from file_io.failure import Failure
|
||||
|
||||
# The one statement of when the judge interrupts, quotes an answer or finds the chunk finished; Reply's field descriptions say only what each field holds.
|
||||
JUDGE_RULES = (
|
||||
"Decide whether the human must now be interrupted. interrupt=true when any of these appear:\n"
|
||||
"- the agent asks the human anything, seeks approval, or says a decision is the human's, and neither "
|
||||
"README.md nor the project's records settle the matter\n"
|
||||
"- it asks again a question a [loop] line already answered\n"
|
||||
"- it carried on under an assumption that decides the deliverable, that the human could "
|
||||
"reasonably reject and that neither README.md nor the project's records settle; a caveat about "
|
||||
"transient state it then handled is not one\n"
|
||||
"- it is blocked on something only the human can supply: a credential, a version bump, a choice "
|
||||
"outside the repo, or a release ready to cut, which only the human may publish\n"
|
||||
"- it flags that the goals, the plan in todo.md or the task itself is unclear, missing or in need of change\n"
|
||||
"- it chose between alternatives that neither README.md nor the project's records discriminate "
|
||||
"between - neither rules the rejected one out - and the choice materially changes the deliverable\n"
|
||||
"interrupt=false otherwise. Bookkeeping the agent settled silently - what to document, wording, "
|
||||
"where a note goes - is not on its own a reason to interrupt. The human may have replied inside the transcript, on a [human] line: judge only what "
|
||||
"is still open, never a question they already answered there. A [loop] line is what loop-claude told "
|
||||
"the agent, such as an answer quoted from README.md or the project's records.\n"
|
||||
"Where the agent asked or flagged something itself, the question is that one, in the agent's own "
|
||||
"wording; never construct another from facts it stated.\n"
|
||||
"Where README.md or the project's records answer the agent's own question and no [loop] line has "
|
||||
"quoted that answer yet, set interrupt=false and quote the answer; the agent is "
|
||||
"then given that passage in place of the human's reply.\n"
|
||||
"The chunk is the piece of work the agent is on, shipped as one pull request. "
|
||||
"finished=true when the chunk is complete: merged, or nothing remains that the agent could do on "
|
||||
"its own. finished=false when work remains that a further run can continue: an open PR, a review "
|
||||
"not yet run, CI not yet reported."
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Reply:
|
||||
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the project's records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."})
|
||||
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."})
|
||||
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished. Read only when interrupt is false and answer is empty."})
|
||||
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted. True outranks answer and finished."})
|
||||
question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the agent's question that the answer field answers. Empty otherwise."})
|
||||
|
||||
|
||||
REPLY_TYPES = typing.get_type_hints(Reply)
|
||||
JSON_TYPES = {bool: "boolean", str: "string"}
|
||||
JUDGE_SCHEMA = {
|
||||
"properties": {field.name: {**field.metadata, "type": JSON_TYPES[REPLY_TYPES[field.name]]} for field in dataclasses.fields(Reply)},
|
||||
"required": list(REPLY_TYPES),
|
||||
"type": "object",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -26,3 +76,21 @@ class Unfinished:
|
||||
|
||||
|
||||
Verdict = Interrupt | QuotedAnswer | Finished | Unfinished
|
||||
|
||||
|
||||
def verdict_from(output: object) -> Verdict | Failure:
|
||||
if not isinstance(output, dict):
|
||||
return Failure("the judge returned no reply")
|
||||
for name, kind in REPLY_TYPES.items():
|
||||
if not isinstance(output.get(name), kind):
|
||||
return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]}")
|
||||
# The human answers before the chunk can close.
|
||||
if output["interrupt"]:
|
||||
if not output["question"].strip():
|
||||
return Failure("the judge interrupted without a question")
|
||||
return Interrupt(output["evidence"], output["question"])
|
||||
if output["answer"].strip():
|
||||
if not output["question"].strip():
|
||||
return Failure("the judge quoted an answer without a question")
|
||||
return QuotedAnswer(output["evidence"], output["question"], output["answer"])
|
||||
return Finished(output["evidence"]) if output["finished"] else Unfinished(output["evidence"])
|
||||
|
||||
+17
-17
@@ -21,7 +21,7 @@ from loop_claude.project import checkout, model_choice, records
|
||||
from loop_claude.effects import Effects
|
||||
from loop_claude.claude_code import entries, session, transcript, trust
|
||||
from file_io.failure import Failure
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, verdict_from
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from loop_claude.loop import main, reload
|
||||
from file_io import markdown, writes
|
||||
@@ -132,23 +132,23 @@ class Transitions(unittest.TestCase):
|
||||
legal = (
|
||||
(states.Working(), states.WorkerRan("/t"), states.Judging("/t")),
|
||||
(states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
|
||||
(states.Judging("/t"), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", "Which one?")),
|
||||
(states.Judging("/t", 3), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", "Which one?", 3)),
|
||||
(states.Judging("/t"), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"))),
|
||||
(states.Judging("/t", 3), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"), 3)),
|
||||
(states.Judging("/t"), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 1)),
|
||||
(states.Judging("/t", 2), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 3)),
|
||||
(states.Judging("/t", 3), answered, states.Interrupting("/t", "Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.", 3)),
|
||||
(states.Judging("/t", 3), answered, states.Interrupting("/t", QuotedAnswer("e", "Which one?", "A."), 3)),
|
||||
(states.Judging("/t"), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
|
||||
(states.Judging("/t", 3), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
|
||||
(states.Judging("/t"), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
|
||||
(states.Judging("/t", 3), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
|
||||
(states.Interrupting("/t", "q"), states.HumanAnswered(), states.Judging("/t")),
|
||||
(states.Interrupting("/t", "q", 3), states.HumanAnswered(), states.Judging("/t", 3)),
|
||||
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanAnswered(), states.Judging("/t")),
|
||||
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanAnswered(), states.Judging("/t", 3)),
|
||||
(states.Closing("/t", Finished("e")), states.Archived(), states.Ended()),
|
||||
)
|
||||
for state, event, after in legal:
|
||||
with self.subTest(state=state, event=event):
|
||||
self.assertEqual(states.transition(state, event), after)
|
||||
every_state = (states.Working(), states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.Judging("/t"), states.Judging("/t", 3), states.Interrupting("/t", "q"), states.Interrupting("/t", "q", 3), states.Closing("/t", Finished("e")), states.Ended())
|
||||
every_state = (states.Working(), states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.Judging("/t"), states.Judging("/t", 3), states.Interrupting("/t", Interrupt("e", "q")), states.Interrupting("/t", Interrupt("e", "q"), 3), states.Closing("/t", Finished("e")), states.Ended())
|
||||
every_event = (states.WorkerRan("/t"), states.Ruled(Interrupt("e", "Which one?")), answered, states.Ruled(Finished("e")), states.Ruled(Unfinished("e")), states.HumanAnswered(), states.Archived())
|
||||
for state in every_state:
|
||||
for event in every_event:
|
||||
@@ -513,16 +513,16 @@ class Grounded(unittest.TestCase):
|
||||
|
||||
class Verdict(unittest.TestCase):
|
||||
def test_needs_every_field_the_schema_requires(self):
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Finished("e"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored"}), Unfinished("e"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?"}), Interrupt("e", "Which one?"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?"}), QuotedAnswer("e", "Which one?", "A."))
|
||||
self.assertEqual(judge.verdict_from({"answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?"}), Interrupt("e", "Which one?"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Failure("the judge quoted an answer without a question"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": ""}), Failure("the judge's reply has no interrupt of type boolean"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": False, "evidence": None, "finished": True, "question": ""}), Failure("the judge's reply has no evidence of type string"))
|
||||
self.assertEqual(judge.verdict_from({"answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " "}), Failure("the judge interrupted without a question"))
|
||||
self.assertEqual(judge.verdict_from(None), Failure("the judge returned no reply"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Finished("e"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored"}), Unfinished("e"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?"}), Interrupt("e", "Which one?"))
|
||||
self.assertEqual(verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?"}), QuotedAnswer("e", "Which one?", "A."))
|
||||
self.assertEqual(verdict_from({"answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?"}), Interrupt("e", "Which one?"))
|
||||
self.assertEqual(verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Failure("the judge quoted an answer without a question"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": ""}), Failure("the judge's reply has no interrupt of type boolean"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": None, "finished": True, "question": ""}), Failure("the judge's reply has no evidence of type string"))
|
||||
self.assertEqual(verdict_from({"answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " "}), Failure("the judge interrupted without a question"))
|
||||
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
|
||||
|
||||
|
||||
class LoopRuns(unittest.TestCase):
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@
|
||||
|
||||
2026-09-27, lilleman, restating the 2026-09-24 target and extending it to the overall. Serves goal 4. The 2026-09-28 run, Junior A, Mid A, the maintainability senior and the inherited architect at depth 1 over every non-test file, blind to `todo.md` and this log, scored Navigation 7.25, Locality 6.0, Shape 6.5, Self-sufficiency 6.25, overall 6.25; on 2026-09-26 they were 7.75, 6.25, 7.0, 6.75, 6.75. Every seat scored Locality 6, and three of four named `run_cycle`'s three-way return and larv-review's re-entry rules, split across the passes on 2026-09-28. The same seats on `rebuild` scored 7.5, 6.63, 6.88, 6.5, 7.0 at 37edc9a, later that day, 7.5, 6.75, 6.88, 6.75, 7.0 at 7f9de5c, 7.38, 6.25, 6.88, 6.75, 6.88 at 2f9ab2f on 2026-09-29, Locality lowest again and every seat naming the reload state and the two retry counters, and 7.5, 7.0, 7.25, 6.75, 7.13 at 2c74bbc later that day, Self-sufficiency lowest and every seat naming the round's definition split across larv-review's two files, and 7.38, 6.75, 7.25, 6.5, 6.88 at 8585257 on 2026-10-01, lowering every dimension but Shape, and the overall, Self-sufficiency lowest and every seat naming larv-review §3's round rules; the places it named under Self-sufficiency and Locality are filed in `todo.md` under 0.1.0, and a later scoring run may not score any dimension or the overall below its best run on `rebuild`, 2026-10-01, lilleman. `rebuild` merged into main under the floor at 8585257's scores, 2026-10-01, lilleman. The 2026-10-01 run on PR #182 (the loop state machine) scored 7.5, 6.5, 6.88, 6.25, 6.38; lilleman shipped that PR under the floor and below `rebuild`'s best run the same day, its places filed in `todo.md` under 0.1.0. The 2026-10-02 run on PR #187 (`loop_claude`'s names, the exit contract) scored 7.75, 7.25, 7.38, 6.88, 7.25 at 70362d0, every seat naming larv-review §3 under Self-sufficiency, and 7.63, 7.13, 7.38, 7.0, 7.13 at 1971dba later that day, the first run at the floor, with no dimension or the overall below its best run on `rebuild`. The 2026-10-02 runs on PR #191 (the status line in Python) scored 7.5, 7.0, 7.13, 6.88, 7.13; 7.25, 6.75, 7.25, 6.88, 7.0; 7.63, 6.88, 7.38, 7.0, 7.13; and 7.63, 7.0, 7.25, 6.75, 7.25. Every seat rated the status line easy. The places they named under Locality and Self-sufficiency are filed in `todo.md` under 0.1.0, except `entries.py`'s heuristics over Claude Code's transcript format, which the seats scored as the problem's own difficulty. lilleman decided the same day to ship that PR under the floor. The 2026-10-02 runs on PR #194 (each cycle's transcript archived) scored 8.0, 7.25, 7.38, 7.13, 7.5 at 6ccd5b1 and 7.75, 7.13, 7.25, 7.0, 7.25 at d104ea0, both at the floor, every seat naming `archive` as the unit it would least want to modify. On 2026-10-02, applying `todo.md`'s bar deleted every place these scoring runs filed except larv-review §3's round rules and the Claude Code flags no comment explained. The 2026-10-02 runs on PR #196 (`todo.md` as one scored table) scored 7.63, 6.88, 7.25, 6.88, 7.13 at c0dba66; 7.5, 6.88, 7.13, 6.75, 7.0 at 798dfd7; and 7.5, 7.0, 7.0, 6.75, 7.0 at b712076. Under Self-sufficiency, every seat named larv-review §3 and the agents, which a reader follows only with `CLAUDE.md` open. lilleman decided the same day to ship that PR under the floor; the place filed was larv-review §3's round rules. The 2026-10-02 run on PR #198 (a usage limit waited out) scored 7.63, 6.88, 7.38, 6.75, 7.13 at 8e85033. Under Self-sufficiency, every seat named larv-review §3; under Locality, they named `archive`'s invariants spread over `session.py` and `run.py`, `rank_todo.py` reaching into `premises.sections`, and `stopped` comparing reason strings, each under `todo.md`'s bar and not filed. lilleman decided the same day to ship that PR under the floor.
|
||||
|
||||
The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.75, 7.13, 7.25, 7.0, 7.25 in the first round and 7.5, 7.0, 7.0, 6.75, 7.0 in the eighth; between them a dimension moved by up to 0.4 on near-identical code. Under Self-sufficiency every seat named larv-review §3 and the agents leaning on `CLAUDE.md`, the Claude Code flags left unexplained, and the judge's contract spread over its prompt, schema, parser and `--help`; under Shape, that "record", "verdict" and "judge" each name two things, and that `loop_claude/` holds modules `rank_todo.py` also uses. lilleman decided the same day to ship that PR under the floor and to fix those places next: larv-review §3 and `todo.md` items 42, 43 and 44. The flags, and the words the judge's prompt left undefined (chunk, plan, a release ready to cut), were explained on 2026-10-03, and larv-review §3 was made readable on its own the same day. Valid until the next scoring run, which uses the same four seats.
|
||||
The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.75, 7.13, 7.25, 7.0, 7.25 in the first round and 7.5, 7.0, 7.0, 6.75, 7.0 in the eighth; between them a dimension moved by up to 0.4 on near-identical code. Under Self-sufficiency every seat named larv-review §3 and the agents leaning on `CLAUDE.md`, the Claude Code flags left unexplained, and the judge's contract spread over its prompt, schema, parser and `--help`; under Shape, that "record", "verdict" and "judge" each name two things, and that `loop_claude/` holds modules `rank_todo.py` also uses. lilleman decided the same day to ship that PR under the floor and to fix those places next. All were fixed on 2026-10-03: the flags and the words the judge's prompt left undefined (chunk, plan, a release ready to cut) explained, larv-review §3 made readable on its own, the judge's contract given one home in `cycle/verdicts.py`, the three words one meaning each, and the shared modules moved to `bin/file_io/`. Valid until the next scoring run, which uses the same four seats.
|
||||
|
||||
## loop-claude: each project's model lives in `~/.claude/loop-claude-models.json`, which only loop-claude writes
|
||||
|
||||
|
||||
@@ -25,9 +25,6 @@
|
||||
| ID | Release | Exempt | Item | R | S | A | G | Goals | Score |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| 1 | 0.1.0 | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 3 | 5 | 8 | 9 | 2, 3 | 27.8 |
|
||||
| 42 | 0.1.0 | decision | **Give the judge's contract one home, so its prompt, reply schema, parser and `--help` cannot drift apart.** | 3 | 4 | 9 | 9 | 4, 4.1 | 26.1 |
|
||||
| 44 | 0.1.0 | decision | **Move the modules `rank_todo.py` shares out from under `loop_claude/`, keeping them inside reload's fingerprint.** | 2 | 2 | 8 | 8 | 4, 4.1 | 24.2 |
|
||||
| 43 | 0.1.0 | decision | **Give "record", "verdict" and "judge" one meaning each across loop-claude, larv-review and workflow-bench.** | 2 | 3 | 8 | 8 | 4, 4.1 | 24.0 |
|
||||
| 23 | 0.1.0 | | **Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.** | 3 | 4 | 7 | 8 | 4.4 | 13.5 |
|
||||
| 21 | 0.1.0 | | **Give `settings.json`'s model one writer, and sort its keys.** | 3 | 3 | 6 | 7 | 4.3 | 12.9 |
|
||||
| 12 | 0.1.0 | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 2 | 4 | 4 | 6 | 4.1 | 11.7 |
|
||||
@@ -44,18 +41,6 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost
|
||||
|
||||
2026-10-02, the human: ai-sandbox is the VM `loop-claude` will run on, hosted on the server bender. bender-infra, the infrastructure-as-code project that manages bender, will back up ai-sandbox's data volume once ai-sandbox is rebuilt. The item waits for the human to say the rebuild has landed. Before writing the README line, check that `~/.claude` lives on that volume and that the backup is private and encrypted at rest.
|
||||
|
||||
### 42. Give the judge's contract one home, so its prompt, reply schema, parser and `--help` cannot drift apart.
|
||||
|
||||
The 2026-10-03 scoring runs on PR #202 named it under Self-sufficiency and Locality: what a reply means is stated in `JUDGE_RULES`, the `Reply` field descriptions, `verdict_from`'s precedence, `transition`'s cap and the `--help` docstring, and the repeated-question rule lives only in the prompt. Derive each from one definition, or tie it to that definition with a test; the interrupt wording moves out of `transition` into `run.ask`.
|
||||
|
||||
### 44. Move the modules `rank_todo.py` shares out from under `loop_claude/`, keeping them inside reload's fingerprint.
|
||||
|
||||
Named under Shape by the same runs: `failure.py`, `file_writes.py` and `markdown_sections.py` sit in loop-claude's package because `reload.fingerprint` hashes only `loop_claude/**/*.py`, and nothing at the package says a second tool depends on them. Widen the fingerprint to a named shared package.
|
||||
|
||||
### 43. Give "record", "verdict" and "judge" one meaning each across loop-claude, larv-review and workflow-bench.
|
||||
|
||||
Named under Shape by the same runs: loop-claude's records are `AGENTS.md` and `docs/decisions.md`, larv-review's "record" is a decision-log entry or plan item older than the chunk; `Verdict` is the judge's ruling and `<sha>.verdict.md` also holds stop reasons; `cycle/judge.py` and `workflow-bench/judge.py` are different judges.
|
||||
|
||||
### 23. Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.
|
||||
|
||||
On this repo the frontend, REST and logging auditors each read the whole project to report "does not apply", and one 🟢 fix reruns ten whole-project reads.
|
||||
|
||||
Reference in New Issue
Block a user