Answer architecture round 1: Failure and the file modules get truthful homes, the interrupt question is a pure function, the judge grounds on what it saw
This commit is contained in:
+3
-2
@@ -17,8 +17,9 @@
|
||||
!agents/
|
||||
!agents/*.md
|
||||
!bin/
|
||||
!bin/file_io/
|
||||
!bin/file_io/*.py
|
||||
!bin/failure.py
|
||||
!bin/files/
|
||||
!bin/files/*.py
|
||||
!bin/loop-claude
|
||||
!bin/loop_claude/
|
||||
!bin/loop_claude/**/
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
|
||||
|
||||
def make_private(folder: pathlib.Path) -> None:
|
||||
@@ -5,7 +5,7 @@ import pathlib
|
||||
import shutil
|
||||
import uuid
|
||||
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.paths import Paths
|
||||
|
||||
NAMESPACE = uuid.UUID("64fb586b-d926-4bb3-a44c-c903e86d3eec")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
|
||||
from loop_claude.claude_code.entries import assistant_parts, human_text, loop_text, parse_entry, part_text, tool_call
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
|
||||
JUDGED_TOOL_INPUT = 400
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from file_io.failure import Failure
|
||||
from file_io.writes import replace_json
|
||||
from failure import Failure
|
||||
from files.writes import replace_json
|
||||
|
||||
|
||||
# A failure only warns: the interrupt still runs, and the human answers Claude Code's trust dialog.
|
||||
|
||||
@@ -3,9 +3,9 @@ import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.claude_code.transcript import first_timestamped_line
|
||||
from file_io.writes import make_private, replace_file
|
||||
from files.writes import make_private, replace_file
|
||||
from loop_claude.paths import Paths
|
||||
from loop_claude.claude_code.session import session_files
|
||||
|
||||
@@ -24,7 +24,8 @@ def archive(project_key: str, path: str, task: str, outcome: str, paths: Paths)
|
||||
return unkept
|
||||
copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl"
|
||||
outcomes = copy.with_suffix(".outcomes.md")
|
||||
if copy.exists():
|
||||
# A copy whose outcomes file is missing is written afresh, outcomes first.
|
||||
if copy.exists() and outcomes.exists():
|
||||
return add_outcome(outcomes, outcome)
|
||||
# The copy lands last, so a copy on disk always has its outcomes and its session's files beside it.
|
||||
for dest, data in ((outcomes, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)):
|
||||
|
||||
@@ -2,7 +2,7 @@ import sys
|
||||
import uuid
|
||||
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.claude_code.transcript import human_turns, read_transcript
|
||||
from loop_claude.stop import needs_human, needs_terminal, Stop
|
||||
from loop_claude.paths import Paths
|
||||
|
||||
@@ -2,11 +2,10 @@ import json
|
||||
import re
|
||||
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.claude_code.entries import hit_limit, limit_said, parse_entry, structured_output
|
||||
from loop_claude.cycle.verdicts import Interrupt, JUDGE_RULES, JUDGE_SCHEMA, QuotedAnswer, Verdict, verdict_from
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from loop_claude.cycle.retry import retried
|
||||
from loop_claude.cycle.retry import Limited, retried
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
@@ -25,11 +24,11 @@ def judge(root: str, task: str, premises: str, records: str, rendered_transcript
|
||||
"--permission-mode", "bypassPermissions",
|
||||
"--tools", "",
|
||||
]
|
||||
prompt = judge_prompt(task, premises, records, rendered_transcript)
|
||||
return retried("judge", lambda: run_judge(root, args, prompt, rendered_transcript, f"{premises}\n\n{records}", effects), effects)
|
||||
prompt, shown = judge_prompt(task, premises, records, rendered_transcript)
|
||||
return retried("judge", lambda: run_judge(root, args, prompt, shown, f"{premises}\n\n{records}", effects), effects)
|
||||
|
||||
|
||||
def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited:
|
||||
def run_judge(root: str, args: list[str], prompt: str, shown: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited:
|
||||
try:
|
||||
proc = effects.claude(args, root, prompt)
|
||||
except OSError as failure:
|
||||
@@ -43,7 +42,7 @@ def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str,
|
||||
verdict = verdict_from(structured_output(reply))
|
||||
if isinstance(verdict, Failure):
|
||||
return Failure(f"{verdict.reason}; it replied: {proc.stdout.strip() or 'nothing'}")
|
||||
if not grounded(verdict.evidence, rendered_transcript):
|
||||
if not grounded(verdict.evidence, shown):
|
||||
return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the transcript" + (f", for the question: {verdict.question}" if isinstance(verdict, Interrupt | QuotedAnswer) else ""))
|
||||
# Not a Failure: a misquote would otherwise exit an unattended loop. Never elided: stitched fragments can say what the records do not.
|
||||
if isinstance(verdict, QuotedAnswer) and not verbatim(verdict.answer, quotable):
|
||||
@@ -51,7 +50,8 @@ def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str,
|
||||
return verdict
|
||||
|
||||
|
||||
def judge_prompt(task: str, premises: str, records: str, rendered_transcript: str) -> str:
|
||||
def judge_prompt(task: str, premises: str, records: str, rendered_transcript: str) -> tuple[str, str]:
|
||||
"""The prompt, and the part of the transcript it shows, which the judge's evidence must quote."""
|
||||
head = (
|
||||
f"From README.md:\n{premises}\n\n"
|
||||
f"The project's records, its working rules and decisions:\n{records or 'none'}\n\n"
|
||||
@@ -60,7 +60,8 @@ def judge_prompt(task: str, premises: str, records: str, rendered_transcript: st
|
||||
)
|
||||
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2)
|
||||
# The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle.
|
||||
return f"{head}{rendered_transcript[max(0, len(rendered_transcript) - room):]}\n\n{JUDGE_RULES}"
|
||||
shown = rendered_transcript[max(0, len(rendered_transcript) - room):]
|
||||
return f"{head}{shown}\n\n{JUDGE_RULES}", shown
|
||||
|
||||
|
||||
def grounded(evidence: str, rendered_transcript: str) -> bool:
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
import dataclasses
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Limited:
|
||||
"""claude hit a usage limit; said is claude's message about it, and resets_at is when the limit resets, in seconds since the epoch, or None when unknown."""
|
||||
resets_at: int | None
|
||||
said: str
|
||||
@@ -1,17 +1,26 @@
|
||||
import dataclasses
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from failure import Failure
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
T = typing.TypeVar("T")
|
||||
|
||||
FAILURES_RUNNING = 3
|
||||
PAST_RESET = 120
|
||||
RETRY_DELAY = 60
|
||||
UNKNOWN_RESET_POLL = 600
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Limited:
|
||||
"""claude hit a usage limit; said is claude's message about it, and resets_at is when the limit resets, in seconds since the epoch, or None when unknown."""
|
||||
resets_at: int | None
|
||||
said: str
|
||||
|
||||
|
||||
def retried(who: str, attempt: typing.Callable[[], T | Failure | Limited], effects: Effects) -> T | Stop:
|
||||
failures = 0
|
||||
while True:
|
||||
@@ -27,10 +36,10 @@ def retried(who: str, attempt: typing.Callable[[], T | Failure | Limited], effec
|
||||
if not isinstance(result, Failure):
|
||||
return result
|
||||
failures += 1
|
||||
if failures == 3:
|
||||
if failures == FAILURES_RUNNING:
|
||||
return Stop(1, f"the {who} failed three times running: {result.reason}")
|
||||
print(f"retrying the {who} in a minute: {result.reason}", file=sys.stderr)
|
||||
effects.sleep(60)
|
||||
effects.sleep(RETRY_DELAY)
|
||||
|
||||
|
||||
def limit_wait(limited: Limited, now: float) -> float:
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanAnswered, Interrupting, Judging, Ruled, State, transition, WorkerRan, Working
|
||||
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanAnswered, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, transition, WorkerRan, Working
|
||||
from loop_claude.cycle.archive import archive
|
||||
from loop_claude.claude_code.session import delete_session, Session, session_open, transcript_file
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
||||
from loop_claude.cycle.interrupt import interrupt
|
||||
from loop_claude.cycle.judge import judge
|
||||
@@ -57,8 +57,8 @@ def step(cycle: Cycle, state: Active) -> Event | Stop:
|
||||
return work(cycle, answered)
|
||||
case Judging(transcript_path, _):
|
||||
return rule(cycle, transcript_path)
|
||||
case Interrupting(transcript_path, ruling, quoted_answers):
|
||||
return ask(cycle, transcript_path, ruling, quoted_answers)
|
||||
case Interrupting(transcript_path, ruling, _):
|
||||
return ask(cycle, transcript_path, ruling)
|
||||
case Closing(transcript_path, verdict):
|
||||
return close(cycle, transcript_path, verdict)
|
||||
case _:
|
||||
@@ -83,14 +83,20 @@ def rule(cycle: Cycle, transcript_path: str) -> Ruled | Stop:
|
||||
return verdict if isinstance(verdict, Stop) else Ruled(verdict)
|
||||
|
||||
|
||||
def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer, quoted_answers: int) -> HumanAnswered | Stop:
|
||||
question = ruling.question if isinstance(ruling, Interrupt) else f"{ruling.question}\n\n{QUOTE_SOURCES} says:\n{ruling.answer}\n\nThose files have answered {quoted_answers} questions this cycle, the most allowed, so this one comes to you."
|
||||
def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer) -> HumanAnswered | Stop:
|
||||
question = question_for(ruling)
|
||||
# Archived now, so the transcript the question was asked on is kept with it; the human's answer then grows the transcript.
|
||||
if failure := archive(cycle.project_key, transcript_path, cycle.task, f"Verdict: interrupt\n\n{question}", cycle.paths):
|
||||
return Stop(1, failure.reason)
|
||||
return interrupt(cycle.root, cycle.session.id, cycle.models.worker, transcript_path, question, cycle.paths, cycle.effects) or HumanAnswered()
|
||||
|
||||
|
||||
def question_for(ruling: Interrupt | QuotedAnswer) -> str:
|
||||
if isinstance(ruling, Interrupt):
|
||||
return ruling.question
|
||||
return f"{ruling.question}\n\n{QUOTE_SOURCES} says:\n{ruling.answer}\n\nThose files have answered {QUOTED_ANSWERS_PER_CYCLE} questions this cycle, the most allowed, so this one comes to you."
|
||||
|
||||
|
||||
def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) -> Archived | Stop:
|
||||
finished = isinstance(verdict, Finished)
|
||||
if failure := archive(cycle.project_key, transcript_path, cycle.task, f"Verdict: {'finished' if finished else 'unfinished'}", cycle.paths):
|
||||
|
||||
@@ -15,7 +15,7 @@ class Working:
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Judging:
|
||||
"""The judge rules; quoted_answers counts the questions README.md or the records answered this cycle."""
|
||||
"""The judge rules; quoted_answers counts the questions README.md or the records answered this cycle, and an interrupt does not reset it."""
|
||||
transcript_path: str
|
||||
quoted_answers: int = 0
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
|
||||
# The one statement of when the judge interrupts, quotes an answer or finds the chunk finished; Reply's field descriptions say only what each field holds.
|
||||
JUDGE_RULES = (
|
||||
"Decide whether the human must now be interrupted. interrupt=true when any of these appear:\n"
|
||||
"- the agent asks the human anything, seeks approval, or says a decision is the human's, and neither "
|
||||
@@ -25,7 +24,8 @@ JUDGE_RULES = (
|
||||
"wording; never construct another from facts it stated.\n"
|
||||
"Where README.md or the project's records answer the agent's own question and no [loop] line has "
|
||||
"quoted that answer yet, set interrupt=false and quote the answer; the agent is "
|
||||
"then given that passage in place of the human's reply.\n"
|
||||
"then given that passage in place of the human's reply. An interrupt outranks a quoted answer, and either "
|
||||
"outranks finished.\n"
|
||||
"The chunk is the piece of work the agent is on, shipped as one pull request. "
|
||||
"finished=true when the chunk is complete: merged, or nothing remains that the agent could do on "
|
||||
"its own. finished=false when work remains that a further run can continue: an open PR, a review "
|
||||
@@ -37,8 +37,8 @@ JUDGE_RULES = (
|
||||
class Reply:
|
||||
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the project's records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."})
|
||||
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."})
|
||||
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished. Read only when interrupt is false and answer is empty."})
|
||||
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted. True outranks answer and finished."})
|
||||
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."})
|
||||
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."})
|
||||
question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the agent's question that the answer field answers. Empty otherwise."})
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from loop_claude.claude_code.entries import assistant_parts, ends_stream, hit_limit, limit_said, parse_entry, rejected_until
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.retry import Limited
|
||||
from loop_claude.claude_code.transcript import part_line
|
||||
from loop_claude.project.records import QUOTE_SOURCES
|
||||
from loop_claude.cycle.verdicts import QuotedAnswer
|
||||
|
||||
@@ -33,8 +33,8 @@ Every claude loop-claude starts runs the project's model, saved per <key> in
|
||||
run, is asked for one. LOOP_CLAUDE_JUDGE_MODEL overrides it for the judge, LOOP_CLAUDE_WORKER_MODEL
|
||||
for the worker and the interrupt; with both set, nothing is asked.
|
||||
|
||||
A change to bin/loop-claude, bin/file_io/**/*.py or bin/loop_claude/**/*.py takes effect at the next
|
||||
cycle boundary, once loop-claude --help succeeds: the loop reloads into the edited script.
|
||||
A change to bin/loop-claude or any bin/**/*.py takes effect at the next cycle boundary, once
|
||||
loop-claude --help succeeds: the loop reloads into the edited script.
|
||||
|
||||
Exit 1, saying why, when:
|
||||
- no task is given
|
||||
|
||||
@@ -8,7 +8,6 @@ import typing
|
||||
from loop_claude.effects import Effects
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
PACKAGES = ("file_io", "loop_claude")
|
||||
RELOAD_FLAG = "--reloaded"
|
||||
T = typing.TypeVar("T")
|
||||
|
||||
@@ -37,7 +36,7 @@ def decode_handed(handed: str, kind: type[T]) -> T | None:
|
||||
|
||||
def fingerprint(launcher: pathlib.Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in [launcher, *sorted(path for package in PACKAGES for path in (launcher.parent / package).rglob("*.py"))]:
|
||||
for path in [launcher, *sorted(launcher.parent.rglob("*.py"))]:
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError as failure:
|
||||
|
||||
@@ -6,10 +6,10 @@ import typing
|
||||
|
||||
from loop_claude.claude_code.model_catalog import catalog_models
|
||||
from loop_claude.effects import Effects
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.stop import needs_human, needs_terminal, Stop
|
||||
from loop_claude.paths import Paths
|
||||
from file_io.writes import replace_json
|
||||
from files.writes import replace_json
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from loop_claude.project.checkout import read_checkout_file
|
||||
from file_io.markdown import sections
|
||||
from files.markdown import sections
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -57,9 +57,9 @@ import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
from file_io.failure import Failure
|
||||
from file_io.markdown import FENCE, sections
|
||||
from file_io.writes import replace_file
|
||||
from failure import Failure
|
||||
from files.markdown import FENCE, sections
|
||||
from files.writes import replace_file
|
||||
|
||||
EXEMPTIONS = ("", "decision", "defect", "principle", "question", "security")
|
||||
ITEMS_HEADER = "| ID | Release | Exempt | Item | R | S | A | G | Goals | Score |"
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from failure import Failure
|
||||
from files import markdown, writes
|
||||
|
||||
|
||||
class Sections(unittest.TestCase):
|
||||
def test_keeps_every_headed_section_with_its_bodies(self):
|
||||
text = "# Title\n\nintro\n\n## Goals\n\n1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority\n\n## Empty\n\n## Audience\nme\n## Audience\nagain\n# Appendix\nnot audience\n"
|
||||
self.assertEqual(markdown.sections(text), {"Goals": ["1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority"], "Empty": [""], "Audience": ["me", "again"]})
|
||||
|
||||
def test_closes_a_fence_only_on_its_own_marker(self):
|
||||
text = "## Goals\n````md\n~~~\n## Quoted\n```\n````\n ~~~\n## Also quoted\n ~~~\n1. one\n## Audience\nme\n"
|
||||
self.assertEqual(list(markdown.sections(text)), ["Goals", "Audience"])
|
||||
self.assertEqual(list(markdown.sections("```\nquoted:\n```python\n## Inside\n```\n## Goals\n1. one\n")), ["Goals"])
|
||||
|
||||
|
||||
class ReplaceFile(unittest.TestCase):
|
||||
def test_removes_what_it_staged_when_the_replace_fails(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
os.mkdir(os.path.join(root, "in-the-way"))
|
||||
self.assertEqual(writes.replace_file(pathlib.Path(root, "in-the-way"), b"data"), Failure("Is a directory"))
|
||||
self.assertEqual(os.listdir(root), ["in-the-way"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+65
-23
@@ -20,11 +20,10 @@ from loop_claude.cycle import archive, interrupt, judge, retry, run, states, wor
|
||||
from loop_claude.project import checkout, model_choice, records
|
||||
from loop_claude.effects import Effects
|
||||
from loop_claude.claude_code import entries, session, transcript, trust
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, verdict_from
|
||||
from loop_claude.cycle.limited import Limited
|
||||
from loop_claude.cycle.retry import Limited
|
||||
from loop_claude.loop import main, reload
|
||||
from file_io import markdown, writes
|
||||
from loop_claude import paths
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
@@ -157,6 +156,12 @@ class Transitions(unittest.TestCase):
|
||||
states.transition(state, event)
|
||||
|
||||
|
||||
class QuestionFor(unittest.TestCase):
|
||||
def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self):
|
||||
self.assertEqual(run.question_for(Interrupt("e", "Which one?")), "Which one?")
|
||||
self.assertEqual(run.question_for(QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.")
|
||||
|
||||
|
||||
class Cycles(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temp = tempfile.TemporaryDirectory()
|
||||
@@ -395,17 +400,6 @@ class Records(unittest.TestCase):
|
||||
self.assertEqual(records.read_records(root), Stop(1, f"AGENTS.md in {root} is not UTF-8 on line 2 - the judge cannot read the project's records."))
|
||||
|
||||
|
||||
class Sections(unittest.TestCase):
|
||||
def test_keeps_every_headed_section_with_its_bodies(self):
|
||||
text = "# Title\n\nintro\n\n## Goals\n\n1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority\n\n## Empty\n\n## Audience\nme\n## Audience\nagain\n# Appendix\nnot audience\n"
|
||||
self.assertEqual(markdown.sections(text), {"Goals": ["1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority"], "Empty": [""], "Audience": ["me", "again"]})
|
||||
|
||||
def test_closes_a_fence_only_on_its_own_marker(self):
|
||||
text = "## Goals\n````md\n~~~\n## Quoted\n```\n````\n ~~~\n## Also quoted\n ~~~\n1. one\n## Audience\nme\n"
|
||||
self.assertEqual(list(markdown.sections(text)), ["Goals", "Audience"])
|
||||
self.assertEqual(list(markdown.sections("```\nquoted:\n```python\n## Inside\n```\n## Goals\n1. one\n")), ["Goals"])
|
||||
|
||||
|
||||
class Render(unittest.TestCase):
|
||||
def test_joins_assistant_text_tool_calls_human_turns_and_loop_prompts_and_skips_the_rest(self):
|
||||
said = transcript.render([
|
||||
@@ -422,6 +416,20 @@ class Render(unittest.TestCase):
|
||||
self.assertEqual(said, '[human] do it\n\nStarting.\n\n[tool Bash] {"command": "ls"}\n\n[human] yes, merge it\n\n[human] B, while you work\n\n[loop] Continue the task; the previous run stopped before it was finished.')
|
||||
|
||||
|
||||
class Archive(unittest.TestCase):
|
||||
def test_writes_afresh_a_copy_whose_outcomes_file_is_missing(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
kept = paths.paths_from(pathlib.Path(root), {})
|
||||
transcript = pathlib.Path(root, "t.jsonl")
|
||||
transcript.write_text('{"timestamp": "2026-10-03T00:00:00Z"}\n')
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Verdict: unfinished", kept))
|
||||
(outcomes,) = kept.transcript_archive.glob("proj/*/*.outcomes.md")
|
||||
outcomes.unlink()
|
||||
self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Stopped: Ctrl-C", kept))
|
||||
self.assertEqual(outcomes.read_text(), "Task: Do it.\n\nStopped: Ctrl-C\n")
|
||||
|
||||
|
||||
class CopyFor(unittest.TestCase):
|
||||
def test_skips_a_version_already_kept_beside_its_copy(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
@@ -487,6 +495,15 @@ class MarkTrusted(unittest.TestCase):
|
||||
self.assertEqual(trust.mark_trusted({}, ["/a"]), ({"projects": {"/a": {"hasTrustDialogAccepted": True}}}, ["/a"]))
|
||||
|
||||
|
||||
class JudgePrompt(unittest.TestCase):
|
||||
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
||||
prompt, shown = judge.judge_prompt("Do it.", "## Goals", "", "old " + "x" * judge.JUDGE_PROMPT_CHARS + " new")
|
||||
self.assertTrue(shown.endswith(" new"))
|
||||
self.assertNotIn("old", shown)
|
||||
self.assertIn(shown, prompt)
|
||||
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
||||
|
||||
|
||||
class Grounded(unittest.TestCase):
|
||||
def test_every_verdict_must_quote_the_transcript(self):
|
||||
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
|
||||
@@ -549,7 +566,12 @@ class LoopRuns(unittest.TestCase):
|
||||
|
||||
class HelpLimits(unittest.TestCase):
|
||||
def test_help_states_the_limits_the_code_holds(self):
|
||||
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", main.__doc__.replace("\n", " "))
|
||||
said = " ".join(main.__doc__.split())
|
||||
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
|
||||
# The retry messages say these limits in words too.
|
||||
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
|
||||
self.assertIn("fails three times running, retried a minute apart", said)
|
||||
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
|
||||
|
||||
|
||||
class ExitContract(unittest.TestCase):
|
||||
@@ -597,19 +619,39 @@ class ExitContract(unittest.TestCase):
|
||||
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
|
||||
|
||||
|
||||
class JudgeFailures(unittest.TestCase):
|
||||
def test_every_failed_judge_run_has_its_clause_in_help(self):
|
||||
started, field, unasked = "claude cannot start or exits non-zero", "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
|
||||
# Every Failure the judge's run returns, in source order: path, function, start of reason, its --help clause.
|
||||
clauses = [
|
||||
("cycle/judge.py", "run_judge", "f'could not start claude:", started),
|
||||
("cycle/judge.py", "run_judge", "proc.stderr.strip() or f'the judge exited", started),
|
||||
("cycle/judge.py", "run_judge", 'f"{verdict.reason}; it replied:', field),
|
||||
("cycle/judge.py", "run_judge", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
|
||||
("cycle/verdicts.py", "verdict_from", "'the judge returned no reply'", field),
|
||||
("cycle/verdicts.py", "verdict_from", 'f"the judge\'s reply has no {name}', field),
|
||||
("cycle/verdicts.py", "verdict_from", "'the judge interrupted without a question'", unasked),
|
||||
("cycle/verdicts.py", "verdict_from", "'the judge quoted an answer without a question'", unasked),
|
||||
]
|
||||
root = pathlib.Path(main.__file__).parents[1]
|
||||
found = []
|
||||
for path in ("cycle/judge.py", "cycle/verdicts.py"):
|
||||
functions = [node for node in ast.walk(ast.parse((root / path).read_text())) if isinstance(node, ast.FunctionDef)]
|
||||
failures = sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Failure")
|
||||
found += [(path, function, reason) for _, function, reason in failures]
|
||||
self.assertEqual(len(found), len(clauses), found)
|
||||
for (path, function, reason), (want_path, want_function, reason_start, _) in zip(found, clauses):
|
||||
self.assertEqual((path, function), (want_path, want_function))
|
||||
self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}")
|
||||
judge_clause = " ".join(main.__doc__[main.__doc__.index("a judge when"):main.__doc__.index("A usage limit")].split())
|
||||
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
|
||||
|
||||
|
||||
class Worktrees(unittest.TestCase):
|
||||
def test_lists_the_root_alone_when_git_cannot_start(self):
|
||||
fake = FakeEffects(git=[FileNotFoundError(2, "No such file or directory")])
|
||||
self.assertEqual(checkout.worktrees("/r", fake.effects()), ["/r"])
|
||||
|
||||
|
||||
class ReplaceFile(unittest.TestCase):
|
||||
def test_removes_what_it_staged_when_the_replace_fails(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
os.mkdir(os.path.join(root, "in-the-way"))
|
||||
self.assertEqual(writes.replace_file(pathlib.Path(root, "in-the-way"), b"data"), Failure("Is a directory"))
|
||||
self.assertEqual(os.listdir(root), ["in-the-way"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -670,7 +670,7 @@ class Reload(unittest.TestCase):
|
||||
worker(append=["loop-claude", "\ndef\n"]), judge(),
|
||||
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
|
||||
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
|
||||
worker(append=["file_io/writes.py", "# edited\n"]), judge(),
|
||||
worker(append=["files/writes.py", "# edited\n"]), judge(),
|
||||
worker(),
|
||||
])
|
||||
self.assertEqual(run.code, 130)
|
||||
|
||||
@@ -7,7 +7,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from file_io.failure import Failure
|
||||
from failure import Failure
|
||||
import rank_todo
|
||||
from test_python_floor import ROOT
|
||||
|
||||
|
||||
+1
-1
@@ -11,5 +11,5 @@ class Types(unittest.TestCase):
|
||||
self.assertIsNotNone(floor, "README.md's Install names `Python 3.N or newer`")
|
||||
if shutil.which("uv") is None:
|
||||
self.fail("uv on PATH fetches mypy; see README.md's Test line")
|
||||
ran = subprocess.run(["uv", "tool", "run", "--python", "%d.%d" % floor, "mypy==2.3.1", "--cache-dir=/dev/null", "--check-untyped-defs", "--disallow-untyped-defs", "-m", "rank_todo", "-m", "statusline", "-p", "file_io", "-p", "loop_claude"], capture_output=True, cwd=ROOT / "bin", text=True)
|
||||
ran = subprocess.run(["uv", "tool", "run", "--python", "%d.%d" % floor, "mypy==2.3.1", "--cache-dir=/dev/null", "--check-untyped-defs", "--disallow-untyped-defs", "-m", "rank_todo", "-m", "statusline", "-p", "loop_claude"], capture_output=True, cwd=ROOT / "bin", text=True)
|
||||
self.assertEqual(ran.returncode, 0, ran.stdout + ran.stderr)
|
||||
|
||||
@@ -67,7 +67,7 @@ Give a reviewer without Bash the branch diff and `git ls-files`'s output in its
|
||||
**Findings.** A reviewer rates each finding 🔴 must fix, 🟡 should fix, 🟢 nit, ❓ question, or 🔀 escalation naming the reviewer whose concern it is.
|
||||
|
||||
- **Outside the chunk.** A finding outside the chunk's change, whatever its pass or severity, is scored and filed in `todo.md` per `~/.claude/CLAUDE.md` → "Todo items". A 🔴 is marked `defect`. A 🟡 against a file under `~/.claude/principles/` is marked `principle`, its detail naming the principle, where no other reason fits. One under the bar and not exempt is named in the summary with its score. A verified security HIGH is handled per "Security".
|
||||
- **Goal or decision.** A fix that would change a goal, or a decision the repo held before the chunk began, is asked; no user reachable → filed, marked `question`, the goal or decision left as is.
|
||||
- **Goal or decision.** A fix that would change a goal, or a standing decision, is asked; no user reachable → filed, marked `question`, the goal or decision left as is.
|
||||
- **🔴.** A 🔴 is fixed, or shown factually wrong, and what disproved it recorded per `~/.claude/CLAUDE.md` → "Decisions: one home, each linked to a goal or a technical principle": in a comment at the code the finding concerns, or in `docs/decisions.md` when it concerns more than one place. Never declined.
|
||||
- **🟡.** A 🟡 is fixed, shown factually wrong like a 🔴, or declined. Decline one only where the reason is sound, consistent with the goals and the technical principles, and confidence is very high; the decline and its reason are recorded the same way. A reason neither speaks to is a gap in the goals: fix the finding, or ask per `~/.claude/CLAUDE.md` → "Working loop", "Ask, don't guess"; no user reachable → fix. Never drop one silently, and never decline one for convenience or the size of the fix.
|
||||
- **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a standing decision settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
`Bar = 9`
|
||||
|
||||
`Next ID = 46`
|
||||
`Next ID = 53`
|
||||
|
||||
| Goal | W |
|
||||
|---|---|
|
||||
@@ -25,13 +25,20 @@
|
||||
| ID | Release | Exempt | Item | R | S | A | G | Goals | Score |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| 1 | 0.1.0 | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 3 | 5 | 8 | 9 | 2, 3 | 27.8 |
|
||||
| 47 | 0.1.0 | | **Archive the judge's evidence with each verdict, so a wrong finish shows what it rested on.** | 1 | 1 | 6 | 6 | 2, 4 | 21.4 |
|
||||
| 46 | 0.1.0 | question | **Let the judge read and quote `~/.claude/CLAUDE.md` beside the project's records.** | 4 | 3 | 7 | 8 | 4, 4.2 | 20.0 |
|
||||
| 48 | 0.1.0 | | **Stamp each archived outcome with its UTC time.** | 1 | 1 | 4 | 4 | 2 | 13.9 |
|
||||
| 23 | 0.1.0 | | **Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.** | 3 | 4 | 7 | 8 | 4.4 | 13.5 |
|
||||
| 21 | 0.1.0 | | **Give `settings.json`'s model one writer, and sort its keys.** | 3 | 3 | 6 | 7 | 4.3 | 12.9 |
|
||||
| 49 | 0.1.0 | principle | **Check that `AGENTS.md` lists exactly the titles of `docs/decisions.md`.** | 1 | 2 | 5 | 6 | 4.3 | 12.5 |
|
||||
| 12 | 0.1.0 | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 2 | 4 | 4 | 6 | 4.1 | 11.7 |
|
||||
| 5 | 0.1.0 | decision | **Key a project on its checkout's full path, and refuse to start while another session uses that key.** | 5 | 5 | 5 | 6 | 4.1 | 10.5 |
|
||||
| 4 | 0.1.0 | | **Name each chunk's archive folder by the chunk's start time, so `<key>/` lists chunks in the order they ran.** | 3 | 2 | 4 | 3 | 2 | 9.8 |
|
||||
| 50 | 0.1.0 | principle | **Grant the bench's runs only the tools a review needs, in place of skipping every permission.** | 2 | 2 | 3 | 4 | 3 | 9.7 |
|
||||
| 29 | 0.1.0 | | **Make the contributor a standing member of every project's audience, unless its `## Audience` says maintainability does not matter.** | 2 | 2 | 3 | 5 | 4.1, 4.4 | 9.1 |
|
||||
| 52 | 0.1.0 | principle | **Record why this checkout lives in Claude Code's own directory, or move it out and link it in.** | 6 | 6 | 3 | 5 | 1 | 8.5 |
|
||||
| 6 | 0.1.0 | decision | **Move what `loop-claude` writes out of `~/.claude.json`, and ask the human about each write that must stay.** | 5 | 5 | 4 | 5 | 4.1 | 7.4 |
|
||||
| 51 | 0.1.0 | principle | **Hand loop-claude's terminal output to an injected effect, so no storage or retry function prints.** | 3 | 5 | 3 | 5 | 4.1 | 7.4 |
|
||||
|
||||
## Details
|
||||
|
||||
@@ -41,6 +48,18 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost
|
||||
|
||||
2026-10-02, the human: ai-sandbox is the VM `loop-claude` will run on, hosted on the server bender. bender-infra, the infrastructure-as-code project that manages bender, will back up ai-sandbox's data volume once ai-sandbox is rebuilt. The item waits for the human to say the rebuild has landed. Before writing the README line, check that `~/.claude` lives on that volume and that the backup is private and encrypted at rest.
|
||||
|
||||
### 47. Archive the judge's evidence with each verdict, so a wrong finish shows what it rested on.
|
||||
|
||||
The judge runs without session persistence, and `close` and `ask` archive only the verdict and the question. A wrong `finished` deletes the session, and nothing then shows the passage the judge quoted.
|
||||
|
||||
### 46. Let the judge read and quote `~/.claude/CLAUDE.md` beside the project's records.
|
||||
|
||||
The goals audit on PR #208 rated it 🔴 against goals 4 and 4.2: the worker follows `CLAUDE.md`, but the judge sees only the premises, `AGENTS.md` and `docs/decisions.md`. A choice `CLAUDE.md` settles therefore reads to the judge as one nothing discriminates, and a question it answers goes to the human. The fix revises the decision "loop-claude: a judge decides each interrupt and each quoted answer from the transcript, the README's premises, `AGENTS.md` and `docs/decisions.md`", so the human decides it.
|
||||
|
||||
### 48. Stamp each archived outcome with its UTC time.
|
||||
|
||||
After an overnight exit 1, `<sha>.outcomes.md` holds each outcome without saying when it happened.
|
||||
|
||||
### 23. Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.
|
||||
|
||||
On this repo the frontend, REST and logging auditors each read the whole project to report "does not apply", and one 🟢 fix reruns ten whole-project reads.
|
||||
@@ -51,6 +70,10 @@ Claude Code rewrites `"model"` in this tracked file whenever a model is picked,
|
||||
|
||||
Every write Claude Code makes to the file rewrites `"model"`, `claude plugin install` included: Claude Code 2.1.288 turned `"opus"` into `"opus[1m]"`. Each write also puts the keys back in Claude Code's own order, so a sorted file stays sorted only until the next write.
|
||||
|
||||
### 49. Check that `AGENTS.md` lists exactly the titles of `docs/decisions.md`.
|
||||
|
||||
DRY, in `principles/technical-principles.md`: the list is copied by hand, and sessions work from the copy.
|
||||
|
||||
### 12. Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.
|
||||
|
||||
The repo has no CI, so larv-review's gate is the local run, and larv-review's Gate skips even that when the last stability round left the suite green: an unattended chunk can merge on the worker's own word.
|
||||
@@ -63,10 +86,22 @@ The key names the project's session, archive folder and models entry. `project_k
|
||||
|
||||
Today the folder is the SHA-256 of the transcript's first timestamped line, which tells the human nothing; that line carries the start time. Drop the fallback to the first line, which every chunk of a project shares, and give the tests' fake transcripts ISO timestamps.
|
||||
|
||||
### 50. Grant the bench's runs only the tools a review needs, in place of skipping every permission.
|
||||
|
||||
"Always grant explicit access to data", in `principles/technical-principles.md`: `workflow-bench/bench.py` starts claude with `--dangerously-skip-permissions`, and no rule confines it to a sandbox, so a run on the host reaches the host's credentials and other checkouts.
|
||||
|
||||
### 29. Make the contributor a standing member of every project's audience, unless its `## Audience` says maintainability does not matter.
|
||||
|
||||
State that in `CLAUDE.md`'s goals-and-audience rule, and have larv-review's comprehension floor name the contributor as the reader it serves. The floor then gates every chunk while the contributor stands, and a project that drops them skips it.
|
||||
|
||||
### 52. Record why this checkout lives in Claude Code's own directory, or move it out and link it in.
|
||||
|
||||
"One owner per value" and "Prefer the vendor-neutral option", in `principles/technical-principles.md`: git, Claude Code and loop-claude all write `~/.claude`, kept apart only by the `.gitignore` allowlist, and the repo's records live in a vendor directory.
|
||||
|
||||
### 6. Move what `loop-claude` writes out of `~/.claude.json`, and ask the human about each write that must stay.
|
||||
|
||||
Claude Code owns that file and writes it. `trust` reads it, changes it and writes it back, so a write that lands in between is lost.
|
||||
|
||||
### 51. Hand loop-claude's terminal output to an injected effect, so no storage or retry function prints.
|
||||
|
||||
"Functional core, imperative shell", in `principles/technical-principles.md`: `archive`, `trust`, `retried`, the worker, `interrupt` and `model_choice` print to stderr themselves, and the tests swap the global stream.
|
||||
|
||||
+13
-1
@@ -1,5 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run larv-review bench runs: BENCH_SOURCES=<dir of the <case>-main clones, their <case>-main.md reports and comparison.md> bench.py <case>-<version>-<n> ..."""
|
||||
"""Run larv-review bench runs: BENCH_SOURCES=<dir> bench.py <case>-<version>-<n> ...
|
||||
|
||||
The 2026-09-30 bench had two versions of larv-review, main and rebuild, each review four real chunks,
|
||||
the cases, three times; compare.py then rules which version reviews better. Its inputs are untracked:
|
||||
- versions/<version>/ beside this script: a copy of ~/.claude at that version, which each run reads
|
||||
in place of ~/.claude (run-prompt.md).
|
||||
- $BENCH_SOURCES/<case>-main/: a clone of the case's project; CASES names the chunk's base and head
|
||||
commits in it.
|
||||
- $BENCH_SOURCES/<case>-main.md: main's own review of the case, run on 2026-09-29.
|
||||
- $BENCH_SOURCES/comparison.md: what each case's real review found, one table row per issue, its ID
|
||||
led by the case's letter in compare.py's PREFIX.
|
||||
Each run leaves its clone in <run>/, its report in <run>.md and its claude stream in logs/.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
|
||||
@@ -29,6 +29,7 @@ def runs(case, version):
|
||||
|
||||
|
||||
def blind(text):
|
||||
"""The report with every name that tells which version wrote it replaced."""
|
||||
return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user