diff --git a/.gitignore b/.gitignore index 9d5124af..09d4752e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,9 @@ !agents/ !agents/*.md !bin/ -!bin/file_io/ -!bin/file_io/*.py +!bin/failure.py +!bin/files/ +!bin/files/*.py !bin/loop-claude !bin/loop_claude/ !bin/loop_claude/**/ diff --git a/bin/file_io/failure.py b/bin/failure.py similarity index 100% rename from bin/file_io/failure.py rename to bin/failure.py diff --git a/bin/file_io/__init__.py b/bin/files/__init__.py similarity index 100% rename from bin/file_io/__init__.py rename to bin/files/__init__.py diff --git a/bin/file_io/markdown.py b/bin/files/markdown.py similarity index 100% rename from bin/file_io/markdown.py rename to bin/files/markdown.py diff --git a/bin/file_io/writes.py b/bin/files/writes.py similarity index 95% rename from bin/file_io/writes.py rename to bin/files/writes.py index 3ab80661..0a799cf7 100644 --- a/bin/file_io/writes.py +++ b/bin/files/writes.py @@ -3,7 +3,7 @@ import os import pathlib import typing -from file_io.failure import Failure +from failure import Failure def make_private(folder: pathlib.Path) -> None: diff --git a/bin/loop_claude/claude_code/session.py b/bin/loop_claude/claude_code/session.py index c1f965d5..8496af13 100644 --- a/bin/loop_claude/claude_code/session.py +++ b/bin/loop_claude/claude_code/session.py @@ -5,7 +5,7 @@ import pathlib import shutil import uuid -from file_io.failure import Failure +from failure import Failure from loop_claude.paths import Paths NAMESPACE = uuid.UUID("64fb586b-d926-4bb3-a44c-c903e86d3eec") diff --git a/bin/loop_claude/claude_code/transcript.py b/bin/loop_claude/claude_code/transcript.py index f30efb17..0eef90b3 100644 --- a/bin/loop_claude/claude_code/transcript.py +++ b/bin/loop_claude/claude_code/transcript.py @@ -1,7 +1,7 @@ import json from loop_claude.claude_code.entries import assistant_parts, human_text, loop_text, parse_entry, part_text, tool_call -from file_io.failure import Failure +from failure import Failure JUDGED_TOOL_INPUT = 400 diff --git a/bin/loop_claude/claude_code/trust.py b/bin/loop_claude/claude_code/trust.py index ee8ec4a9..854d7d6c 100644 --- a/bin/loop_claude/claude_code/trust.py +++ b/bin/loop_claude/claude_code/trust.py @@ -2,8 +2,8 @@ import json import pathlib import sys -from file_io.failure import Failure -from file_io.writes import replace_json +from failure import Failure +from files.writes import replace_json # A failure only warns: the interrupt still runs, and the human answers Claude Code's trust dialog. diff --git a/bin/loop_claude/cycle/archive.py b/bin/loop_claude/cycle/archive.py index c9c6f3f4..160691b0 100644 --- a/bin/loop_claude/cycle/archive.py +++ b/bin/loop_claude/cycle/archive.py @@ -3,9 +3,9 @@ import os import pathlib import sys -from file_io.failure import Failure +from failure import Failure from loop_claude.claude_code.transcript import first_timestamped_line -from file_io.writes import make_private, replace_file +from files.writes import make_private, replace_file from loop_claude.paths import Paths from loop_claude.claude_code.session import session_files @@ -24,7 +24,8 @@ def archive(project_key: str, path: str, task: str, outcome: str, paths: Paths) return unkept copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl" outcomes = copy.with_suffix(".outcomes.md") - if copy.exists(): + # A copy whose outcomes file is missing is written afresh, outcomes first. + if copy.exists() and outcomes.exists(): return add_outcome(outcomes, outcome) # The copy lands last, so a copy on disk always has its outcomes and its session's files beside it. for dest, data in ((outcomes, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)): diff --git a/bin/loop_claude/cycle/interrupt.py b/bin/loop_claude/cycle/interrupt.py index b92f25ea..c61883cf 100644 --- a/bin/loop_claude/cycle/interrupt.py +++ b/bin/loop_claude/cycle/interrupt.py @@ -2,7 +2,7 @@ import sys import uuid from loop_claude.effects import Effects -from file_io.failure import Failure +from failure import Failure from loop_claude.claude_code.transcript import human_turns, read_transcript from loop_claude.stop import needs_human, needs_terminal, Stop from loop_claude.paths import Paths diff --git a/bin/loop_claude/cycle/judge.py b/bin/loop_claude/cycle/judge.py index 4dfe32a6..cd2e7fa9 100644 --- a/bin/loop_claude/cycle/judge.py +++ b/bin/loop_claude/cycle/judge.py @@ -2,11 +2,10 @@ import json import re from loop_claude.effects import Effects -from file_io.failure import Failure +from failure import Failure from loop_claude.claude_code.entries import hit_limit, limit_said, parse_entry, structured_output from loop_claude.cycle.verdicts import Interrupt, JUDGE_RULES, JUDGE_SCHEMA, QuotedAnswer, Verdict, verdict_from -from loop_claude.cycle.limited import Limited -from loop_claude.cycle.retry import retried +from loop_claude.cycle.retry import Limited, retried from loop_claude.stop import Stop # At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03. @@ -25,11 +24,11 @@ def judge(root: str, task: str, premises: str, records: str, rendered_transcript "--permission-mode", "bypassPermissions", "--tools", "", ] - prompt = judge_prompt(task, premises, records, rendered_transcript) - return retried("judge", lambda: run_judge(root, args, prompt, rendered_transcript, f"{premises}\n\n{records}", effects), effects) + prompt, shown = judge_prompt(task, premises, records, rendered_transcript) + return retried("judge", lambda: run_judge(root, args, prompt, shown, f"{premises}\n\n{records}", effects), effects) -def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited: +def run_judge(root: str, args: list[str], prompt: str, shown: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited: try: proc = effects.claude(args, root, prompt) except OSError as failure: @@ -43,7 +42,7 @@ def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str, verdict = verdict_from(structured_output(reply)) if isinstance(verdict, Failure): return Failure(f"{verdict.reason}; it replied: {proc.stdout.strip() or 'nothing'}") - if not grounded(verdict.evidence, rendered_transcript): + if not grounded(verdict.evidence, shown): return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the transcript" + (f", for the question: {verdict.question}" if isinstance(verdict, Interrupt | QuotedAnswer) else "")) # Not a Failure: a misquote would otherwise exit an unattended loop. Never elided: stitched fragments can say what the records do not. if isinstance(verdict, QuotedAnswer) and not verbatim(verdict.answer, quotable): @@ -51,7 +50,8 @@ def run_judge(root: str, args: list[str], prompt: str, rendered_transcript: str, return verdict -def judge_prompt(task: str, premises: str, records: str, rendered_transcript: str) -> str: +def judge_prompt(task: str, premises: str, records: str, rendered_transcript: str) -> tuple[str, str]: + """The prompt, and the part of the transcript it shows, which the judge's evidence must quote.""" head = ( f"From README.md:\n{premises}\n\n" f"The project's records, its working rules and decisions:\n{records or 'none'}\n\n" @@ -60,7 +60,8 @@ def judge_prompt(task: str, premises: str, records: str, rendered_transcript: st ) room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2) # The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle. - return f"{head}{rendered_transcript[max(0, len(rendered_transcript) - room):]}\n\n{JUDGE_RULES}" + shown = rendered_transcript[max(0, len(rendered_transcript) - room):] + return f"{head}{shown}\n\n{JUDGE_RULES}", shown def grounded(evidence: str, rendered_transcript: str) -> bool: diff --git a/bin/loop_claude/cycle/limited.py b/bin/loop_claude/cycle/limited.py deleted file mode 100644 index 5ed69bc6..00000000 --- a/bin/loop_claude/cycle/limited.py +++ /dev/null @@ -1,8 +0,0 @@ -import dataclasses - - -@dataclasses.dataclass(frozen=True) -class Limited: - """claude hit a usage limit; said is claude's message about it, and resets_at is when the limit resets, in seconds since the epoch, or None when unknown.""" - resets_at: int | None - said: str diff --git a/bin/loop_claude/cycle/retry.py b/bin/loop_claude/cycle/retry.py index 8e35edad..bb9a4215 100644 --- a/bin/loop_claude/cycle/retry.py +++ b/bin/loop_claude/cycle/retry.py @@ -1,17 +1,26 @@ +import dataclasses import sys import typing from loop_claude.effects import Effects -from file_io.failure import Failure -from loop_claude.cycle.limited import Limited +from failure import Failure from loop_claude.stop import Stop T = typing.TypeVar("T") +FAILURES_RUNNING = 3 PAST_RESET = 120 +RETRY_DELAY = 60 UNKNOWN_RESET_POLL = 600 +@dataclasses.dataclass(frozen=True) +class Limited: + """claude hit a usage limit; said is claude's message about it, and resets_at is when the limit resets, in seconds since the epoch, or None when unknown.""" + resets_at: int | None + said: str + + def retried(who: str, attempt: typing.Callable[[], T | Failure | Limited], effects: Effects) -> T | Stop: failures = 0 while True: @@ -27,10 +36,10 @@ def retried(who: str, attempt: typing.Callable[[], T | Failure | Limited], effec if not isinstance(result, Failure): return result failures += 1 - if failures == 3: + if failures == FAILURES_RUNNING: return Stop(1, f"the {who} failed three times running: {result.reason}") print(f"retrying the {who} in a minute: {result.reason}", file=sys.stderr) - effects.sleep(60) + effects.sleep(RETRY_DELAY) def limit_wait(limited: Limited, now: float) -> float: diff --git a/bin/loop_claude/cycle/run.py b/bin/loop_claude/cycle/run.py index 4dc4cf21..79173f30 100644 --- a/bin/loop_claude/cycle/run.py +++ b/bin/loop_claude/cycle/run.py @@ -1,11 +1,11 @@ import dataclasses import typing -from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanAnswered, Interrupting, Judging, Ruled, State, transition, WorkerRan, Working +from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanAnswered, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, transition, WorkerRan, Working from loop_claude.cycle.archive import archive from loop_claude.claude_code.session import delete_session, Session, session_open, transcript_file from loop_claude.effects import Effects -from file_io.failure import Failure +from failure import Failure from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished from loop_claude.cycle.interrupt import interrupt from loop_claude.cycle.judge import judge @@ -57,8 +57,8 @@ def step(cycle: Cycle, state: Active) -> Event | Stop: return work(cycle, answered) case Judging(transcript_path, _): return rule(cycle, transcript_path) - case Interrupting(transcript_path, ruling, quoted_answers): - return ask(cycle, transcript_path, ruling, quoted_answers) + case Interrupting(transcript_path, ruling, _): + return ask(cycle, transcript_path, ruling) case Closing(transcript_path, verdict): return close(cycle, transcript_path, verdict) case _: @@ -83,14 +83,20 @@ def rule(cycle: Cycle, transcript_path: str) -> Ruled | Stop: return verdict if isinstance(verdict, Stop) else Ruled(verdict) -def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer, quoted_answers: int) -> HumanAnswered | Stop: - question = ruling.question if isinstance(ruling, Interrupt) else f"{ruling.question}\n\n{QUOTE_SOURCES} says:\n{ruling.answer}\n\nThose files have answered {quoted_answers} questions this cycle, the most allowed, so this one comes to you." +def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer) -> HumanAnswered | Stop: + question = question_for(ruling) # Archived now, so the transcript the question was asked on is kept with it; the human's answer then grows the transcript. if failure := archive(cycle.project_key, transcript_path, cycle.task, f"Verdict: interrupt\n\n{question}", cycle.paths): return Stop(1, failure.reason) return interrupt(cycle.root, cycle.session.id, cycle.models.worker, transcript_path, question, cycle.paths, cycle.effects) or HumanAnswered() +def question_for(ruling: Interrupt | QuotedAnswer) -> str: + if isinstance(ruling, Interrupt): + return ruling.question + return f"{ruling.question}\n\n{QUOTE_SOURCES} says:\n{ruling.answer}\n\nThose files have answered {QUOTED_ANSWERS_PER_CYCLE} questions this cycle, the most allowed, so this one comes to you." + + def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) -> Archived | Stop: finished = isinstance(verdict, Finished) if failure := archive(cycle.project_key, transcript_path, cycle.task, f"Verdict: {'finished' if finished else 'unfinished'}", cycle.paths): diff --git a/bin/loop_claude/cycle/states.py b/bin/loop_claude/cycle/states.py index 92808e41..85896989 100644 --- a/bin/loop_claude/cycle/states.py +++ b/bin/loop_claude/cycle/states.py @@ -15,7 +15,7 @@ class Working: @dataclasses.dataclass(frozen=True) class Judging: - """The judge rules; quoted_answers counts the questions README.md or the records answered this cycle.""" + """The judge rules; quoted_answers counts the questions README.md or the records answered this cycle, and an interrupt does not reset it.""" transcript_path: str quoted_answers: int = 0 diff --git a/bin/loop_claude/cycle/verdicts.py b/bin/loop_claude/cycle/verdicts.py index e51dddb0..fd023bf7 100644 --- a/bin/loop_claude/cycle/verdicts.py +++ b/bin/loop_claude/cycle/verdicts.py @@ -1,9 +1,8 @@ import dataclasses import typing -from file_io.failure import Failure +from failure import Failure -# The one statement of when the judge interrupts, quotes an answer or finds the chunk finished; Reply's field descriptions say only what each field holds. JUDGE_RULES = ( "Decide whether the human must now be interrupted. interrupt=true when any of these appear:\n" "- the agent asks the human anything, seeks approval, or says a decision is the human's, and neither " @@ -25,7 +24,8 @@ JUDGE_RULES = ( "wording; never construct another from facts it stated.\n" "Where README.md or the project's records answer the agent's own question and no [loop] line has " "quoted that answer yet, set interrupt=false and quote the answer; the agent is " - "then given that passage in place of the human's reply.\n" + "then given that passage in place of the human's reply. An interrupt outranks a quoted answer, and either " + "outranks finished.\n" "The chunk is the piece of work the agent is on, shipped as one pull request. " "finished=true when the chunk is complete: merged, or nothing remains that the agent could do on " "its own. finished=false when work remains that a further run can continue: an open PR, a review " @@ -37,8 +37,8 @@ JUDGE_RULES = ( class Reply: answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the project's records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."}) evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."}) - finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished. Read only when interrupt is false and answer is empty."}) - interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted. True outranks answer and finished."}) + finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."}) + interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."}) question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the agent's question that the answer field answers. Empty otherwise."}) diff --git a/bin/loop_claude/cycle/worker.py b/bin/loop_claude/cycle/worker.py index 243138e3..65df05ec 100644 --- a/bin/loop_claude/cycle/worker.py +++ b/bin/loop_claude/cycle/worker.py @@ -1,7 +1,7 @@ from loop_claude.claude_code.entries import assistant_parts, ends_stream, hit_limit, limit_said, parse_entry, rejected_until from loop_claude.effects import Effects -from file_io.failure import Failure -from loop_claude.cycle.limited import Limited +from failure import Failure +from loop_claude.cycle.retry import Limited from loop_claude.claude_code.transcript import part_line from loop_claude.project.records import QUOTE_SOURCES from loop_claude.cycle.verdicts import QuotedAnswer diff --git a/bin/loop_claude/loop/main.py b/bin/loop_claude/loop/main.py index f2092761..70b91729 100644 --- a/bin/loop_claude/loop/main.py +++ b/bin/loop_claude/loop/main.py @@ -33,8 +33,8 @@ Every claude loop-claude starts runs the project's model, saved per in run, is asked for one. LOOP_CLAUDE_JUDGE_MODEL overrides it for the judge, LOOP_CLAUDE_WORKER_MODEL for the worker and the interrupt; with both set, nothing is asked. -A change to bin/loop-claude, bin/file_io/**/*.py or bin/loop_claude/**/*.py takes effect at the next -cycle boundary, once loop-claude --help succeeds: the loop reloads into the edited script. +A change to bin/loop-claude or any bin/**/*.py takes effect at the next cycle boundary, once +loop-claude --help succeeds: the loop reloads into the edited script. Exit 1, saying why, when: - no task is given diff --git a/bin/loop_claude/loop/reload.py b/bin/loop_claude/loop/reload.py index 5bdb987e..22acc436 100644 --- a/bin/loop_claude/loop/reload.py +++ b/bin/loop_claude/loop/reload.py @@ -8,7 +8,6 @@ import typing from loop_claude.effects import Effects from loop_claude.stop import Stop -PACKAGES = ("file_io", "loop_claude") RELOAD_FLAG = "--reloaded" T = typing.TypeVar("T") @@ -37,7 +36,7 @@ def decode_handed(handed: str, kind: type[T]) -> T | None: def fingerprint(launcher: pathlib.Path) -> str: digest = hashlib.sha256() - for path in [launcher, *sorted(path for package in PACKAGES for path in (launcher.parent / package).rglob("*.py"))]: + for path in [launcher, *sorted(launcher.parent.rglob("*.py"))]: try: content = path.read_bytes() except OSError as failure: diff --git a/bin/loop_claude/project/model_choice.py b/bin/loop_claude/project/model_choice.py index 03fdf0a5..0b3932fa 100644 --- a/bin/loop_claude/project/model_choice.py +++ b/bin/loop_claude/project/model_choice.py @@ -6,10 +6,10 @@ import typing from loop_claude.claude_code.model_catalog import catalog_models from loop_claude.effects import Effects -from file_io.failure import Failure +from failure import Failure from loop_claude.stop import needs_human, needs_terminal, Stop from loop_claude.paths import Paths -from file_io.writes import replace_json +from files.writes import replace_json @dataclasses.dataclass(frozen=True) diff --git a/bin/loop_claude/project/premises.py b/bin/loop_claude/project/premises.py index c3b02ca6..f88a8542 100644 --- a/bin/loop_claude/project/premises.py +++ b/bin/loop_claude/project/premises.py @@ -1,5 +1,5 @@ from loop_claude.project.checkout import read_checkout_file -from file_io.markdown import sections +from files.markdown import sections from loop_claude.stop import Stop diff --git a/bin/rank_todo.py b/bin/rank_todo.py index 42178fd5..ca992238 100755 --- a/bin/rank_todo.py +++ b/bin/rank_todo.py @@ -57,9 +57,9 @@ import pathlib import re import sys -from file_io.failure import Failure -from file_io.markdown import FENCE, sections -from file_io.writes import replace_file +from failure import Failure +from files.markdown import FENCE, sections +from files.writes import replace_file EXEMPTIONS = ("", "decision", "defect", "principle", "question", "security") ITEMS_HEADER = "| ID | Release | Exempt | Item | R | S | A | G | Goals | Score |" diff --git a/bin/test_files.py b/bin/test_files.py new file mode 100644 index 00000000..b6ad243c --- /dev/null +++ b/bin/test_files.py @@ -0,0 +1,33 @@ +import os +import pathlib +import sys +import tempfile +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from failure import Failure +from files import markdown, writes + + +class Sections(unittest.TestCase): + def test_keeps_every_headed_section_with_its_bodies(self): + text = "# Title\n\nintro\n\n## Goals\n\n1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority\n\n## Empty\n\n## Audience\nme\n## Audience\nagain\n# Appendix\nnot audience\n" + self.assertEqual(markdown.sections(text), {"Goals": ["1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority"], "Empty": [""], "Audience": ["me", "again"]}) + + def test_closes_a_fence_only_on_its_own_marker(self): + text = "## Goals\n````md\n~~~\n## Quoted\n```\n````\n ~~~\n## Also quoted\n ~~~\n1. one\n## Audience\nme\n" + self.assertEqual(list(markdown.sections(text)), ["Goals", "Audience"]) + self.assertEqual(list(markdown.sections("```\nquoted:\n```python\n## Inside\n```\n## Goals\n1. one\n")), ["Goals"]) + + +class ReplaceFile(unittest.TestCase): + def test_removes_what_it_staged_when_the_replace_fails(self): + with tempfile.TemporaryDirectory() as root: + os.mkdir(os.path.join(root, "in-the-way")) + self.assertEqual(writes.replace_file(pathlib.Path(root, "in-the-way"), b"data"), Failure("Is a directory")) + self.assertEqual(os.listdir(root), ["in-the-way"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/test_loop_claude.py b/bin/test_loop_claude.py index 1dbe6069..feab2e8b 100644 --- a/bin/test_loop_claude.py +++ b/bin/test_loop_claude.py @@ -20,11 +20,10 @@ from loop_claude.cycle import archive, interrupt, judge, retry, run, states, wor from loop_claude.project import checkout, model_choice, records from loop_claude.effects import Effects from loop_claude.claude_code import entries, session, transcript, trust -from file_io.failure import Failure +from failure import Failure from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, verdict_from -from loop_claude.cycle.limited import Limited +from loop_claude.cycle.retry import Limited from loop_claude.loop import main, reload -from file_io import markdown, writes from loop_claude import paths from loop_claude.stop import Stop @@ -157,6 +156,12 @@ class Transitions(unittest.TestCase): states.transition(state, event) +class QuestionFor(unittest.TestCase): + def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self): + self.assertEqual(run.question_for(Interrupt("e", "Which one?")), "Which one?") + self.assertEqual(run.question_for(QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.") + + class Cycles(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory() @@ -395,17 +400,6 @@ class Records(unittest.TestCase): self.assertEqual(records.read_records(root), Stop(1, f"AGENTS.md in {root} is not UTF-8 on line 2 - the judge cannot read the project's records.")) -class Sections(unittest.TestCase): - def test_keeps_every_headed_section_with_its_bodies(self): - text = "# Title\n\nintro\n\n## Goals\n\n1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority\n\n## Empty\n\n## Audience\nme\n## Audience\nagain\n# Appendix\nnot audience\n" - self.assertEqual(markdown.sections(text), {"Goals": ["1. one\n```sh\n# not a heading\n```\n### Later\n2. two\n#1 priority"], "Empty": [""], "Audience": ["me", "again"]}) - - def test_closes_a_fence_only_on_its_own_marker(self): - text = "## Goals\n````md\n~~~\n## Quoted\n```\n````\n ~~~\n## Also quoted\n ~~~\n1. one\n## Audience\nme\n" - self.assertEqual(list(markdown.sections(text)), ["Goals", "Audience"]) - self.assertEqual(list(markdown.sections("```\nquoted:\n```python\n## Inside\n```\n## Goals\n1. one\n")), ["Goals"]) - - class Render(unittest.TestCase): def test_joins_assistant_text_tool_calls_human_turns_and_loop_prompts_and_skips_the_rest(self): said = transcript.render([ @@ -422,6 +416,20 @@ class Render(unittest.TestCase): self.assertEqual(said, '[human] do it\n\nStarting.\n\n[tool Bash] {"command": "ls"}\n\n[human] yes, merge it\n\n[human] B, while you work\n\n[loop] Continue the task; the previous run stopped before it was finished.') +class Archive(unittest.TestCase): + def test_writes_afresh_a_copy_whose_outcomes_file_is_missing(self): + with tempfile.TemporaryDirectory() as root: + kept = paths.paths_from(pathlib.Path(root), {}) + transcript = pathlib.Path(root, "t.jsonl") + transcript.write_text('{"timestamp": "2026-10-03T00:00:00Z"}\n') + with contextlib.redirect_stderr(io.StringIO()): + self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Verdict: unfinished", kept)) + (outcomes,) = kept.transcript_archive.glob("proj/*/*.outcomes.md") + outcomes.unlink() + self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Stopped: Ctrl-C", kept)) + self.assertEqual(outcomes.read_text(), "Task: Do it.\n\nStopped: Ctrl-C\n") + + class CopyFor(unittest.TestCase): def test_skips_a_version_already_kept_beside_its_copy(self): with tempfile.TemporaryDirectory() as temp: @@ -487,6 +495,15 @@ class MarkTrusted(unittest.TestCase): self.assertEqual(trust.mark_trusted({}, ["/a"]), ({"projects": {"/a": {"hasTrustDialogAccepted": True}}}, ["/a"])) +class JudgePrompt(unittest.TestCase): + def test_shows_the_transcripts_tail_and_returns_what_it_showed(self): + prompt, shown = judge.judge_prompt("Do it.", "## Goals", "", "old " + "x" * judge.JUDGE_PROMPT_CHARS + " new") + self.assertTrue(shown.endswith(" new")) + self.assertNotIn("old", shown) + self.assertIn(shown, prompt) + self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS) + + class Grounded(unittest.TestCase): def test_every_verdict_must_quote_the_transcript(self): self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n")) @@ -549,7 +566,12 @@ class LoopRuns(unittest.TestCase): class HelpLimits(unittest.TestCase): def test_help_states_the_limits_the_code_holds(self): - self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", main.__doc__.replace("\n", " ")) + said = " ".join(main.__doc__.split()) + self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said) + # The retry messages say these limits in words too. + self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60)) + self.assertIn("fails three times running, retried a minute apart", said) + self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said) class ExitContract(unittest.TestCase): @@ -597,19 +619,39 @@ class ExitContract(unittest.TestCase): self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], []) +class JudgeFailures(unittest.TestCase): + def test_every_failed_judge_run_has_its_clause_in_help(self): + started, field, unasked = "claude cannot start or exits non-zero", "its reply misses or mistypes a field", "interrupts or quotes an answer without a question" + # Every Failure the judge's run returns, in source order: path, function, start of reason, its --help clause. + clauses = [ + ("cycle/judge.py", "run_judge", "f'could not start claude:", started), + ("cycle/judge.py", "run_judge", "proc.stderr.strip() or f'the judge exited", started), + ("cycle/judge.py", "run_judge", 'f"{verdict.reason}; it replied:', field), + ("cycle/judge.py", "run_judge", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"), + ("cycle/verdicts.py", "verdict_from", "'the judge returned no reply'", field), + ("cycle/verdicts.py", "verdict_from", 'f"the judge\'s reply has no {name}', field), + ("cycle/verdicts.py", "verdict_from", "'the judge interrupted without a question'", unasked), + ("cycle/verdicts.py", "verdict_from", "'the judge quoted an answer without a question'", unasked), + ] + root = pathlib.Path(main.__file__).parents[1] + found = [] + for path in ("cycle/judge.py", "cycle/verdicts.py"): + functions = [node for node in ast.walk(ast.parse((root / path).read_text())) if isinstance(node, ast.FunctionDef)] + failures = sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Failure") + found += [(path, function, reason) for _, function, reason in failures] + self.assertEqual(len(found), len(clauses), found) + for (path, function, reason), (want_path, want_function, reason_start, _) in zip(found, clauses): + self.assertEqual((path, function), (want_path, want_function)) + self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}") + judge_clause = " ".join(main.__doc__[main.__doc__.index("a judge when"):main.__doc__.index("A usage limit")].split()) + self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], []) + + class Worktrees(unittest.TestCase): def test_lists_the_root_alone_when_git_cannot_start(self): fake = FakeEffects(git=[FileNotFoundError(2, "No such file or directory")]) self.assertEqual(checkout.worktrees("/r", fake.effects()), ["/r"]) -class ReplaceFile(unittest.TestCase): - def test_removes_what_it_staged_when_the_replace_fails(self): - with tempfile.TemporaryDirectory() as root: - os.mkdir(os.path.join(root, "in-the-way")) - self.assertEqual(writes.replace_file(pathlib.Path(root, "in-the-way"), b"data"), Failure("Is a directory")) - self.assertEqual(os.listdir(root), ["in-the-way"]) - - if __name__ == "__main__": unittest.main() diff --git a/bin/test_loop_claude_black_box.py b/bin/test_loop_claude_black_box.py index 77d68d5c..6f7790a1 100644 --- a/bin/test_loop_claude_black_box.py +++ b/bin/test_loop_claude_black_box.py @@ -670,7 +670,7 @@ class Reload(unittest.TestCase): worker(append=["loop-claude", "\ndef\n"]), judge(), worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(), worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(), - worker(append=["file_io/writes.py", "# edited\n"]), judge(), + worker(append=["files/writes.py", "# edited\n"]), judge(), worker(), ]) self.assertEqual(run.code, 130) diff --git a/bin/test_rank_todo.py b/bin/test_rank_todo.py index 202c717b..52d44512 100644 --- a/bin/test_rank_todo.py +++ b/bin/test_rank_todo.py @@ -7,7 +7,7 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from file_io.failure import Failure +from failure import Failure import rank_todo from test_python_floor import ROOT diff --git a/bin/test_types.py b/bin/test_types.py index a505de47..e071bb85 100644 --- a/bin/test_types.py +++ b/bin/test_types.py @@ -11,5 +11,5 @@ class Types(unittest.TestCase): self.assertIsNotNone(floor, "README.md's Install names `Python 3.N or newer`") if shutil.which("uv") is None: self.fail("uv on PATH fetches mypy; see README.md's Test line") - ran = subprocess.run(["uv", "tool", "run", "--python", "%d.%d" % floor, "mypy==2.3.1", "--cache-dir=/dev/null", "--check-untyped-defs", "--disallow-untyped-defs", "-m", "rank_todo", "-m", "statusline", "-p", "file_io", "-p", "loop_claude"], capture_output=True, cwd=ROOT / "bin", text=True) + ran = subprocess.run(["uv", "tool", "run", "--python", "%d.%d" % floor, "mypy==2.3.1", "--cache-dir=/dev/null", "--check-untyped-defs", "--disallow-untyped-defs", "-m", "rank_todo", "-m", "statusline", "-p", "loop_claude"], capture_output=True, cwd=ROOT / "bin", text=True) self.assertEqual(ran.returncode, 0, ran.stdout + ran.stderr) diff --git a/skills/larv-review/SKILL.md b/skills/larv-review/SKILL.md index 0aa3acd9..04d6fec5 100644 --- a/skills/larv-review/SKILL.md +++ b/skills/larv-review/SKILL.md @@ -67,7 +67,7 @@ Give a reviewer without Bash the branch diff and `git ls-files`'s output in its **Findings.** A reviewer rates each finding 🔴 must fix, 🟡 should fix, 🟢 nit, ❓ question, or 🔀 escalation naming the reviewer whose concern it is. - **Outside the chunk.** A finding outside the chunk's change, whatever its pass or severity, is scored and filed in `todo.md` per `~/.claude/CLAUDE.md` → "Todo items". A 🔴 is marked `defect`. A 🟡 against a file under `~/.claude/principles/` is marked `principle`, its detail naming the principle, where no other reason fits. One under the bar and not exempt is named in the summary with its score. A verified security HIGH is handled per "Security". -- **Goal or decision.** A fix that would change a goal, or a decision the repo held before the chunk began, is asked; no user reachable → filed, marked `question`, the goal or decision left as is. +- **Goal or decision.** A fix that would change a goal, or a standing decision, is asked; no user reachable → filed, marked `question`, the goal or decision left as is. - **🔴.** A 🔴 is fixed, or shown factually wrong, and what disproved it recorded per `~/.claude/CLAUDE.md` → "Decisions: one home, each linked to a goal or a technical principle": in a comment at the code the finding concerns, or in `docs/decisions.md` when it concerns more than one place. Never declined. - **🟡.** A 🟡 is fixed, shown factually wrong like a 🔴, or declined. Decline one only where the reason is sound, consistent with the goals and the technical principles, and confidence is very high; the decline and its reason are recorded the same way. A reason neither speaks to is a gap in the goals: fix the finding, or ask per `~/.claude/CLAUDE.md` → "Working loop", "Ask, don't guess"; no user reachable → fix. Never drop one silently, and never decline one for convenience or the size of the fix. - **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a standing decision settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`. diff --git a/todo.md b/todo.md index 505ed3a0..7fbce108 100644 --- a/todo.md +++ b/todo.md @@ -6,7 +6,7 @@ `Bar = 9` -`Next ID = 46` +`Next ID = 53` | Goal | W | |---|---| @@ -25,13 +25,20 @@ | ID | Release | Exempt | Item | R | S | A | G | Goals | Score | |---|---|---|---|---|---|---|---|---|---| | 1 | 0.1.0 | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 3 | 5 | 8 | 9 | 2, 3 | 27.8 | +| 47 | 0.1.0 | | **Archive the judge's evidence with each verdict, so a wrong finish shows what it rested on.** | 1 | 1 | 6 | 6 | 2, 4 | 21.4 | +| 46 | 0.1.0 | question | **Let the judge read and quote `~/.claude/CLAUDE.md` beside the project's records.** | 4 | 3 | 7 | 8 | 4, 4.2 | 20.0 | +| 48 | 0.1.0 | | **Stamp each archived outcome with its UTC time.** | 1 | 1 | 4 | 4 | 2 | 13.9 | | 23 | 0.1.0 | | **Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.** | 3 | 4 | 7 | 8 | 4.4 | 13.5 | | 21 | 0.1.0 | | **Give `settings.json`'s model one writer, and sort its keys.** | 3 | 3 | 6 | 7 | 4.3 | 12.9 | +| 49 | 0.1.0 | principle | **Check that `AGENTS.md` lists exactly the titles of `docs/decisions.md`.** | 1 | 2 | 5 | 6 | 4.3 | 12.5 | | 12 | 0.1.0 | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 2 | 4 | 4 | 6 | 4.1 | 11.7 | | 5 | 0.1.0 | decision | **Key a project on its checkout's full path, and refuse to start while another session uses that key.** | 5 | 5 | 5 | 6 | 4.1 | 10.5 | | 4 | 0.1.0 | | **Name each chunk's archive folder by the chunk's start time, so `/` lists chunks in the order they ran.** | 3 | 2 | 4 | 3 | 2 | 9.8 | +| 50 | 0.1.0 | principle | **Grant the bench's runs only the tools a review needs, in place of skipping every permission.** | 2 | 2 | 3 | 4 | 3 | 9.7 | | 29 | 0.1.0 | | **Make the contributor a standing member of every project's audience, unless its `## Audience` says maintainability does not matter.** | 2 | 2 | 3 | 5 | 4.1, 4.4 | 9.1 | +| 52 | 0.1.0 | principle | **Record why this checkout lives in Claude Code's own directory, or move it out and link it in.** | 6 | 6 | 3 | 5 | 1 | 8.5 | | 6 | 0.1.0 | decision | **Move what `loop-claude` writes out of `~/.claude.json`, and ask the human about each write that must stay.** | 5 | 5 | 4 | 5 | 4.1 | 7.4 | +| 51 | 0.1.0 | principle | **Hand loop-claude's terminal output to an injected effect, so no storage or retry function prints.** | 3 | 5 | 3 | 5 | 4.1 | 7.4 | ## Details @@ -41,6 +48,18 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost 2026-10-02, the human: ai-sandbox is the VM `loop-claude` will run on, hosted on the server bender. bender-infra, the infrastructure-as-code project that manages bender, will back up ai-sandbox's data volume once ai-sandbox is rebuilt. The item waits for the human to say the rebuild has landed. Before writing the README line, check that `~/.claude` lives on that volume and that the backup is private and encrypted at rest. +### 47. Archive the judge's evidence with each verdict, so a wrong finish shows what it rested on. + +The judge runs without session persistence, and `close` and `ask` archive only the verdict and the question. A wrong `finished` deletes the session, and nothing then shows the passage the judge quoted. + +### 46. Let the judge read and quote `~/.claude/CLAUDE.md` beside the project's records. + +The goals audit on PR #208 rated it 🔴 against goals 4 and 4.2: the worker follows `CLAUDE.md`, but the judge sees only the premises, `AGENTS.md` and `docs/decisions.md`. A choice `CLAUDE.md` settles therefore reads to the judge as one nothing discriminates, and a question it answers goes to the human. The fix revises the decision "loop-claude: a judge decides each interrupt and each quoted answer from the transcript, the README's premises, `AGENTS.md` and `docs/decisions.md`", so the human decides it. + +### 48. Stamp each archived outcome with its UTC time. + +After an overnight exit 1, `.outcomes.md` holds each outcome without saying when it happened. + ### 23. Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers. On this repo the frontend, REST and logging auditors each read the whole project to report "does not apply", and one 🟢 fix reruns ten whole-project reads. @@ -51,6 +70,10 @@ Claude Code rewrites `"model"` in this tracked file whenever a model is picked, Every write Claude Code makes to the file rewrites `"model"`, `claude plugin install` included: Claude Code 2.1.288 turned `"opus"` into `"opus[1m]"`. Each write also puts the keys back in Claude Code's own order, so a sorted file stays sorted only until the next write. +### 49. Check that `AGENTS.md` lists exactly the titles of `docs/decisions.md`. + +DRY, in `principles/technical-principles.md`: the list is copied by hand, and sessions work from the copy. + ### 12. Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough. The repo has no CI, so larv-review's gate is the local run, and larv-review's Gate skips even that when the last stability round left the suite green: an unattended chunk can merge on the worker's own word. @@ -63,10 +86,22 @@ The key names the project's session, archive folder and models entry. `project_k Today the folder is the SHA-256 of the transcript's first timestamped line, which tells the human nothing; that line carries the start time. Drop the fallback to the first line, which every chunk of a project shares, and give the tests' fake transcripts ISO timestamps. +### 50. Grant the bench's runs only the tools a review needs, in place of skipping every permission. + +"Always grant explicit access to data", in `principles/technical-principles.md`: `workflow-bench/bench.py` starts claude with `--dangerously-skip-permissions`, and no rule confines it to a sandbox, so a run on the host reaches the host's credentials and other checkouts. + ### 29. Make the contributor a standing member of every project's audience, unless its `## Audience` says maintainability does not matter. State that in `CLAUDE.md`'s goals-and-audience rule, and have larv-review's comprehension floor name the contributor as the reader it serves. The floor then gates every chunk while the contributor stands, and a project that drops them skips it. +### 52. Record why this checkout lives in Claude Code's own directory, or move it out and link it in. + +"One owner per value" and "Prefer the vendor-neutral option", in `principles/technical-principles.md`: git, Claude Code and loop-claude all write `~/.claude`, kept apart only by the `.gitignore` allowlist, and the repo's records live in a vendor directory. + ### 6. Move what `loop-claude` writes out of `~/.claude.json`, and ask the human about each write that must stay. Claude Code owns that file and writes it. `trust` reads it, changes it and writes it back, so a write that lands in between is lost. + +### 51. Hand loop-claude's terminal output to an injected effect, so no storage or retry function prints. + +"Functional core, imperative shell", in `principles/technical-principles.md`: `archive`, `trust`, `retried`, the worker, `interrupt` and `model_choice` print to stderr themselves, and the tests swap the global stream. diff --git a/workflow-bench/bench.py b/workflow-bench/bench.py index 3c127b28..e6d2a4d0 100755 --- a/workflow-bench/bench.py +++ b/workflow-bench/bench.py @@ -1,5 +1,17 @@ #!/usr/bin/env python3 -"""Run larv-review bench runs: BENCH_SOURCES=-main clones, their -main.md reports and comparison.md> bench.py -- ...""" +"""Run larv-review bench runs: BENCH_SOURCES= bench.py -- ... + +The 2026-09-30 bench had two versions of larv-review, main and rebuild, each review four real chunks, +the cases, three times; compare.py then rules which version reviews better. Its inputs are untracked: +- versions// beside this script: a copy of ~/.claude at that version, which each run reads + in place of ~/.claude (run-prompt.md). +- $BENCH_SOURCES/-main/: a clone of the case's project; CASES names the chunk's base and head + commits in it. +- $BENCH_SOURCES/-main.md: main's own review of the case, run on 2026-09-29. +- $BENCH_SOURCES/comparison.md: what each case's real review found, one table row per issue, its ID + led by the case's letter in compare.py's PREFIX. +Each run leaves its clone in /, its report in .md and its claude stream in logs/. +""" import concurrent.futures import json import os diff --git a/workflow-bench/compare.py b/workflow-bench/compare.py index 8a030127..2c266b31 100755 --- a/workflow-bench/compare.py +++ b/workflow-bench/compare.py @@ -29,6 +29,7 @@ def runs(case, version): def blind(text): + """The report with every name that tells which version wrote it replaced.""" return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text)