Work in progress: Bash output ceiling and refusing Reads of session transcripts
This commit is contained in:
@@ -191,7 +191,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
|
||||
- `project-record open <record>` prints the path of a file holding one of a project's records, for an agent to read and edit, and `project-record save <record>` stores the edit; for a record kept as a file, that file is the record. `goals-and-audience` opens the goals file the same way.
|
||||
- `rank_todo.py [--check] [todo.md]` scores and ranks a project's `todo.md`, by default the one its config names; `rank_todo.py --help` states the format.
|
||||
- `reading-cost [--reset] [record]` appends a figure for each chunk `loop-claude` finished to a project's `docs/reading-cost.md`, and says whether the figures rise; `reading-cost --help` states the measure.
|
||||
- `review-replay <pass> <model>` replays a review pass's archived rounds from the projects run on the model, on the code each reviewed, and prints the lowest rung below the project's that, like each rung above it, qualifies: it finds no less of the 🔴 and 🟡 findings than two replays on the project's rung find, within a margin, and misses no 🔴 that both found. `review-replay --help` states the replay. With `--omit-claude-md` it tests instead whether the pass's reviewer may start without `CLAUDE.md`: it replays each round three times as archived and once without the instruction files, and prints whether the reviewer passes: whether the last replay missed no more findings than the most an unchanged replay missed, with each count and each finding it missed. Each replayed reviewer skips every permission prompt, stability-reviewer has Bash, and an archived prompt may name paths on the host.
|
||||
- `review-replay <pass> <model>` replays a review pass's archived rounds from the projects run on the model, on the code each reviewed, and prints the lowest rung below the project's that, like each rung above it, qualifies: it finds no less of the 🔴 and 🟡 findings than two replays on the project's rung find, within a margin, and misses no 🔴 that both found. `review-replay --help` states the replay. With `--omit-claude-md`, `--bash-ceiling` or `--read-limit` it tests instead whether the pass's reviewer may start without `CLAUDE.md`, or with the Bash or Read output limit loop-claude hands the worker: it replays each round three times as archived and once with the change, and prints whether the reviewer passes: whether the last replay missed no more findings than the most an unchanged replay missed, with each count and each finding it missed. Each replayed reviewer skips every permission prompt, stability-reviewer has Bash, and an archived prompt may name paths on the host.
|
||||
- `rule-hook [overrides]` blocks a tool call that breaks a rule a program can check, unless the call overrides the rule; `settings.json` → `hooks` runs it before each Agent, Bash, Edit, Grep, MultiEdit, Read, Skill and Write call, and `rule-hook --help` states the rules and the override.
|
||||
- `run-heavy <command>` runs a heavy command once it holds one of the machine's heavy slots, one per 4 CPUs, which every session shares; `run-heavy --help` states the slots, and the memory cap it gives a container that sets no limit.
|
||||
- `run-tests [<pytest args>...]` runs this repo's test suite, as Test says.
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""The environment variables that cut what one tool result adds to a claude's context, and the values loop-claude hands the worker's claude and review-replay tests: docs/decisions.md → `loop-claude: the worker's claude saves a long Bash result to a file`.
|
||||
|
||||
A subagent inherits both variables, so they reach every reviewer the worker launches."""
|
||||
|
||||
# Claude Code saves a Bash result longer than this many characters to a file, and hands the model the file's path and the first 2,000 characters. The file sits under ~/.claude/projects/<project>/<session>/tool-results/.
|
||||
BASH_CEILING = "BASH_MAX_OUTPUT_LENGTH"
|
||||
BASH_CEILING_CHARACTERS = 6_000
|
||||
# A Read of more than this many tokens fails and asks for an offset and a limit; Claude Code's default is 25,000.
|
||||
READ_LIMIT = "CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS"
|
||||
READ_LIMIT_TOKENS = 5_000
|
||||
@@ -4,6 +4,7 @@ import typing
|
||||
|
||||
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanTalked, HumanTyped, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, to_the_human, transition, TypedAnswer, WorkerRan, Working
|
||||
from transcript_archive.write import archive
|
||||
from claude_code.output_limits import BASH_CEILING, BASH_CEILING_CHARACTERS
|
||||
from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args
|
||||
from loop_claude.cycle.judge import Brief, judge, Rejected
|
||||
from loop_claude.cycle import chunk_items
|
||||
@@ -128,8 +129,8 @@ def work(cycle: Cycle, answered: QuotedAnswer | TypedAnswer | None, stepped: boo
|
||||
|
||||
|
||||
def worker_environment(cycle: Cycle) -> dict[str, str]:
|
||||
"""The rungs the worker's claude is handed, which rule-hook holds its review launches to, and the auto-compact window of its context."""
|
||||
return {**worker_rungs.environment(cycle.project, cycle.route), AUTO_COMPACT_WINDOW: str(COMPACT_WINDOW)}
|
||||
"""The rungs the worker's claude is handed, which rule-hook holds its review launches to, the auto-compact window of its context, and the length past which it saves a Bash result to a file."""
|
||||
return {**worker_rungs.environment(cycle.project, cycle.route), AUTO_COMPACT_WINDOW: str(COMPACT_WINDOW), BASH_CEILING: str(BASH_CEILING_CHARACTERS)}
|
||||
|
||||
|
||||
def worker_session(cycle: Cycle, transcript_path: str) -> WorkerSession:
|
||||
|
||||
@@ -83,7 +83,9 @@ available (bin/machine_load/memory.py).
|
||||
|
||||
Each worker's claude, the session /talk opens and the subagents either launches run with a 200k
|
||||
auto-compact window (CLAUDE_CODE_AUTO_COMPACT_WINDOW, bin/loop_claude/cycle/worker.py), so each
|
||||
compacts its context at about 170k tokens.
|
||||
compacts its context at about 170k tokens. They also save a Bash result of more than 6,000
|
||||
characters to a file and see its path and the first 2,000 characters (BASH_MAX_OUTPUT_LENGTH,
|
||||
bin/claude_code/output_limits.py).
|
||||
|
||||
Every claude loop-claude starts runs the project's model, saved per <key> in
|
||||
~/.claude/loop-claude-models.json and read each cycle, unless routing, below, puts the worker,
|
||||
|
||||
@@ -26,6 +26,7 @@ from project_config.test_project_config import configured
|
||||
from release_plan.release_standing import DefaultPlan, LastCut, UNSEEN_RELEASES
|
||||
from loop_effects.effects import Effects, PYTHON_TIMEOUT
|
||||
from claude_code import entries, reply, session
|
||||
from claude_code.output_limits import BASH_CEILING_CHARACTERS
|
||||
from failure import Failure
|
||||
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
|
||||
from git_cli.git_repo import git
|
||||
@@ -344,7 +345,7 @@ Worker: {OFF_LADDER}
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
||||
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
||||
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000"}])
|
||||
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000", "BASH_MAX_OUTPUT_LENGTH": "6000"}])
|
||||
|
||||
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
|
||||
def answer(args):
|
||||
@@ -2441,6 +2442,7 @@ class HelpLimits(unittest.TestCase):
|
||||
self.assertIn(f"still running after {PYTHON_TIMEOUT} s", said)
|
||||
self.assertIn(f"Once {state.CYCLES_PER_CHECK} cycles have closed across all projects", said)
|
||||
self.assertIn(f"run with a {worker.COMPACT_WINDOW // 1000}k auto-compact window", said)
|
||||
self.assertIn(f"{BASH_CEILING_CHARACTERS:,} characters", said)
|
||||
self.assertIn(f"unfinished {close.UNFINISHED_LIMIT} cycles running", said)
|
||||
self.assertIn(f"rerun to give it {close.UNFINISHED_LIMIT} more", said)
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""The experiment behind review-replay's --omit-claude-md, --bash-ceiling and --read-limit: three replays of a pass's rounds unchanged, one with the change, and the report that holds the one with it to what the others found: docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed`."""
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from failure import Failure
|
||||
from claude_code.reply import Limited
|
||||
from rung_replay.search import replayed_on
|
||||
from review_replay.changes import Change
|
||||
from review_replay.rounds import Round
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
|
||||
def compared(reviewed: str, project: Rung, chosen: list[Round], change: Change, found_on: typing.Callable[[Round, Change | None], set[int] | Failure | Limited]) -> str | Limited:
|
||||
"""Replays chosen three times unchanged and once with change, as found_on(round, change or None) finds, and reports the comparison of the rounds no replay failed on; the Limited where claude hit its usage limit."""
|
||||
runs: list[list[set[int] | Failure]] = []
|
||||
for applied in (None, None, None, change):
|
||||
label = (applied or change).changed if applied else change.unchanged
|
||||
|
||||
def replay_one(one: Round, _: Rung, applied: Change | None = applied, label: str = label) -> set[int] | Failure | Limited:
|
||||
found = found_on(one, applied)
|
||||
if isinstance(found, Failure):
|
||||
print(f" {one.root} {one.at} {label} failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True)
|
||||
return found
|
||||
|
||||
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {label}")
|
||||
if isinstance(found, Limited):
|
||||
return found
|
||||
runs.append(found)
|
||||
# A failed replay found no findings, which would lower the bar where it was unchanged and count as misses where it had the change.
|
||||
kept = rounds_without_failure(chosen, runs)
|
||||
first, second, third, without = ([found[run] for _, found in kept] for run in range(4))
|
||||
return comparison(reviewed, project, change, [one for one, _ in kept], (first, second, third), without, len(chosen) - len(kept))
|
||||
|
||||
|
||||
def rounds_without_failure(chosen: list[Round], runs: list[list[set[int] | Failure]]) -> list[tuple[Round, list[set[int]]]]:
|
||||
"""Each round of chosen that no run failed on, with what each run found in it."""
|
||||
kept = []
|
||||
for one, *found in zip(chosen, *runs):
|
||||
finds = [each for each in found if not isinstance(each, Failure)]
|
||||
if len(finds) == len(found):
|
||||
kept.append((one, finds))
|
||||
return kept
|
||||
|
||||
|
||||
def comparison(reviewed: str, reference: Rung, change: Change, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str:
|
||||
"""What the three unchanged replays and the one with change found in rounds. Each unchanged replay missed some findings the other two found: the control, the noise of an unchanged reviewer. The one with the change is held to the findings the first two found, as the third is, and passes where it missed no more than the most an unchanged replay did. left_out counts the rounds a replay failed on, which rounds leaves out."""
|
||||
first, second, third = with_it
|
||||
|
||||
def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]:
|
||||
return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)]
|
||||
|
||||
def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]:
|
||||
return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)]
|
||||
|
||||
def count(run: list[set[int]]) -> int:
|
||||
return sum(len(found) for found in run)
|
||||
|
||||
both = shared(first, second)
|
||||
missed = missing(both, without)
|
||||
control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))]
|
||||
passes = len(missed) <= max(control)
|
||||
return "\n".join([
|
||||
f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three {change.unchanged}, one {change.changed}.",
|
||||
*([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []),
|
||||
f"{capitalized(change.unchanged)} the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.",
|
||||
*([f"{capitalized(change.changed)} it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"{capitalized(change.changed)} it found {count(without)} of them. No evidence: the first two replays {change.unchanged} found none in common, so there is no finding to hold it to."]),
|
||||
*(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed),
|
||||
*([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay {change.unchanged} missed, {max(control)}."] if count(both) else []),
|
||||
])
|
||||
|
||||
|
||||
def capitalized(words: str) -> str:
|
||||
return words[0].upper() + words[1:]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""The changes to how a reviewer starts that review-replay tests against the reviewer unchanged: each a flag, the settings it hands claude, and the words the report uses for the replays with and without it."""
|
||||
import dataclasses
|
||||
|
||||
from claude_code.output_limits import BASH_CEILING, BASH_CEILING_CHARACTERS, READ_LIMIT, READ_LIMIT_TOKENS
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Change:
|
||||
flag: str
|
||||
# What the report calls a replay without the change, and one with it, as in "three with CLAUDE.md, one without CLAUDE.md".
|
||||
unchanged: str
|
||||
changed: str
|
||||
# The environment the replay with the change starts claude under, through its settings.
|
||||
env: dict[str, str]
|
||||
# Where the change turns the instruction files off, the prompt holds what the SubagentStart hook and the header's last line would hand the reviewer.
|
||||
without_instruction_files: bool = False
|
||||
|
||||
|
||||
OMIT = Change("--omit-claude-md", "with CLAUDE.md", "without CLAUDE.md", {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}, without_instruction_files=True)
|
||||
BASH = Change("--bash-ceiling", "as archived", f"under a Bash ceiling of {BASH_CEILING_CHARACTERS:,} characters", {BASH_CEILING: str(BASH_CEILING_CHARACTERS)})
|
||||
READ = Change("--read-limit", "as archived", f"under a Read limit of {READ_LIMIT_TOKENS:,} tokens", {READ_LIMIT: str(READ_LIMIT_TOKENS)})
|
||||
CHANGES = (OMIT, BASH, READ)
|
||||
+22
-13
@@ -1,6 +1,6 @@
|
||||
"""Replay a review pass's archived rounds on the rungs below the project's, to find which rungs may run its reviewer: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.
|
||||
|
||||
usage: review-replay <pass> <model> [--omit-claude-md]
|
||||
usage: review-replay <pass> <model> [--omit-claude-md | --bash-ceiling | --read-limit]
|
||||
|
||||
<pass> is one of architecture, principles, overrides, stability, product-owner or prose.
|
||||
<model> is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the
|
||||
@@ -45,8 +45,16 @@ replay, and runs no SubagentStart hook there. So the replay without CLAUDE.md tu
|
||||
instruction files off in its settings and opens the prompt with what the hook hands a reviewer, and
|
||||
closes it with the header's line on the project principles.
|
||||
|
||||
--bash-ceiling and --read-limit run the same comparison for the output limits loop-claude hands the worker's
|
||||
claude, which a subagent inherits (bin/claude_code/output_limits.py): the replay with the change starts under
|
||||
BASH_MAX_OUTPUT_LENGTH, which saves a longer Bash result to a file and hands the reviewer its path and the
|
||||
first 2,000 characters, or under CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS, which fails a Read of more tokens
|
||||
than that. It passes where it missed no more of the findings both of the first two found than the most an
|
||||
unchanged replay missed. It doesn't count what the replays read: each run's stream, with every tool result,
|
||||
is in the archive.
|
||||
|
||||
It keeps each run, a reviewer's tool calls included, and the report it prints in loop-claude's archive, in
|
||||
replays/review-replay-<pass>-<model>[-omit-claude-md]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
|
||||
replays/review-replay-<pass>-<model>[-<flag without its dashes>]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
|
||||
or $LOOP_CLAUDE_TRANSCRIPTS. A run's .jsonl file starts with a line holding claude's arguments, working
|
||||
directory, prompt, exit code and stderr, then holds claude's stream-json output; result.md holds the report.
|
||||
|
||||
@@ -64,7 +72,8 @@ from review_replay.rounds import archived_rounds, Round, sample
|
||||
from rung_replay.archive import archiving, print_report, replay_folder
|
||||
from rung_replay.command import arguments_or_exit, failed, limited_exit
|
||||
from review_replay.replay import codes_found, findings_replayed
|
||||
from review_replay.omit_comparison import compared
|
||||
from review_replay.change_comparison import compared
|
||||
from review_replay.changes import Change, CHANGES
|
||||
from loop_effects.effects import Effects
|
||||
from failure import Failure
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
@@ -74,8 +83,7 @@ from review_replay.recall import recall_report, recalled
|
||||
from review_rules.review_passes import ReviewedPass, REVIEWERS
|
||||
|
||||
TOOL = "review-replay"
|
||||
OMIT = "--omit-claude-md"
|
||||
USAGE = f"{TOOL} <pass> <model> [{OMIT}], <pass> one of {', '.join(REVIEWERS)} and <model> a model on the routing ladder"
|
||||
USAGE = f"{TOOL} <pass> <model> [{' | '.join(change.flag for change in CHANGES)}], <pass> one of {', '.join(REVIEWERS)} and <model> a model on the routing ladder"
|
||||
LATEST = 10
|
||||
|
||||
|
||||
@@ -87,7 +95,7 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
|
||||
read = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: pass_and_model(ladder, argv))
|
||||
if isinstance(read, int):
|
||||
return read
|
||||
reviewed, project, without = read
|
||||
reviewed, project, change = read
|
||||
rounds = archived_rounds(REVIEWERS[reviewed], project.model, archive_root(home, env), home)
|
||||
if isinstance(rounds, Failure):
|
||||
return failed(TOOL, rounds.reason)
|
||||
@@ -97,15 +105,15 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
|
||||
chosen = sample(list(codes), LATEST)
|
||||
folder = replay_folder(archive_root(home, env), TOOL, argv, effects.now())
|
||||
effects = archiving(effects, folder)
|
||||
if without:
|
||||
report = compared(reviewed, project, chosen, lambda found, bare: findings_replayed(found, codes[found], project, project, home, effects, bare))
|
||||
if change:
|
||||
report = compared(reviewed, project, chosen, change, lambda found, applied: findings_replayed(found, codes[found], project, project, home, effects, applied))
|
||||
if isinstance(report, Limited):
|
||||
return limited_exit(TOOL, report)
|
||||
print_report(folder, report)
|
||||
return 0
|
||||
|
||||
def replay_one(found: Round, rung: Rung) -> set[int] | Failure | Limited:
|
||||
matched = findings_replayed(found, codes[found], rung, project, home, effects, False)
|
||||
matched = findings_replayed(found, codes[found], rung, project, home, effects, None)
|
||||
if isinstance(matched, Failure):
|
||||
print(f" {found.root} {found.at} on {rung} failed: {matched.reason}", file=sys.stderr, flush=True)
|
||||
return matched
|
||||
@@ -117,10 +125,11 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
|
||||
return 0
|
||||
|
||||
|
||||
def pass_and_model(ladder: Ladder, argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None:
|
||||
"""The pass, the project's rung on ladder of the model, and whether OMIT follows, as argv names them in `<pass> <model> [--omit-claude-md]`; None for any other argv."""
|
||||
if argv[2:] not in ([], [OMIT]) or len(argv) < 2:
|
||||
def pass_and_model(ladder: Ladder, argv: list[str]) -> tuple[ReviewedPass, Rung, Change | None] | None:
|
||||
"""The pass, the project's rung on ladder of the model, and the change a flag names, as argv names them in `<pass> <model> [<flag>]`; None for any other argv."""
|
||||
change = next((each for each in CHANGES if argv[2:] == [each.flag]), None)
|
||||
if argv[2:] and change is None or len(argv) < 2:
|
||||
return None
|
||||
reviewed = next((kind for kind in REVIEWERS if kind == argv[0]), None)
|
||||
project = ladder.project_rung(argv[1])
|
||||
return (reviewed, project.rung, argv[2:] == [OMIT]) if reviewed and project else None
|
||||
return (reviewed, project.rung, change) if reviewed and project else None
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
"""The experiment behind review-replay's --omit-claude-md: three replays of a pass's rounds with CLAUDE.md, one without, and the report that holds the one without to what the others found: docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed`."""
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from failure import Failure
|
||||
from claude_code.reply import Limited
|
||||
from rung_replay.search import replayed_on
|
||||
from review_replay.rounds import Round
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
|
||||
def compared(reviewed: str, project: Rung, chosen: list[Round], found_on: typing.Callable[[Round, bool], set[int] | Failure | Limited]) -> str | Limited:
|
||||
"""Replays chosen three times with CLAUDE.md and once without, as found_on(round, bare) finds, and reports the comparison of the rounds no replay failed on; the Limited where claude hit its usage limit."""
|
||||
runs: list[list[set[int] | Failure]] = []
|
||||
for bare in (False, False, False, True):
|
||||
def replay_one(one: Round, _: Rung, bare: bool = bare) -> set[int] | Failure | Limited:
|
||||
found = found_on(one, bare)
|
||||
if isinstance(found, Failure):
|
||||
print(f" {one.root} {one.at} {'without' if bare else 'with'} the instruction files failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True)
|
||||
return found
|
||||
|
||||
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {'without' if bare else 'with'} the instruction files")
|
||||
if isinstance(found, Limited):
|
||||
return found
|
||||
runs.append(found)
|
||||
# A failed replay found no findings, which would lower the bar where it had CLAUDE.md and count as misses where it had none.
|
||||
kept = rounds_without_failure(chosen, runs)
|
||||
first, second, third, without = ([found[run] for _, found in kept] for run in range(4))
|
||||
return comparison(reviewed, project, [one for one, _ in kept], (first, second, third), without, len(chosen) - len(kept))
|
||||
|
||||
|
||||
def rounds_without_failure(chosen: list[Round], runs: list[list[set[int] | Failure]]) -> list[tuple[Round, list[set[int]]]]:
|
||||
"""Each round of chosen that no run failed on, with what each run found in it."""
|
||||
kept = []
|
||||
for one, *found in zip(chosen, *runs):
|
||||
finds = [each for each in found if not isinstance(each, Failure)]
|
||||
if len(finds) == len(found):
|
||||
kept.append((one, finds))
|
||||
return kept
|
||||
|
||||
|
||||
def comparison(reviewed: str, reference: Rung, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str:
|
||||
"""What the three replays with CLAUDE.md and the one without found in rounds. Each replay with it missed some findings the other two found: the control, the noise of an unchanged reviewer. The one without it is held to the findings the first two found, as the third is, and passes where it missed no more than the most a replay with it did. left_out counts the rounds a replay failed on, which rounds leaves out."""
|
||||
first, second, third = with_it
|
||||
|
||||
def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]:
|
||||
return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)]
|
||||
|
||||
def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]:
|
||||
return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)]
|
||||
|
||||
def count(run: list[set[int]]) -> int:
|
||||
return sum(len(found) for found in run)
|
||||
|
||||
both = shared(first, second)
|
||||
missed = missing(both, without)
|
||||
control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))]
|
||||
passes = len(missed) <= max(control)
|
||||
return "\n".join([
|
||||
f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three with CLAUDE.md, one without.",
|
||||
*([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []),
|
||||
f"With CLAUDE.md the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.",
|
||||
*([f"Without CLAUDE.md it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"Without CLAUDE.md it found {count(without)} of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to."]),
|
||||
*(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed),
|
||||
*([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay with CLAUDE.md missed, {max(control)}."] if count(both) else []),
|
||||
])
|
||||
+17
-14
@@ -7,6 +7,7 @@ import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from review_replay.changes import Change
|
||||
from review_replay.rounds import authored_by, lost_inputs, reviewed_path, Round
|
||||
from claude_code.reply import claude_reply, Limited
|
||||
from git_cli.git_repo import default_ref
|
||||
@@ -21,8 +22,6 @@ from review_rules.review_folder import REPORTS, review_folder
|
||||
from model_routing.ladder import Rung
|
||||
from claude_code.entries import structured_output
|
||||
|
||||
# Starts claude without its instruction files, as `omitClaudeMd` starts a subagent.
|
||||
NO_CLAUDE_MD = ["--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})]
|
||||
MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False}
|
||||
|
||||
|
||||
@@ -76,9 +75,9 @@ def commit_log(listed: str) -> list[tuple[str, float]]:
|
||||
return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())]
|
||||
|
||||
|
||||
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> set[int] | Failure | Limited:
|
||||
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where bare, the replay runs without CLAUDE.md, as `launch` says."""
|
||||
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, bare))
|
||||
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, change: Change | None) -> set[int] | Failure | Limited:
|
||||
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where change is given, the replay runs with it, as `launch` says."""
|
||||
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, change))
|
||||
if isinstance(reviewed, (Failure, Limited)):
|
||||
return reviewed
|
||||
return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects))
|
||||
@@ -92,17 +91,17 @@ class Copies:
|
||||
|
||||
|
||||
# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is.
|
||||
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
|
||||
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, change: Change | None) -> str | Failure | Limited:
|
||||
try:
|
||||
(home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="review-replay-", ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports:
|
||||
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, bare)
|
||||
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, change)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not prepare the replay: {failure}")
|
||||
|
||||
|
||||
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
|
||||
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; without CLAUDE.md where bare."""
|
||||
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, change: Change | None) -> str | Failure | Limited:
|
||||
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; with change where given."""
|
||||
folder = pathlib.Path(review_folder(str(home), code.path))
|
||||
copy_reports(found, folder, copies.reports)
|
||||
prompt = redirected(found.prompt, home, folder, code.path, copies)
|
||||
@@ -112,7 +111,7 @@ def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: path
|
||||
return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's")
|
||||
if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure):
|
||||
return cloned
|
||||
launched = launch(found.reviewer, rung, prompt, found.root, copies, home, bare)
|
||||
launched = launch(found.reviewer, rung, prompt, found.root, copies, home, change)
|
||||
if isinstance(launched, Failure):
|
||||
return launched
|
||||
reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude")
|
||||
@@ -121,10 +120,14 @@ def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: path
|
||||
return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report")
|
||||
|
||||
|
||||
def launch(reviewer: str, rung: Rung, prompt: str, checkout: str, copies: Copies, home: pathlib.Path, bare: bool) -> tuple[list[str], str] | Failure:
|
||||
"""claude's arguments and prompt for reviewer on rung, in copies.clone, a copy of the project whose checkout is checkout. Where bare, it starts without CLAUDE.md, and the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
|
||||
def launch(reviewer: str, rung: Rung, prompt: str, checkout: str, copies: Copies, home: pathlib.Path, change: Change | None) -> tuple[list[str], str] | Failure:
|
||||
"""claude's arguments and prompt for reviewer on rung, in copies.clone, a copy of the project whose checkout is checkout. Where change is given, claude starts under its environment, through its settings; where change turns the instruction files off, the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
|
||||
args = ["--agent", reviewer, *rung.flags()]
|
||||
if not bare:
|
||||
if change is None:
|
||||
return args, prompt
|
||||
# Another --settings would replace this one, so the whole environment goes in one.
|
||||
args += ["--settings", json.dumps({"env": change.env})]
|
||||
if not change.without_instruction_files:
|
||||
return args, prompt
|
||||
context = reviewer_context(home)
|
||||
if isinstance(context, Failure):
|
||||
@@ -140,7 +143,7 @@ def launch(reviewer: str, rung: Rung, prompt: str, checkout: str, copies: Copies
|
||||
tracked = config.tracked("working-rules")
|
||||
# Likewise for an `AGENTS.md` outside the checkout.
|
||||
rules = config.paths["working-rules"] if tracked is None else pathlib.Path(copies.clone, tracked)
|
||||
return [*args, *NO_CLAUDE_MD], "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(rules, decisions)) else [])])
|
||||
return args, "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(rules, decisions)) else [])])
|
||||
|
||||
|
||||
def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None:
|
||||
|
||||
@@ -20,7 +20,7 @@ from project_config.test_project_config import configured, write_config
|
||||
from loop_effects.effects import Effects, system_effects
|
||||
from claude_code.entries import entry_from
|
||||
from failure import Failure
|
||||
from review_replay import main, recall, replay
|
||||
from review_replay import changes, main, recall, replay
|
||||
from review_rules.reviewer_context import reviewer_context
|
||||
from review_rules.review_passes import REVIEWERS
|
||||
from model_routing.ladder import Rung
|
||||
@@ -130,11 +130,11 @@ class Launch(unittest.TestCase):
|
||||
self.config = configured(self, self.checkout, home=self.home) / "config.toml"
|
||||
|
||||
def test_runs_the_reviewer_as_archived_with_claude_md(self):
|
||||
self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, False), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
|
||||
self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, None), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
|
||||
|
||||
def test_drops_claude_md_and_hands_the_reviewer_what_the_hook_and_the_header_would(self):
|
||||
(self.clone / "AGENTS.md").write_text("")
|
||||
args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, True)
|
||||
args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)
|
||||
self.assertEqual(args, ["--agent", "prose-reviewer", *REFERENCE.flags(), "--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})])
|
||||
self.assertEqual(prompt, f"{reviewer_context(self.home)}\n\nReview it.\nPrinciples: read {self.clone}/AGENTS.md first, for the project principles and the decision titles.")
|
||||
|
||||
@@ -142,7 +142,7 @@ class Launch(unittest.TestCase):
|
||||
self.config.write_text(self.config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"'))
|
||||
(self.clone / "AGENTS.md").write_text("")
|
||||
(self.config.parent / "AGENTS.md").write_text("")
|
||||
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, True)[1]
|
||||
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
|
||||
self.assertTrue(prompt.endswith(f"Principles: read {self.config.parent}/AGENTS.md first, for the project principles and the decision titles."), prompt)
|
||||
|
||||
def test_names_the_decision_log_in_the_clone_or_where_it_stands_outside_the_checkout_without_agents_md(self):
|
||||
@@ -152,16 +152,16 @@ class Launch(unittest.TestCase):
|
||||
self.config.write_text(self.config.read_text().replace('"checkout:docs/decisions.md"', f'"{setting}"'))
|
||||
decisions.parent.mkdir(exist_ok=True)
|
||||
decisions.write_text("1. A principle.\n\n# Decisions\n")
|
||||
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, True)[1]
|
||||
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
|
||||
self.assertTrue(prompt.endswith(f"Principles: read {decisions} first, up to `# Decisions`, for the project principles and the decision titles."), prompt)
|
||||
|
||||
def test_fails_without_a_config_for_the_checkout(self):
|
||||
self.config.unlink()
|
||||
self.assertIn("no config names the checkout", replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, True).reason)
|
||||
self.assertIn("no config names the checkout", replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT).reason)
|
||||
|
||||
def test_fails_without_the_sections_to_hand_over(self):
|
||||
(self.home / ".claude" / "CLAUDE.md").write_text("")
|
||||
self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, True), Failure)
|
||||
self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT), Failure)
|
||||
|
||||
|
||||
class Match(unittest.TestCase):
|
||||
@@ -224,6 +224,7 @@ class Main(unittest.TestCase):
|
||||
# The fixture holds one round.
|
||||
self.enterContext(unittest.mock.patch.object(recall, "ENOUGH_ROUNDS", 1))
|
||||
self.reviews = []
|
||||
self.review_settings = []
|
||||
self.agents = set()
|
||||
self.matched_in = []
|
||||
|
||||
@@ -259,6 +260,7 @@ class Main(unittest.TestCase):
|
||||
if "--agent" in args:
|
||||
shown = [subprocess.run(["git", *command], cwd=cwd, capture_output=True, text=True).stdout for command in (["show", "HEAD:a.txt"], ["branch", "--show-current"], ["show", "origin/main:a.txt"])]
|
||||
self.reviews.append((rung, *shown, prompt, cwd))
|
||||
self.review_settings.append(json.loads(args[args.index("--settings") + 1])["env"] if "--settings" in args else None)
|
||||
self.agents.add(args[args.index("--agent") + 1])
|
||||
result = review(rung)
|
||||
else:
|
||||
@@ -292,14 +294,40 @@ class Main(unittest.TestCase):
|
||||
context = reviewer_context(self.home)
|
||||
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False, False, False, True])
|
||||
self.assertEqual(out, (
|
||||
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three with CLAUDE.md, one without.\n"
|
||||
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three with CLAUDE.md, one without CLAUDE.md.\n"
|
||||
"With CLAUDE.md the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
|
||||
"Without CLAUDE.md it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
|
||||
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
|
||||
"It passes: 1 missed is no more than the most a replay with CLAUDE.md missed, 1.\n"
|
||||
))
|
||||
with_them = f"replaying 1 stability rounds with the instruction files on {REFERENCE}\n"
|
||||
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without the instruction files on {REFERENCE}\n")
|
||||
with_them = f"replaying 1 stability rounds with CLAUDE.md on {REFERENCE}\n"
|
||||
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without CLAUDE.md on {REFERENCE}\n")
|
||||
|
||||
def test_a_bash_ceiling_replays_each_round_three_unchanged_and_once_under_the_ceiling_with_nothing_else_changed(self):
|
||||
finds = iter(([0, 1], [0], [0, 1], [1]))
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": next(finds)})
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(self.review_settings, [None, None, None, {"BASH_MAX_OUTPUT_LENGTH": "6000"}])
|
||||
context = reviewer_context(self.home)
|
||||
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False] * 4)
|
||||
self.assertEqual(out, (
|
||||
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three as archived, one under a Bash ceiling of 6,000 characters.\n"
|
||||
"As archived the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
|
||||
"Under a Bash ceiling of 6,000 characters it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
|
||||
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
|
||||
"It passes: 1 missed is no more than the most a replay as archived missed, 1.\n"
|
||||
))
|
||||
|
||||
def test_a_read_limit_hands_the_one_replay_under_it_the_limit_in_tokens(self):
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--read-limit"], lambda rung: {"found": [0]})
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"}])
|
||||
self.assertIn("four times on", out)
|
||||
self.assertEqual(out.splitlines()[-2], "Under a Read limit of 5,000 tokens it found 1 of them, and missed 0 of the 1 findings that both of the first two found.")
|
||||
|
||||
def test_without_claude_md_hands_the_replay_without_it_the_setting_that_turns_the_instruction_files_off(self):
|
||||
self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0]})
|
||||
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}])
|
||||
|
||||
def test_without_claude_md_fails_where_it_missed_more_than_every_replay_with_it(self):
|
||||
finds = iter(([0, 1], [0, 1], [0, 1], []))
|
||||
@@ -326,13 +354,13 @@ class Main(unittest.TestCase):
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0, 1]}, review=review)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(out, (
|
||||
f"Replayed 0 stability rounds of claude-opus-5-5 projects, which hold 0 🔴 and 🟡 findings, four times on {REFERENCE}: three with CLAUDE.md, one without.\n"
|
||||
f"Replayed 0 stability rounds of claude-opus-5-5 projects, which hold 0 🔴 and 🟡 findings, four times on {REFERENCE}: three with CLAUDE.md, one without CLAUDE.md.\n"
|
||||
"Left out 1 more, which a replay failed on; stderr says why.\n"
|
||||
"With CLAUDE.md the three replays found 0, 0 and 0 of them, and missed 0, 0 and 0 of the findings that both of the other two found.\n"
|
||||
"Without CLAUDE.md it found 0 of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to.\n"
|
||||
))
|
||||
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z with the instruction files failed, so the round is left out: claude printed no json reply", err)
|
||||
self.assertNotIn("without the instruction files failed", err)
|
||||
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z with CLAUDE.md failed, so the round is left out: claude printed no json reply", err)
|
||||
self.assertNotIn("without CLAUDE.md failed", err)
|
||||
|
||||
def test_without_claude_md_stops_at_a_usage_limit(self):
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
||||
@@ -584,7 +612,7 @@ class Main(unittest.TestCase):
|
||||
|
||||
def test_prints_the_help(self):
|
||||
code, out, _ = self.run_main(["-h"], unused)
|
||||
self.assertEqual((code, out.splitlines()[2]), (0, "usage: review-replay <pass> <model> [--omit-claude-md]"))
|
||||
self.assertEqual((code, out.splitlines()[2]), (0, "usage: review-replay <pass> <model> [--omit-claude-md | --bash-ceiling | --read-limit]"))
|
||||
self.assertIn(f"{', '.join(list(REVIEWERS)[:-1])} or {list(REVIEWERS)[-1]}.", " ".join(out.split()))
|
||||
|
||||
def test_the_launcher_prints_the_help(self):
|
||||
|
||||
@@ -61,6 +61,7 @@ from rule_hook.round_report import round_report_breaches
|
||||
from command_lines.shell_command import simple_commands
|
||||
from project_config import templates
|
||||
from rule_hook.todo_check import todo_breaches
|
||||
from rule_hook.transcript_read import transcript_breaches
|
||||
|
||||
|
||||
def main(argv: list[str], stdin: str) -> int:
|
||||
@@ -96,7 +97,7 @@ def breaches_of(call: Call, home: str, environ: typing.Mapping[str, str]) -> tup
|
||||
command = text(call.given, "command")
|
||||
return command, command.split("\n", 1)[0], bash_breaches(command, call.cwd, home) + read_breaches(call, home) + env_file_breaches(call, home)
|
||||
if call.tool == "Read":
|
||||
return "", text(call.given, "file_path"), read_breaches(call, home) + env_file_breaches(call, home)
|
||||
return "", text(call.given, "file_path"), read_breaches(call, home) + env_file_breaches(call, home) + transcript_breaches(call, home)
|
||||
if call.tool == "Grep":
|
||||
# Only env-file-read checks a Grep, and it takes no override, so no marker or log line concerns the call.
|
||||
return "", "", env_file_breaches(call, home)
|
||||
|
||||
@@ -12,7 +12,7 @@ from review_rules.review_folder import REPORTS
|
||||
from project_config import templates
|
||||
from command_lines.docker_command import unlimited
|
||||
|
||||
RuleName = typing.Literal["action-pin", "commit-one-line", "commit-title", "docker-run-memory", "env-file-read", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check"]
|
||||
RuleName = typing.Literal["action-pin", "commit-one-line", "commit-title", "docker-run-memory", "env-file-read", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check", "transcript-read"]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -94,6 +94,12 @@ RULES: dict[RuleName, Rule] = {
|
||||
"while todo.md has changes, staged or not, a git commit waits until rank_todo.py --check passes on it: the checkout's own bin/rank_todo.py where the checkout is of this setup's repo, else ~/.claude/bin/rank_todo.py",
|
||||
"a git commit in a checkout whose todo.md has changes, and every git commit where the project's config puts todo.md outside the checkout, where the checkout's commits don't say whether it changed; and a git commit in the config's folder, `~/.claude-projects/<key>/`, whose todo.md has changes",
|
||||
),
|
||||
# A Read carries no text for a marker, so no marker waives it.
|
||||
"transcript-read": Rule(
|
||||
"an agent never reads a session transcript under ~/.claude/projects with Read, since the file holds the whole thread of its session and a read puts all of it in context; it prints the lines it needs with grep or jq; the `tool-results/` files that Claude Code saves a long Bash result to are not transcripts and may be read",
|
||||
"a Read call whose file is a `.jsonl` file under ~/.claude/projects; a Grep or a Bash command that reads one is not checked",
|
||||
overridable=False,
|
||||
),
|
||||
}
|
||||
# The lookahead ends each reason at the next marker on its line, or at the line's end. The reason is lazy, so it stops at the next marker instead of running on and swallowing the markers after it.
|
||||
# An HTML marker's closing `-->` stays in its reason; `overrides` strips it.
|
||||
|
||||
@@ -1493,6 +1493,40 @@ class CommittedFiles(unittest.TestCase):
|
||||
self.assertEqual(self.commit().returncode, 0)
|
||||
|
||||
|
||||
class TranscriptRead(unittest.TestCase):
|
||||
HOME = "/home/h"
|
||||
TRANSCRIPT = "/home/h/.claude/projects/-home-h-code-acme/abc-123.jsonl"
|
||||
|
||||
def breached(self, tool: str, given: dict[str, typing.Any], cwd: str = "/r") -> list[str]:
|
||||
found = breaches_of(Call(tool, given, cwd), self.HOME, {})
|
||||
return [rule for rule in rules(found[2]) if rule == "transcript-read"] if found else []
|
||||
|
||||
def test_a_read_of_a_session_s_transcript_or_a_subagent_s_is_refused(self):
|
||||
self.assertEqual(self.breached("Read", {"file_path": self.TRANSCRIPT}), ["transcript-read"])
|
||||
self.assertEqual(self.breached("Read", {"file_path": "/home/h/.claude/projects/-home-h-code-acme/abc-123/subagents/agent-a1.jsonl"}), ["transcript-read"])
|
||||
|
||||
def test_a_read_spelt_from_home_or_from_the_working_directory_is_refused(self):
|
||||
self.assertEqual(self.breached("Read", {"file_path": "~/.claude/projects/-home-h-code-acme/abc-123.jsonl"}), ["transcript-read"])
|
||||
self.assertEqual(self.breached("Read", {"file_path": "abc-123.jsonl"}, "/home/h/.claude/projects/-home-h-code-acme"), ["transcript-read"])
|
||||
|
||||
def test_the_refusal_names_the_file_and_what_to_use_instead(self):
|
||||
found = breaches_of(Call("Read", {"file_path": self.TRANSCRIPT}, "/r"), self.HOME, {})
|
||||
self.assertEqual([breach.detail for breach in found[2] if breach.rule == "transcript-read"], [f"the call reads {self.TRANSCRIPT}, a session transcript that holds every tool result of its session; print only the lines you need with grep or jq"])
|
||||
|
||||
def test_a_read_of_a_long_result_claude_code_saved_to_a_file_or_of_a_memory_file_passes(self):
|
||||
for path in ("/home/h/.claude/projects/-home-h-code-acme/abc-123/tool-results/b3ij9u144.txt", "/home/h/.claude/projects/-home-h-code-acme/memory/MEMORY.md"):
|
||||
with self.subTest(path):
|
||||
self.assertEqual(self.breached("Read", {"file_path": path}), [])
|
||||
|
||||
def test_a_jsonl_file_outside_the_projects_folder_passes(self):
|
||||
self.assertEqual(self.breached("Read", {"file_path": "/home/h/code/acme/cases.jsonl"}), [])
|
||||
self.assertEqual(self.breached("Read", {"file_path": "/home/h/.claude/loop-claude-transcripts/x.jsonl"}), [])
|
||||
|
||||
def test_a_search_of_a_transcript_is_not_checked(self):
|
||||
self.assertEqual(self.breached("Grep", {"pattern": "x", "path": self.TRANSCRIPT}), [])
|
||||
self.assertEqual(self.breached("Bash", {"command": f"grep x {self.TRANSCRIPT}"}), [])
|
||||
|
||||
|
||||
class RuleNames(unittest.TestCase):
|
||||
def test_every_rule_name_has_a_rule_and_every_rule_a_name(self):
|
||||
self.assertEqual(set(RULES), set(typing.get_args(RuleName)))
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Whether a Read asks for a session transcript under ~/.claude/projects, for the `transcript-read` rule.
|
||||
|
||||
A transcript is a `.jsonl` file there, a session's own or a subagent's. Its size is the session's whole thread, so a Read of one puts it all in context. The files Claude Code saves a long Bash result to, under the session's `tool-results/` folder, are `.txt` files and pass: the model is handed their path to read."""
|
||||
import posixpath
|
||||
|
||||
from rule_hook.call import Call, text
|
||||
from rule_hook.rules import Breach
|
||||
from command_lines.shell_command import resolved
|
||||
|
||||
READS = "the call reads {}, a session transcript that holds every tool result of its session; print only the lines you need with grep or jq"
|
||||
|
||||
|
||||
def transcript_breaches(call: Call, home: str) -> list[Breach]:
|
||||
if call.tool != "Read":
|
||||
return []
|
||||
path = resolved(call.cwd, text(call.given, "file_path"), home)
|
||||
if path.endswith(".jsonl") and path.startswith(posixpath.join(home, ".claude", "projects") + "/"):
|
||||
return [Breach("transcript-read", READS.format(path), call.cwd)]
|
||||
return []
|
||||
@@ -2525,3 +2525,19 @@ Read most (opens and searches):
|
||||
- `bin/loop_claude/cycle/run.py` 4
|
||||
- `bin/loop_claude/loop/close.py` 3
|
||||
- `bin/model_routing/test_review_choice.py` 3
|
||||
|
||||
## Reading cost, chunk cfb3b6b8c710, finished 2026-11-17T08:13:19Z
|
||||
|
||||
182657 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `docs/decisions.md` 13
|
||||
- `README.md` 5
|
||||
- `skills/larv-review/SKILL.md` 4
|
||||
- `CLAUDE.md` 3
|
||||
- `CHANGELOG.md` 1
|
||||
- `bin/loop_claude/cycle/prompts.py` 1
|
||||
- `bin/todo-item` 1
|
||||
- `rules/decisions.md` 1
|
||||
- `todo.md` 1
|
||||
|
||||
Reference in New Issue
Block a user