Answer stability round 1: 0 🔴, 2 🟡 (0 misread); refuse a change whose variable the environment holds, and pin the refusal's path

This commit is contained in:
2026-11-18 15:54:24 +01:00
parent 897fe3ed1a
commit 033989f6be
3 changed files with 26 additions and 11 deletions
+14 -8
View File
@@ -45,16 +45,19 @@ replay, and runs no SubagentStart hook there. So the replay without CLAUDE.md tu
instruction files off in its settings and opens the prompt with what the hook hands a reviewer, and
closes it with the header's line on the project principles.
--bash-ceiling and --read-limit run the same comparison for the output limits loop-claude hands the worker's
claude, which a subagent inherits (bin/claude_code/output_limits.py): the replay with the change starts under
BASH_MAX_OUTPUT_LENGTH, which saves a longer Bash result to a file and hands the reviewer its path and the
first 2,000 characters, or under CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS, which fails a Read of more tokens
than that. It passes where it missed no more of the findings both of the first two found than the most an
unchanged replay missed. It doesn't count what the replays read: each run's stream, with every tool result,
is in the archive.
--bash-ceiling and --read-limit run the same comparison for two limits on what one tool result adds to a
claude's context, which a subagent inherits (bin/claude_code/output_limits.py). loop-claude hands the worker's
claude the Bash ceiling, and doesn't hand it the Read limit yet. The replay with --bash-ceiling starts under
BASH_MAX_OUTPUT_LENGTH=6000, which saves a Bash result of more than 6,000 characters to a file and hands the
reviewer its path and the first 2,000 characters. The replay with --read-limit starts under
CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS=5000, which fails a Read of more than 5,000 tokens. The replay with
the limit passes where it missed no more of the findings both of the first two found than the most an
unchanged replay missed. The tool doesn't count what the replays read: each run's stream, with every tool
result, is in the archive. It refuses to run where its own environment already holds the variable, since the
replays as archived would inherit it.
It keeps each run, a reviewer's tool calls included, and the report it prints in loop-claude's archive, in
replays/review-replay-<pass>-<model>[-<flag without its dashes>]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
replays/review-replay-<pass>-<model>[-<flag without its leading dashes>]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
or $LOOP_CLAUDE_TRANSCRIPTS. A run's .jsonl file starts with a line holding claude's arguments, working
directory, prompt, exit code and stderr, then holds claude's stream-json output; result.md holds the report.
@@ -96,6 +99,9 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
if isinstance(read, int):
return read
reviewed, project, change = read
# The replays as archived start under this environment too, so a change it already holds would reach all four.
if change and (inherited := [name for name in change.env if name in env]):
return failed(TOOL, f"the replays as archived would inherit {', '.join(inherited)} from this environment, and the comparison needs them without it: unset it and run again")
rounds = archived_rounds(REVIEWERS[reviewed], project.model, archive_root(home, env), home)
if isinstance(rounds, Failure):
return failed(TOOL, rounds.reason)
+9 -2
View File
@@ -252,7 +252,7 @@ class Main(unittest.TestCase):
def set_prompt(self, prompt):
self.rewrite_round(lambda first: first["message"]["content"][0].update(text=prompt))
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}):
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}, env={}):
"""found(rung) is the indexes of the recorded findings the reviewer on rung finds; review(rung), that reviewer's reply."""
def claude(args, cwd, prompt):
@@ -273,7 +273,7 @@ class Main(unittest.TestCase):
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=system_effects().git, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, sleep=unused, stream_claude=unused, tmux=unused, tmux_on_terminal=unused)
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = main.main(argv, {}, self.home, effects)
code = main.main(argv, env, self.home, effects)
return code, out.getvalue(), err.getvalue()
def test_keeps_each_reviewer_and_match_run_and_the_printed_report_in_one_folder_of_the_archive(self):
@@ -303,6 +303,13 @@ class Main(unittest.TestCase):
with_them = f"replaying 1 stability rounds with CLAUDE.md on {REFERENCE}\n"
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without CLAUDE.md on {REFERENCE}\n")
def test_a_change_is_refused_where_the_replays_would_inherit_its_variable_from_the_environment(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"BASH_MAX_OUTPUT_LENGTH": "6000"})
self.assertEqual((code, out, self.reviews), (1, "", []))
self.assertIn("BASH_MAX_OUTPUT_LENGTH", err)
code, _, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"})
self.assertEqual(code, 0, err)
def test_a_bash_ceiling_replays_each_round_three_unchanged_and_once_under_the_ceiling_with_nothing_else_changed(self):
finds = iter(([0, 1], [0], [0, 1], [1]))
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": next(finds)})
+3 -1
View File
@@ -1511,7 +1511,9 @@ class TranscriptRead(unittest.TestCase):
def test_the_refusal_names_the_file_and_what_to_use_instead(self):
found = breaches_of(Call("Read", {"file_path": self.TRANSCRIPT}, "/r"), self.HOME, {})
self.assertEqual([breach.detail for breach in found[2] if breach.rule == "transcript-read"], [f"the call reads {self.TRANSCRIPT}, a session transcript that holds every tool result of its session; print only the lines you need with grep or jq"])
[detail] = [breach.detail for breach in found[2] if breach.rule == "transcript-read"]
self.assertIn(self.TRANSCRIPT, detail)
self.assertIn("grep or jq", detail)
def test_the_refusal_is_logged_in_the_checkout_of_the_working_directory(self):
found = breaches_of(Call("Read", {"file_path": self.TRANSCRIPT}, "/r"), self.HOME, {})