Answer stability round 1: 0 🔴, 2 🟡 (0 misread); refuse a change whose variable the environment holds, and pin the refusal's path
This commit is contained in:
@@ -45,16 +45,19 @@ replay, and runs no SubagentStart hook there. So the replay without CLAUDE.md tu
|
||||
instruction files off in its settings and opens the prompt with what the hook hands a reviewer, and
|
||||
closes it with the header's line on the project principles.
|
||||
|
||||
--bash-ceiling and --read-limit run the same comparison for the output limits loop-claude hands the worker's
|
||||
claude, which a subagent inherits (bin/claude_code/output_limits.py): the replay with the change starts under
|
||||
BASH_MAX_OUTPUT_LENGTH, which saves a longer Bash result to a file and hands the reviewer its path and the
|
||||
first 2,000 characters, or under CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS, which fails a Read of more tokens
|
||||
than that. It passes where it missed no more of the findings both of the first two found than the most an
|
||||
unchanged replay missed. It doesn't count what the replays read: each run's stream, with every tool result,
|
||||
is in the archive.
|
||||
--bash-ceiling and --read-limit run the same comparison for two limits on what one tool result adds to a
|
||||
claude's context, which a subagent inherits (bin/claude_code/output_limits.py). loop-claude hands the worker's
|
||||
claude the Bash ceiling, and doesn't hand it the Read limit yet. The replay with --bash-ceiling starts under
|
||||
BASH_MAX_OUTPUT_LENGTH=6000, which saves a Bash result of more than 6,000 characters to a file and hands the
|
||||
reviewer its path and the first 2,000 characters. The replay with --read-limit starts under
|
||||
CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS=5000, which fails a Read of more than 5,000 tokens. The replay with
|
||||
the limit passes where it missed no more of the findings both of the first two found than the most an
|
||||
unchanged replay missed. The tool doesn't count what the replays read: each run's stream, with every tool
|
||||
result, is in the archive. It refuses to run where its own environment already holds the variable, since the
|
||||
replays as archived would inherit it.
|
||||
|
||||
It keeps each run, a reviewer's tool calls included, and the report it prints in loop-claude's archive, in
|
||||
replays/review-replay-<pass>-<model>[-<flag without its dashes>]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
|
||||
replays/review-replay-<pass>-<model>[-<flag without its leading dashes>]-<UTC stamp>/ under ~/.claude/loop-claude-transcripts
|
||||
or $LOOP_CLAUDE_TRANSCRIPTS. A run's .jsonl file starts with a line holding claude's arguments, working
|
||||
directory, prompt, exit code and stderr, then holds claude's stream-json output; result.md holds the report.
|
||||
|
||||
@@ -96,6 +99,9 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
|
||||
if isinstance(read, int):
|
||||
return read
|
||||
reviewed, project, change = read
|
||||
# The replays as archived start under this environment too, so a change it already holds would reach all four.
|
||||
if change and (inherited := [name for name in change.env if name in env]):
|
||||
return failed(TOOL, f"the replays as archived would inherit {', '.join(inherited)} from this environment, and the comparison needs them without it: unset it and run again")
|
||||
rounds = archived_rounds(REVIEWERS[reviewed], project.model, archive_root(home, env), home)
|
||||
if isinstance(rounds, Failure):
|
||||
return failed(TOOL, rounds.reason)
|
||||
|
||||
@@ -252,7 +252,7 @@ class Main(unittest.TestCase):
|
||||
def set_prompt(self, prompt):
|
||||
self.rewrite_round(lambda first: first["message"]["content"][0].update(text=prompt))
|
||||
|
||||
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}):
|
||||
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}, env={}):
|
||||
"""found(rung) is the indexes of the recorded findings the reviewer on rung finds; review(rung), that reviewer's reply."""
|
||||
|
||||
def claude(args, cwd, prompt):
|
||||
@@ -273,7 +273,7 @@ class Main(unittest.TestCase):
|
||||
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=system_effects().git, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, sleep=unused, stream_claude=unused, tmux=unused, tmux_on_terminal=unused)
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
code = main.main(argv, {}, self.home, effects)
|
||||
code = main.main(argv, env, self.home, effects)
|
||||
return code, out.getvalue(), err.getvalue()
|
||||
|
||||
def test_keeps_each_reviewer_and_match_run_and_the_printed_report_in_one_folder_of_the_archive(self):
|
||||
@@ -303,6 +303,13 @@ class Main(unittest.TestCase):
|
||||
with_them = f"replaying 1 stability rounds with CLAUDE.md on {REFERENCE}\n"
|
||||
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without CLAUDE.md on {REFERENCE}\n")
|
||||
|
||||
def test_a_change_is_refused_where_the_replays_would_inherit_its_variable_from_the_environment(self):
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"BASH_MAX_OUTPUT_LENGTH": "6000"})
|
||||
self.assertEqual((code, out, self.reviews), (1, "", []))
|
||||
self.assertIn("BASH_MAX_OUTPUT_LENGTH", err)
|
||||
code, _, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"})
|
||||
self.assertEqual(code, 0, err)
|
||||
|
||||
def test_a_bash_ceiling_replays_each_round_three_unchanged_and_once_under_the_ceiling_with_nothing_else_changed(self):
|
||||
finds = iter(([0, 1], [0], [0, 1], [1]))
|
||||
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": next(finds)})
|
||||
|
||||
@@ -1511,7 +1511,9 @@ class TranscriptRead(unittest.TestCase):
|
||||
|
||||
def test_the_refusal_names_the_file_and_what_to_use_instead(self):
|
||||
found = breaches_of(Call("Read", {"file_path": self.TRANSCRIPT}, "/r"), self.HOME, {})
|
||||
self.assertEqual([breach.detail for breach in found[2] if breach.rule == "transcript-read"], [f"the call reads {self.TRANSCRIPT}, a session transcript that holds every tool result of its session; print only the lines you need with grep or jq"])
|
||||
[detail] = [breach.detail for breach in found[2] if breach.rule == "transcript-read"]
|
||||
self.assertIn(self.TRANSCRIPT, detail)
|
||||
self.assertIn("grep or jq", detail)
|
||||
|
||||
def test_the_refusal_is_logged_in_the_checkout_of_the_working_directory(self):
|
||||
found = breaches_of(Call("Read", {"file_path": self.TRANSCRIPT}, "/r"), self.HOME, {})
|
||||
|
||||
Reference in New Issue
Block a user