Answer stability round 1: 0 🔴, 2 🟡 (0 misread); tie the help's transcript floor to TRANSCRIPT_FLOOR, split the floor test, and name the shortfall in the judge's failure, with the prose round's wording
This commit is contained in:
@@ -20,7 +20,7 @@ from claude_code.tokens import tokens_of
|
||||
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
JUDGE_PROMPT_CHARS = 400000
|
||||
# The least of the transcript's tail the judge reads, about 15,000 tokens. Where the head leaves less, the judge stops, since a cut record could hide the rule that makes it interrupt.
|
||||
# The judge reads the transcript's last 50,000 characters at least, about 15,000 tokens, or all of a shorter one. Where the rest of its prompt leaves less room, the judge doesn't run, since a cut record could hide the rule that makes it interrupt.
|
||||
TRANSCRIPT_FLOOR = 50000
|
||||
INTERRUPT_RULES = (
|
||||
"""Decide whether the human must now be interrupted. ~/.claude/CLAUDE.md → "Working loop", "Ask, don't guess" says """
|
||||
@@ -298,7 +298,7 @@ def answer_checked(verdict: Verdict, quotable: str) -> Verdict:
|
||||
|
||||
|
||||
def judge_prompt(brief: Brief) -> tuple[str, Grounds] | Failure:
|
||||
"""The prompt, and what the judge's quotes are checked against; a Failure where the rest leaves less of the transcript's tail than TRANSCRIPT_FLOOR."""
|
||||
"""The prompt, and what the judge's quotes are checked against; a Failure where the premises, the records, the task and the rules leave room for less than the transcript's last TRANSCRIPT_FLOOR characters, or all of a shorter transcript."""
|
||||
head = (
|
||||
f"From the goals file, {brief.premises}\n\n"
|
||||
f"The records: {brief.records_named}:\n{brief.records or 'none'}\n\n"
|
||||
@@ -306,10 +306,11 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds] | Failure:
|
||||
f"{transcript_heading(brief.driver)}"
|
||||
)
|
||||
rules = judge_rules(brief.answer_destinations, brief.driver)
|
||||
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(rules) - 2)
|
||||
room = JUDGE_PROMPT_CHARS - len(head) - len(rules) - 2
|
||||
transcript = escalation_mark.transcript(brief.entries, brief.escalated_at, typed_answer, brief.driver)
|
||||
if room < min(len(transcript), TRANSCRIPT_FLOOR):
|
||||
return Failure(f"the premises, the records and the task leave the transcript {room:,} characters of the judge's {JUDGE_PROMPT_CHARS:,}, under the {TRANSCRIPT_FLOOR:,} of its tail it must read; the records take {len(brief.records):,}. Shorten them.")
|
||||
needed = min(len(transcript), TRANSCRIPT_FLOOR)
|
||||
if room < needed:
|
||||
return Failure(f"the premises, the records, the task and the judge's rules leave {max(0, room):,} of the judge's {JUDGE_PROMPT_CHARS:,} characters for the transcript, less than the last {needed:,} characters of it that the judge must read. Cut the records, which take {len(brief.records):,}, by at least {needed - room:,}.")
|
||||
# The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle.
|
||||
shown = transcript[max(0, len(transcript) - room):]
|
||||
grounds = Grounds(
|
||||
|
||||
@@ -149,22 +149,39 @@ class JudgePrompt(unittest.TestCase):
|
||||
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
||||
self.assertEqual(grounds.quotable, "## Goals\n\n")
|
||||
|
||||
def test_stops_rather_than_show_less_of_the_transcripts_tail_than_its_floor_whatever_the_size_of_the_records(self):
|
||||
entries = [assistant({"type": "text", "text": "old " + "x" * judge.TRANSCRIPT_FLOOR}), assistant({"type": "text", "text": "Merged."})]
|
||||
too_long = self.brief(entries, 0, records="r" * judge.JUDGE_PROMPT_CHARS)
|
||||
self.assertEqual(judge.judge_prompt(too_long), Failure(f"the premises, the records and the task leave the transcript 0 characters of the judge's 400,000, under the {judge.TRANSCRIPT_FLOOR:,} of its tail it must read; the records take 400,000. Shorten them."))
|
||||
self.assertEqual(judge.judge_once("/r", too_long, Rung("judge-model"), FakeEffects(claude=[]).effects()), judge.judge_prompt(too_long))
|
||||
prompt, grounds = judge.judge_prompt(self.brief(entries, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - judge.TRANSCRIPT_FLOOR - 10000)))
|
||||
# A transcript longer than the floor: an old message, then the last one.
|
||||
LONG = [assistant({"type": "text", "text": "old " + "x" * judge.TRANSCRIPT_FLOOR}), assistant({"type": "text", "text": "Merged."})]
|
||||
|
||||
def overhead(self):
|
||||
"""The characters of the judge's prompt that are neither the records nor the transcript."""
|
||||
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r"))
|
||||
return len(prompt) - len(grounds.shown) - 1
|
||||
|
||||
def test_fails_naming_the_shortfall_where_the_records_leave_less_than_the_transcripts_floor(self):
|
||||
failed = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS + 1)))
|
||||
self.assertIsInstance(failed, Failure)
|
||||
for fact in ("400,000", "400,001", f"{judge.TRANSCRIPT_FLOOR:,}", f"by at least {judge.TRANSCRIPT_FLOOR + self.overhead() + 1:,}"):
|
||||
self.assertIn(fact, failed.reason)
|
||||
|
||||
def test_judge_once_returns_that_failure_without_starting_claude(self):
|
||||
failed = judge.judge_once("/r", self.brief(self.LONG, 0, records="r" * judge.JUDGE_PROMPT_CHARS), Rung("judge-model"), FakeEffects(claude=[]).effects())
|
||||
self.assertIsInstance(failed, Failure)
|
||||
self.assertIn(f"{judge.TRANSCRIPT_FLOOR:,}", failed.reason)
|
||||
|
||||
def test_shows_more_than_the_floor_of_the_transcripts_tail_where_the_records_leave_room(self):
|
||||
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - judge.TRANSCRIPT_FLOOR - 10000)))
|
||||
self.assertTrue(grounds.shown.endswith("\n\nMerged."))
|
||||
self.assertGreaterEqual(len(grounds.shown), judge.TRANSCRIPT_FLOOR)
|
||||
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
||||
room = judge.JUDGE_PROMPT_CHARS - (len(prompt) - len(grounds.shown))
|
||||
at_floor = judge.JUDGE_PROMPT_CHARS - 10000 + room - 2 * judge.TRANSCRIPT_FLOOR
|
||||
self.assertEqual(len(judge.judge_prompt(self.brief(entries, 0, records="r" * at_floor))[1].shown), judge.TRANSCRIPT_FLOOR, "a head that leaves exactly the floor still shows it")
|
||||
self.assertIsInstance(judge.judge_prompt(self.brief(entries, 0, records="r" * (at_floor + 1))), Failure)
|
||||
short = [assistant({"type": "text", "text": "Merged."})]
|
||||
_, grounds = judge.judge_prompt(self.brief(short, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - 10000)))
|
||||
self.assertEqual(grounds.shown, "Merged.", "a transcript shorter than the floor needs only its own length")
|
||||
|
||||
def test_shows_the_floor_where_the_records_leave_exactly_that(self):
|
||||
at_floor = judge.JUDGE_PROMPT_CHARS - self.overhead() - judge.TRANSCRIPT_FLOOR
|
||||
self.assertEqual(len(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * at_floor))[1].shown), judge.TRANSCRIPT_FLOOR)
|
||||
self.assertIsInstance(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (at_floor + 1))), Failure)
|
||||
|
||||
def test_shows_a_transcript_shorter_than_the_floor_whole(self):
|
||||
_, grounds = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "Merged."})], 0, records="r" * (judge.JUDGE_PROMPT_CHARS - 10000)))
|
||||
self.assertEqual(grounds.shown, "Merged.")
|
||||
|
||||
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
|
||||
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 1))
|
||||
|
||||
@@ -22,7 +22,7 @@ CONSEQUENCE = "the judge cannot read the records"
|
||||
|
||||
|
||||
def read_records(root: str | None, home: pathlib.Path, claude_dir: pathlib.Path, first: tuple[tuple[str, str], ...] = ()) -> str | Failure:
|
||||
"""The project's `AGENTS.md` and decision log, then the human's rules, CLAUDE.md first, each headed by its name; one that does not exist, or is empty, is left out, and one with the same text as an earlier one is read once, under both names.
|
||||
"""The project's `AGENTS.md` and decision log, then the human's rules, CLAUDE.md first, each headed by its name. One that does not exist, or is empty, is left out. A text that several of them hold appears once, headed by all their names.
|
||||
|
||||
`AGENTS.md` is read where the config under home of root's project puts it, and the decision log from the store that config names. Each is headed by its name in RECORD_FILES wherever it is kept.
|
||||
|
||||
@@ -37,7 +37,7 @@ def read_records(root: str | None, home: pathlib.Path, claude_dir: pathlib.Path,
|
||||
decisions = store(config, DECISIONS[1]).read()
|
||||
texts += [(WORKING_RULES[0], file_text(config.paths[WORKING_RULES[1]])), (DECISIONS[0], Failure(f"{decisions.reason} - {CONSEQUENCE}.") if isinstance(decisions, Failure) else decisions)]
|
||||
texts += [(named(rule, home, claude_dir), file_text(rule)) for rule in rules]
|
||||
# Each text, with the headings it is read under: a config may put the working rules in the decision log.
|
||||
# Each text, with the names of the records that hold it: a config may keep the working rules in the decision log.
|
||||
records: dict[str, list[str]] = {}
|
||||
for heading, text in texts:
|
||||
if isinstance(text, Failure):
|
||||
|
||||
@@ -41,7 +41,7 @@ class Records(unittest.TestCase):
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).endswith(f"From {root}/standards/quality.md:\n# Elsewhere"), "a folder outside home is named in full")
|
||||
|
||||
config.write_text(config.read_text().replace('"checkout:AGENTS.md"', '"checkout:docs/decisions.md"'))
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From AGENTS.md and docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global"), "a record the config names twice is read once")
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From AGENTS.md and docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global"), "a file the config names for two records appears once")
|
||||
config.write_text(config.read_text().replace('"checkout:docs/decisions.md"', '"checkout:AGENTS.md"', 1))
|
||||
|
||||
config.write_text(config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"').replace('"checkout:docs/decisions.md"', '"folder:log/decisions.md"'))
|
||||
|
||||
@@ -101,7 +101,7 @@ class JudgeFailures(unittest.TestCase):
|
||||
("run_judge", "verdict.reason", field),
|
||||
("run_judge", "checked.reason", "quotes a passage that neither the task nor the transcript holds"),
|
||||
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the task or the transcript", "quotes a passage that neither the task nor the transcript holds"),
|
||||
("judge_prompt", "f\"the premises, the records and the task leave the transcript", "leave it room for less than the last 50,000 characters of the transcript"),
|
||||
("judge_prompt", "f\"the premises, the records, the task and the judge's rules leave", f"leave room for less than the last {judge.TRANSCRIPT_FLOOR:,} characters of the transcript, or all of a shorter one"),
|
||||
("claude_reply", "f'could not start claude:", started),
|
||||
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
|
||||
("claude_reply", 'f"{who} printed no json reply', unparsed),
|
||||
|
||||
@@ -186,11 +186,12 @@ Exit 1, saying why, when:
|
||||
handover
|
||||
- the worker or the judge fails three times running, retried a minute apart within the cycle. A
|
||||
worker fails when claude cannot start, exits non-zero, is killed, or ends without a result event;
|
||||
a judge when the premises, the records and the task leave it room for less than the last 50,000
|
||||
characters of the transcript, when claude cannot start, exits non-zero or prints no json reply, or
|
||||
when its reply misses or mistypes a field, interrupts or quotes an answer without a question, or
|
||||
quotes a passage that neither the task nor the transcript holds, where each fragment of an elided
|
||||
quote must appear in order, in the task then the transcript.
|
||||
a judge when the premises, the records, the task and the judge's rules leave room for less than
|
||||
the last 50,000 characters of the transcript, or all of a shorter one, when claude cannot start,
|
||||
exits non-zero or prints no json reply, or when its reply misses or mistypes a field, interrupts
|
||||
or quotes an answer without a question, or quotes a passage that neither the task nor the
|
||||
transcript holds, where each fragment of an elided quote must appear in order, in the task then
|
||||
the transcript.
|
||||
A usage limit is not a failure, and restarts the count of three: the worker or the judge is
|
||||
retried 2 minutes after the limit resets, or every 10 minutes when the reset time is unknown or
|
||||
more than 2 minutes past. The judge's reply gives its reset time only as text, so the judge is
|
||||
|
||||
@@ -117,7 +117,7 @@ class StreamingClaude:
|
||||
def system_effects(home: pathlib.Path, env: typing.Mapping[str, str], backends: typing.Mapping[Notifier, Backend[typing.Any]] = BACKENDS, forge_timeout: float = FORGE_TIMEOUT) -> Effects:
|
||||
"""The effects on this machine: the configs of its DM are under home, and tmux runs in env.
|
||||
|
||||
A test hands in `backends`, the notifiers loop-claude's DM can go through, that send nowhere, and a `forge_timeout` short enough to wait out.
|
||||
A test hands in `backends` that don't send anything, in place of the notifiers loop-claude's DM goes through. It hands in a short `forge_timeout`, so a command that outlasts it ends quickly.
|
||||
"""
|
||||
return Effects(
|
||||
claude=lambda args, cwd, stdin: subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=stdin, text=True),
|
||||
|
||||
@@ -110,7 +110,7 @@ def awaited_merges(found: list[Item], release: Release) -> dict[str, list[int]]:
|
||||
|
||||
|
||||
def merge_stop(pulls: dict[str, list[int]]) -> str:
|
||||
"""The sentences that stop the working loop where pulls await a person's merge, naming each with the items it finishes; empty where none does."""
|
||||
"""The sentences that stop the working loop, naming each of pulls, the pull requests that await a person's merge, with the items it finishes; empty where pulls is empty."""
|
||||
if not pulls:
|
||||
return ""
|
||||
named = "; ".join(f"{pull}, which finishes item{'s' if len(ids) > 1 else ''} {listed([str(item_id) for item_id in ids], 'and')}" for pull, ids in pulls.items())
|
||||
|
||||
Reference in New Issue
Block a user