Stop the judge rather than show it under 50,000 characters of the transcript, read a twice-named record once, test the forge timeout and decide the merge stop from the pull requests found
This commit is contained in:
@@ -20,6 +20,8 @@ from claude_code.tokens import tokens_of
|
||||
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
JUDGE_PROMPT_CHARS = 400000
|
||||
# The least of the transcript's tail the judge reads, about 15,000 tokens. Where the head leaves less, the judge stops, since a cut record could hide the rule that makes it interrupt.
|
||||
TRANSCRIPT_FLOOR = 50000
|
||||
INTERRUPT_RULES = (
|
||||
"""Decide whether the human must now be interrupted. ~/.claude/CLAUDE.md → "Working loop", "Ask, don't guess" says """
|
||||
"which choices are reserved for the human. interrupt=true when any of these appear:\n"
|
||||
@@ -186,7 +188,10 @@ def judge(root: str, brief: Brief, rung: Rung, effects: Effects, keep: typing.Ca
|
||||
|
||||
def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
|
||||
"""One judge run on the brief, as judge retries it; judge-replay replays it."""
|
||||
prompt, grounds = judge_prompt(brief)
|
||||
built = judge_prompt(brief)
|
||||
if isinstance(built, Failure):
|
||||
return built
|
||||
prompt, grounds = built
|
||||
verdict = run_judge(root, judge_args(rung, brief.driver), prompt, grounds, effects)
|
||||
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
|
||||
|
||||
@@ -292,8 +297,8 @@ def answer_checked(verdict: Verdict, quotable: str) -> Verdict:
|
||||
return verdict
|
||||
|
||||
|
||||
def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
|
||||
"""The prompt, and what the judge's quotes are checked against."""
|
||||
def judge_prompt(brief: Brief) -> tuple[str, Grounds] | Failure:
|
||||
"""The prompt, and what the judge's quotes are checked against; a Failure where the rest leaves less of the transcript's tail than TRANSCRIPT_FLOOR."""
|
||||
head = (
|
||||
f"From the goals file, {brief.premises}\n\n"
|
||||
f"The records: {brief.records_named}:\n{brief.records or 'none'}\n\n"
|
||||
@@ -303,6 +308,8 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
|
||||
rules = judge_rules(brief.answer_destinations, brief.driver)
|
||||
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(rules) - 2)
|
||||
transcript = escalation_mark.transcript(brief.entries, brief.escalated_at, typed_answer, brief.driver)
|
||||
if room < min(len(transcript), TRANSCRIPT_FLOOR):
|
||||
return Failure(f"the premises, the records and the task leave the transcript {room:,} characters of the judge's {JUDGE_PROMPT_CHARS:,}, under the {TRANSCRIPT_FLOOR:,} of its tail it must read; the records take {len(brief.records):,}. Shorten them.")
|
||||
# The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle.
|
||||
shown = transcript[max(0, len(transcript) - room):]
|
||||
grounds = Grounds(
|
||||
|
||||
@@ -135,9 +135,9 @@ class Verdict(unittest.TestCase):
|
||||
|
||||
|
||||
class JudgePrompt(unittest.TestCase):
|
||||
def brief(self, entries, escalated_at, answer_destinations="the plan or a comment"):
|
||||
def brief(self, entries, escalated_at, answer_destinations="the plan or a comment", records=""):
|
||||
"""What the judge of a-driver reads of a transcript holding entries."""
|
||||
return judge.Brief("Do it.", "## Goals", "", "the project's decisions", answer_destinations, kinds(*entries), escalated_at, "a-driver")
|
||||
return judge.Brief("Do it.", "## Goals", records, "the project's decisions", answer_destinations, kinds(*entries), escalated_at, "a-driver")
|
||||
|
||||
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
||||
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
||||
@@ -149,6 +149,19 @@ class JudgePrompt(unittest.TestCase):
|
||||
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
||||
self.assertEqual(grounds.quotable, "## Goals\n\n")
|
||||
|
||||
def test_stops_rather_than_show_less_of_the_transcripts_tail_than_its_floor_whatever_the_size_of_the_records(self):
|
||||
entries = [assistant({"type": "text", "text": "old " + "x" * judge.TRANSCRIPT_FLOOR}), assistant({"type": "text", "text": "Merged."})]
|
||||
too_long = self.brief(entries, 0, records="r" * judge.JUDGE_PROMPT_CHARS)
|
||||
self.assertEqual(judge.judge_prompt(too_long), Failure(f"the premises, the records and the task leave the transcript 0 characters of the judge's 400,000, under the {judge.TRANSCRIPT_FLOOR:,} of its tail it must read; the records take 400,000. Shorten them."))
|
||||
self.assertEqual(judge.judge_once("/r", too_long, Rung("judge-model"), FakeEffects(claude=[]).effects()), judge.judge_prompt(too_long))
|
||||
prompt, grounds = judge.judge_prompt(self.brief(entries, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - judge.TRANSCRIPT_FLOOR - 10000)))
|
||||
self.assertTrue(grounds.shown.endswith("\n\nMerged."))
|
||||
self.assertGreaterEqual(len(grounds.shown), judge.TRANSCRIPT_FLOOR)
|
||||
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
||||
short = [assistant({"type": "text", "text": "Merged."})]
|
||||
_, grounds = judge.judge_prompt(self.brief(short, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - 10000)))
|
||||
self.assertEqual(grounds.shown, "Merged.", "a transcript shorter than the floor needs only its own length")
|
||||
|
||||
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
|
||||
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 1))
|
||||
self.assertNotRegex(prompt + json.dumps(judge.judge_schema("a-driver")), r"\b[Aa]gent")
|
||||
|
||||
@@ -22,7 +22,7 @@ CONSEQUENCE = "the judge cannot read the records"
|
||||
|
||||
|
||||
def read_records(root: str | None, home: pathlib.Path, claude_dir: pathlib.Path, first: tuple[tuple[str, str], ...] = ()) -> str | Failure:
|
||||
"""The project's `AGENTS.md` and decision log, then the human's rules, CLAUDE.md first, each headed by its name; one that does not exist, or is empty, is left out.
|
||||
"""The project's `AGENTS.md` and decision log, then the human's rules, CLAUDE.md first, each headed by its name; one that does not exist, or is empty, is left out, and one whose text an earlier one holds is read once, under both names.
|
||||
|
||||
`AGENTS.md` is read where the config under home of root's project puts it, and the decision log from the store that config names. Each is headed by its name in RECORD_FILES wherever it is kept.
|
||||
|
||||
@@ -37,13 +37,14 @@ def read_records(root: str | None, home: pathlib.Path, claude_dir: pathlib.Path,
|
||||
decisions = store(config, DECISIONS[1]).read()
|
||||
texts += [(WORKING_RULES[0], file_text(config.paths[WORKING_RULES[1]])), (DECISIONS[0], Failure(f"{decisions.reason} - {CONSEQUENCE}.") if isinstance(decisions, Failure) else decisions)]
|
||||
texts += [(named(rule, home, claude_dir), file_text(rule)) for rule in rules]
|
||||
records = []
|
||||
# Each text, with the headings it is read under: a config may put the working rules in the decision log.
|
||||
records: dict[str, list[str]] = {}
|
||||
for heading, text in texts:
|
||||
if isinstance(text, Failure):
|
||||
return text
|
||||
if text.strip():
|
||||
records.append(f"From {heading}:\n{text.strip()}")
|
||||
return "\n\n".join(records)
|
||||
records.setdefault(text.strip(), []).append(heading)
|
||||
return "\n\n".join(f"From {' and '.join(headings)}:\n{text}" for text, headings in records.items())
|
||||
|
||||
|
||||
def file_text(path: pathlib.Path) -> str | Failure:
|
||||
|
||||
@@ -40,6 +40,10 @@ class Records(unittest.TestCase):
|
||||
pathlib.Path(root, "standards", "quality.md").write_text("# Elsewhere\n")
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).endswith(f"From {root}/standards/quality.md:\n# Elsewhere"), "a folder outside home is named in full")
|
||||
|
||||
config.write_text(config.read_text().replace('"checkout:AGENTS.md"', '"checkout:docs/decisions.md"'))
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From AGENTS.md and docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global"), "a record the config names twice is read once")
|
||||
config.write_text(config.read_text().replace('"checkout:docs/decisions.md"', '"checkout:AGENTS.md"', 1))
|
||||
|
||||
config.write_text(config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"').replace('"checkout:docs/decisions.md"', '"folder:log/decisions.md"'))
|
||||
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From ~/.claude/CLAUDE.md:\n# Global"), "the checkout's AGENTS.md and docs/decisions.md are its owners'")
|
||||
pathlib.Path(config.parent, "log").mkdir()
|
||||
|
||||
@@ -114,10 +114,10 @@ class StreamingClaude:
|
||||
return self.proc.wait()
|
||||
|
||||
|
||||
def system_effects(home: pathlib.Path, env: typing.Mapping[str, str], backends: typing.Mapping[Notifier, Backend[typing.Any]] = BACKENDS) -> Effects:
|
||||
def system_effects(home: pathlib.Path, env: typing.Mapping[str, str], backends: typing.Mapping[Notifier, Backend[typing.Any]] = BACKENDS, forge_timeout: float = FORGE_TIMEOUT) -> Effects:
|
||||
"""The effects on this machine: the configs of its DM are under home, and tmux runs in env.
|
||||
|
||||
A test hands in `backends`, the notifiers loop-claude's DM can go through, that send nowhere.
|
||||
A test hands in `backends`, the notifiers loop-claude's DM can go through, that send nowhere, and a `forge_timeout` short enough to wait out.
|
||||
"""
|
||||
return Effects(
|
||||
claude=lambda args, cwd, stdin: subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=stdin, text=True),
|
||||
@@ -125,7 +125,7 @@ def system_effects(home: pathlib.Path, env: typing.Mapping[str, str], backends:
|
||||
discard_typed=lambda: termios.tcflush(sys.stdin, termios.TCIFLUSH),
|
||||
dm=lambda text, directory: notify(text, directory, home, backends),
|
||||
execv=os.execv,
|
||||
forge=lambda cwd: runner(cwd, FORGE_TIMEOUT),
|
||||
forge=lambda cwd: runner(cwd, forge_timeout),
|
||||
git=git_repo.git,
|
||||
input=input,
|
||||
local_time=lambda seconds: time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds)),
|
||||
|
||||
@@ -13,6 +13,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from failure import Failure
|
||||
from loop_effects import child_shells
|
||||
from loop_effects.effects import run_python, say, start_timer, StreamingClaude, system_effects
|
||||
|
||||
@@ -104,6 +105,10 @@ class ForgeEffect(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
self.assertEqual(system_effects(pathlib.Path.home(), os.environ).forge(folder)(["pwd", "-P"]), os.path.realpath(folder) + "\n")
|
||||
|
||||
def test_stops_a_command_of_the_forges_cli_that_outlasts_its_timeout(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
self.assertEqual(system_effects(pathlib.Path.home(), os.environ, forge_timeout=0.1).forge(folder)(["sleep", "5"]), Failure("sleep 5: stopped after 0.1 s"))
|
||||
|
||||
|
||||
class TmuxEffect(unittest.TestCase):
|
||||
"""Each test runs a fake tmux off its own PATH, without $TMUX, so the human's tmux server is never reached."""
|
||||
|
||||
@@ -84,8 +84,8 @@ def planned(read: Plan | Failure, cut: LastCut, default: str, branch: str, outsi
|
||||
return NoItem(f"{where} holds no item for {upcoming}, and no change is unreleased, so that release is to be planned first.")
|
||||
first = pick.item
|
||||
if first is None and pick.release is not None:
|
||||
merges = awaited_merges(weighed.carried, pick.release)
|
||||
return NoItem(f"every item of release {version(pick.release)} in {where} waits.{merges}", awaits_merge=bool(merges))
|
||||
pulls = awaited_merges(weighed.carried, pick.release)
|
||||
return NoItem(f"every item of release {version(pick.release)} in {where} waits.{merge_stop(pulls)}", awaits_merge=bool(pulls))
|
||||
if first is None:
|
||||
return NoItem(f"{where}'s items table holds no item.")
|
||||
held = weighed.carried
|
||||
@@ -100,13 +100,18 @@ def planned(read: Plan | Failure, cut: LastCut, default: str, branch: str, outsi
|
||||
return NextChunk(held, first, f"{first.tagline}\n\n{first.detail}".strip(), branch, over, item_id is not None)
|
||||
|
||||
|
||||
def awaited_merges(found: list[Item], release: Release) -> str:
|
||||
"""The sentences that stop the working loop where items of release await a person's merge, naming each pull request with the items it finishes; empty where none does."""
|
||||
pulls: dict[str, list[str]] = {}
|
||||
def awaited_merges(found: list[Item], release: Release) -> dict[str, list[int]]:
|
||||
"""Each pull request, as `#<number> <URL>`, that finishes items of release and awaits a person's merge, with those items' IDs."""
|
||||
pulls: dict[str, list[int]] = {}
|
||||
for item in found:
|
||||
if item.release == release and item.awaits_merge:
|
||||
pulls.setdefault(item.awaits_merge, []).append(str(item.id))
|
||||
pulls.setdefault(item.awaits_merge, []).append(item.id)
|
||||
return pulls
|
||||
|
||||
|
||||
def merge_stop(pulls: dict[str, list[int]]) -> str:
|
||||
"""The sentences that stop the working loop where pulls await a person's merge, naming each with the items it finishes; empty where none does."""
|
||||
if not pulls:
|
||||
return ""
|
||||
named = "; ".join(f"{pull}, which finishes item{'s' if len(ids) > 1 else ''} {listed(ids, 'and')}" for pull, ids in pulls.items())
|
||||
named = "; ".join(f"{pull}, which finishes item{'s' if len(ids) > 1 else ''} {listed([str(item_id) for item_id in ids], 'and')}" for pull, ids in pulls.items())
|
||||
return f" Under `merge = \"human\"`, these pull requests await a person's merge: {named}. The working loop stops here. Once a person has merged one, or turned one back into a draft, run loop-claude again."
|
||||
|
||||
@@ -475,4 +475,6 @@ class AwaitingMerge(unittest.TestCase):
|
||||
none = next_chunk.planned(Plan(items), held.cut, "main", "item-{id}")
|
||||
self.assertEqual(none.why, "every item of release 0.1.0 in main's todo.md waits. Under `merge = \"human\"`, these pull requests await a person's merge: #7 https://github.com/o/r/pull/7, which finishes item 1; #8 https://github.com/o/r/pull/8, which finishes items 2 and 3. The working loop stops here. Once a person has merged one, or turned one back into a draft, run loop-claude again.")
|
||||
self.assertTrue(none.awaits_merge)
|
||||
later = dataclasses.replace(items[0], id=9, release=(0, 2, 0), awaits_merge="#10 https://github.com/o/r/pull/10")
|
||||
self.assertEqual(next_chunk.awaited_merges([*items, later], (0, 1, 0)), {"#7 https://github.com/o/r/pull/7": [1], "#8 https://github.com/o/r/pull/8": [2, 3]}, "a pull request that finishes only a later release's items doesn't count")
|
||||
self.assertFalse(next_chunk.planned(Plan([dataclasses.replace(item, awaits_merge="", exempt="question") for item in items]), held.cut, "main", "item-{id}").awaits_merge)
|
||||
|
||||
Reference in New Issue
Block a user