Parse each transcript entry once into a typed value naming its kind, and have every consumer read those types
This commit is contained in:
+147
-45
@@ -1,14 +1,145 @@
|
||||
"""Entries of Claude Code's transcripts and of its stream-json and json output."""
|
||||
import dataclasses
|
||||
import json
|
||||
import typing
|
||||
|
||||
|
||||
def parse_entry(raw: str | bytes) -> dict | None:
|
||||
def json_object(raw: str | bytes) -> dict | None:
|
||||
"""A line of stream-json, or claude's json reply, where it is a JSON object."""
|
||||
try:
|
||||
entry = json.loads(raw)
|
||||
found = json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
return entry if isinstance(entry, dict) else None
|
||||
return found if isinstance(found, dict) else None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class Entry:
|
||||
"""A transcript entry, with the fields every entry may carry, each "" where it has none. Entry itself is an entry of a kind no consumer reads, such as a compaction's boundary; its subclasses name the kinds consumers read."""
|
||||
uuid: str = ""
|
||||
parent_uuid: str = ""
|
||||
timestamp: str = ""
|
||||
cwd: str = ""
|
||||
git_branch: str = ""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class UserText(Entry):
|
||||
"""The text a "user" entry holds, its blocks joined. HumanTurn, LoopPrompt and InjectedText say who wrote it."""
|
||||
text: str
|
||||
|
||||
|
||||
class HumanTurn(UserText):
|
||||
"""What the human wrote, at the prompt or while claude worked."""
|
||||
|
||||
|
||||
class LoopPrompt(UserText):
|
||||
"""What loop-claude told the worker in a -p prompt."""
|
||||
|
||||
|
||||
class InjectedText(UserText):
|
||||
"""Text Claude Code wrote into the prompt, such as a slash command, a task notification or a compaction's summary."""
|
||||
|
||||
|
||||
class Text(typing.NamedTuple):
|
||||
text: str
|
||||
|
||||
|
||||
class ToolCall(typing.NamedTuple):
|
||||
id: str
|
||||
name: str
|
||||
input: dict
|
||||
|
||||
|
||||
class ToolResult(typing.NamedTuple):
|
||||
tool_use_id: str
|
||||
text: str
|
||||
failed: bool
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class AssistantTurn(Entry):
|
||||
"""A content block of an assistant message, kept where it is text or a tool call, with the message's id, model and usage. Claude Code writes one entry per block of a message, each with the message's usage."""
|
||||
parts: tuple[Text | ToolCall, ...] = ()
|
||||
message_id: str = ""
|
||||
model: str = ""
|
||||
usage: dict = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class ToolResults(Entry):
|
||||
"""A "user" entry in which Claude Code returns tool results."""
|
||||
results: tuple[ToolResult, ...]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class CostState(Entry):
|
||||
"""The cost in dollars Claude Code records for each model the session has called so far, the subagents' and the compactions' calls too."""
|
||||
costs: dict[str, float]
|
||||
|
||||
|
||||
def parse_entry(raw: str | bytes) -> Entry | None:
|
||||
"""A transcript line as its kind; None where it is not a JSON object."""
|
||||
found = json_object(raw)
|
||||
return None if found is None else entry_from(found)
|
||||
|
||||
|
||||
def entry_from(found: dict) -> Entry:
|
||||
"""An entry of a transcript, or an event of stream-json, which writes assistant messages alike, as its kind."""
|
||||
carried = Carried(uuid=field(found, "uuid"), parent_uuid=field(found, "parentUuid"), timestamp=field(found, "timestamp"), cwd=field(found, "cwd"), git_branch=field(found, "gitBranch"))
|
||||
match found.get("type"):
|
||||
case "assistant":
|
||||
return assistant_turn(found, carried)
|
||||
case "user":
|
||||
return user_entry(found, carried)
|
||||
case "cost-state":
|
||||
return cost_state(found, carried)
|
||||
if (queued := queued_human_text(found)) is not None:
|
||||
return HumanTurn(**carried, text=queued)
|
||||
return Entry(**carried)
|
||||
|
||||
|
||||
class Carried(typing.TypedDict):
|
||||
"""The fields of Entry, read from any kind of entry."""
|
||||
uuid: str
|
||||
parent_uuid: str
|
||||
timestamp: str
|
||||
cwd: str
|
||||
git_branch: str
|
||||
|
||||
|
||||
# Alike in the transcript and in stream-json: {"type": "assistant", "message": {"id": "msg_01…", "model": "claude-opus-5-5", "content": [{"type": "text", "text": "ok"}], "usage": {...}}}
|
||||
def assistant_turn(found: dict, carried: Carried) -> AssistantTurn:
|
||||
message = found["message"] if isinstance(found.get("message"), dict) else {}
|
||||
usage = message.get("usage")
|
||||
parts = tuple(part for block in blocks(found) if (part := part_from(block)))
|
||||
return AssistantTurn(**carried, parts=parts, message_id=field(message, "id"), model=field(message, "model"), usage=usage if isinstance(usage, dict) else {})
|
||||
|
||||
|
||||
def user_entry(found: dict, carried: Carried) -> UserText | ToolResults:
|
||||
if results := tuple(result for block in blocks(found) if (result := tool_result(block))):
|
||||
return ToolResults(**carried, results=results)
|
||||
said = content(found)
|
||||
text = said if isinstance(said, str) else "".join(part_text(block) or "" for block in blocks(found))
|
||||
# {"type": "user", "promptSource": "sdk", "message": {"content": "Continue the task; the previous run ended before the chunk was finished."}}
|
||||
if found.get("promptSource") == "sdk" and isinstance(said, str):
|
||||
return LoopPrompt(**carried, text=text)
|
||||
return HumanTurn(**carried, text=text) if written_by_human(found) else InjectedText(**carried, text=text)
|
||||
|
||||
|
||||
# {"type": "cost-state", "totalCostUSD": 40.1, "modelUsage": {"claude-opus-5-5": {"costUSD": 40.1, "outputTokens": 334504, ...}}, ...}
|
||||
def cost_state(found: dict, carried: Carried) -> CostState:
|
||||
by_model = found["modelUsage"] if isinstance(found.get("modelUsage"), dict) else {}
|
||||
return CostState(**carried, costs={model: cost for model, usage in by_model.items() if isinstance(usage, dict) and isinstance(cost := usage.get("costUSD"), (int, float))})
|
||||
|
||||
|
||||
def tool_calls(entries: typing.Iterable[Entry]) -> list[tuple[AssistantTurn, ToolCall]]:
|
||||
"""Each tool call, with the turn that made it."""
|
||||
return [(entry, part) for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, ToolCall)]
|
||||
|
||||
|
||||
def field(found: dict, key: str) -> str:
|
||||
return value if isinstance(value := found.get(key), str) else ""
|
||||
|
||||
|
||||
def content(entry: dict) -> object:
|
||||
@@ -16,22 +147,17 @@ def content(entry: dict) -> object:
|
||||
return message.get("content") if isinstance(message, dict) else None
|
||||
|
||||
|
||||
def parts(entry: dict) -> list:
|
||||
def blocks(entry: dict) -> list:
|
||||
found = content(entry)
|
||||
return found if isinstance(found, list) else []
|
||||
|
||||
|
||||
# Alike in the transcript and in stream-json: {"type": "assistant", "message": {"content": [{"type": "text", "text": "ok"}]}}
|
||||
def assistant_parts(entry: dict) -> list:
|
||||
return parts(entry) if entry.get("type") == "assistant" else []
|
||||
|
||||
|
||||
def written_by_human(entry: dict) -> bool:
|
||||
"""Whether the human wrote a "user" entry; Claude Code writes the rest."""
|
||||
if tagged_as_injected(entry):
|
||||
return False
|
||||
text = content(entry)
|
||||
return string_by_human(entry, text) if isinstance(text, str) else blocks_by_human(parts(entry))
|
||||
return string_by_human(entry, text) if isinstance(text, str) else blocks_by_human(blocks(entry))
|
||||
|
||||
|
||||
# {"type": "user", "isMeta": true, "message": {"content": "<local-command-caveat>Caveat: ..."}}
|
||||
@@ -50,30 +176,12 @@ def string_by_human(entry: dict, text: str) -> bool:
|
||||
|
||||
|
||||
# {"type": "user", "message": {"content": [{"type": "text", "text": "merge it"}]}}
|
||||
# {"type": "user", "message": {"content": [{"tool_use_id": "toolu_01…", "type": "tool_result", "content": "hi"}]}}
|
||||
# {"type": "user", "message": {"content": [{"type": "text", "text": "[Request interrupted by user]"}]}}
|
||||
def blocks_by_human(blocks: list) -> bool:
|
||||
texts = [part_text(block) for block in blocks]
|
||||
def blocks_by_human(found: list) -> bool:
|
||||
texts = [part_text(block) for block in found]
|
||||
return bool(texts) and all(text is not None and not text.startswith("[Request interrupted") for text in texts)
|
||||
|
||||
|
||||
def human_text(entry: dict) -> str | None:
|
||||
"""What the human wrote in the entry, or None where Claude Code wrote it."""
|
||||
if (queued := queued_human_text(entry)) is not None:
|
||||
return queued
|
||||
if entry.get("type") != "user" or not written_by_human(entry):
|
||||
return None
|
||||
text = content(entry)
|
||||
return text if isinstance(text, str) else "".join(part_text(part) or "" for part in parts(entry))
|
||||
|
||||
|
||||
# {"type": "user", "promptSource": "sdk", "message": {"content": "Continue the task; the previous run ended before the chunk was finished."}}
|
||||
def loop_text(entry: dict) -> str | None:
|
||||
"""What loop-claude told the worker in a -p prompt."""
|
||||
text = content(entry)
|
||||
return text if entry.get("type") == "user" and entry.get("promptSource") == "sdk" and isinstance(text, str) else None
|
||||
|
||||
|
||||
# What the human types while claude works, written only as this entry:
|
||||
# {"type": "attachment", "attachment": {"type": "queued_command", "commandMode": "prompt", "origin": {"kind": "human"}, "prompt": "B"}}
|
||||
def queued_human_text(entry: dict) -> str | None:
|
||||
@@ -83,26 +191,20 @@ def queued_human_text(entry: dict) -> str | None:
|
||||
return prompt if isinstance(prompt := attachment.get("prompt"), str) else None
|
||||
|
||||
|
||||
# {"type": "tool_use", "id": "toolu_01…", "name": "Bash", "input": {"command": "echo hi"}}
|
||||
def part_from(block: object) -> Text | ToolCall | None:
|
||||
if (text := part_text(block)) is not None:
|
||||
return Text(text)
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use" and isinstance(name := block.get("name"), str) and isinstance(arguments := block.get("input"), dict):
|
||||
return ToolCall(field(block, "id"), name, arguments)
|
||||
return None
|
||||
|
||||
|
||||
# {"type": "text", "text": "ok"}
|
||||
def part_text(part: object) -> str | None:
|
||||
return part["text"] if isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str) else None
|
||||
|
||||
|
||||
# {"type": "tool_use", "id": "toolu_01…", "name": "Bash", "input": {"command": "echo hi"}}
|
||||
def tool_call(part: object) -> tuple[object, object] | None:
|
||||
return (part.get("name"), part.get("input")) if isinstance(part, dict) and part.get("type") == "tool_use" else None
|
||||
|
||||
|
||||
def tool_use_id(part: object) -> str | None:
|
||||
return part["id"] if isinstance(part, dict) and part.get("type") == "tool_use" and isinstance(part.get("id"), str) else None
|
||||
|
||||
|
||||
class ToolResult(typing.NamedTuple):
|
||||
tool_use_id: str
|
||||
text: str
|
||||
failed: bool
|
||||
|
||||
|
||||
# {"type": "tool_result", "tool_use_id": "toolu_01…", "content": "hi"}, its content also as [{"type": "text", "text": "hi"}], and with "is_error": true where the call failed
|
||||
def tool_result(part: object) -> ToolResult | None:
|
||||
if not isinstance(part, dict) or part.get("type") != "tool_result":
|
||||
|
||||
@@ -4,7 +4,7 @@ import subprocess
|
||||
import typing
|
||||
|
||||
from failure import Failure
|
||||
from claude_code.entries import hit_limit, limit_said, parse_entry
|
||||
from claude_code.entries import hit_limit, json_object, limit_said
|
||||
|
||||
# Effects.claude: claude's arguments, its working directory and the prompt it reads on stdin.
|
||||
Claude = typing.Callable[[list[str], str, str], subprocess.CompletedProcess[str]]
|
||||
@@ -27,7 +27,7 @@ def claude_reply(claude: Claude, args: list[str], cwd: str, prompt: str, who: st
|
||||
proc = claude(["-p", *args, *HEADLESS], cwd, prompt)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not start claude: {failure.strerror}")
|
||||
reply = parse_entry(proc.stdout)
|
||||
reply = json_object(proc.stdout)
|
||||
# The json output names the reset only in prose, such as "resets 7:40pm (Europe/Stockholm)", so its limit carries no reset time.
|
||||
if reply is not None and hit_limit(reply):
|
||||
return Limited(None, limit_said(reply))
|
||||
|
||||
@@ -18,13 +18,17 @@ def human(content):
|
||||
return {"type": "user", "message": {"content": content}}
|
||||
|
||||
|
||||
def kinds(found):
|
||||
return [entries.entry_from(entry) for entry in found]
|
||||
|
||||
|
||||
def queued(prompt, kind):
|
||||
return {"type": "attachment", "attachment": {"type": "queued_command", "commandMode": "prompt", "origin": {"kind": kind}, "prompt": prompt}}
|
||||
|
||||
|
||||
class Render(unittest.TestCase):
|
||||
def test_joins_assistant_text_tool_calls_human_turns_and_loop_prompts_labelling_a_typed_answer_the_humans_and_skips_the_rest(self):
|
||||
said = transcript.render([
|
||||
said = transcript.render(kinds([
|
||||
human("do it"),
|
||||
assistant({"type": "text", "text": "Starting."}, {"type": "tool_use", "name": "Bash", "input": {"command": "ls"}}),
|
||||
assistant({"type": "text", "text": " "}),
|
||||
@@ -35,7 +39,7 @@ class Render(unittest.TestCase):
|
||||
queued("origin/main moved", "coordinator"),
|
||||
{**human("Continue the task; the previous run stopped before it was finished."), "promptSource": "sdk"},
|
||||
{**human("Answered: use B\nYour question: A or B?"), "promptSource": "sdk"},
|
||||
], lambda prompt: transcript.Typed(*reversed(prompt.removeprefix("Answered: ").split("\n"))) if prompt.startswith("Answered: ") else None)
|
||||
]), lambda prompt: transcript.Typed(*reversed(prompt.removeprefix("Answered: ").split("\n"))) if prompt.startswith("Answered: ") else None)
|
||||
self.assertEqual(said, '[human] do it\n\nStarting.\n\n[tool Bash] {"command": "ls"}\n\n[human] yes, merge it\n\n[human] B, while you work\n\n[loop] Continue the task; the previous run stopped before it was finished.\n\n[loop] Your question: A or B?\n\n[human] use B')
|
||||
|
||||
|
||||
@@ -57,15 +61,15 @@ class FirstTimestampedLine(unittest.TestCase):
|
||||
class CurrentThread(unittest.TestCase):
|
||||
def test_a_parent_cycle_ends_the_thread(self):
|
||||
looped = [{"uuid": "a", "parentUuid": "b"}, {"uuid": "b", "parentUuid": "a"}]
|
||||
self.assertEqual([entry["uuid"] for entry in transcript.current_thread(looped)], ["a", "b"])
|
||||
self.assertEqual([entry.uuid for entry in transcript.current_thread(kinds(looped))], ["a", "b"])
|
||||
|
||||
def test_entries_without_a_uuid_make_no_thread(self):
|
||||
self.assertEqual(transcript.current_thread([{"type": "summary"}]), [])
|
||||
self.assertEqual(transcript.current_thread(kinds([{"type": "summary"}])), [])
|
||||
|
||||
|
||||
class HumanTurns(unittest.TestCase):
|
||||
def test_counts_typed_turns_and_not_tool_results(self):
|
||||
self.assertEqual(transcript.human_turns([
|
||||
self.assertEqual(transcript.human_turns(kinds([
|
||||
human("typed"),
|
||||
human([{"type": "text", "text": "typed as a block"}]),
|
||||
human([{"type": "tool_result", "content": "not typed"}]),
|
||||
@@ -82,25 +86,84 @@ class HumanTurns(unittest.TestCase):
|
||||
queued("<task-notification>done</task-notification>", "task-notification"),
|
||||
{"type": "attachment", "attachment": {"type": "queued_command", "commandMode": "task-notification", "prompt": "<task-notification>done</task-notification>"}},
|
||||
assistant({"type": "text", "text": "said"}),
|
||||
]), 4)
|
||||
])), 4)
|
||||
|
||||
|
||||
class PartLine(unittest.TestCase):
|
||||
def test_renders_text_and_tool_calls_and_nothing_else(self):
|
||||
self.assertEqual(transcript.part_line({"type": "text", "text": "hi"}, 20), "hi")
|
||||
self.assertEqual(transcript.part_line({"type": "text", "text": " \n"}, 20), "")
|
||||
self.assertEqual(transcript.part_line({"type": "tool_use", "name": "Bash", "input": {"command": "x" * 500}}, 20), '[tool Bash] {"command": "xxxxxxx')
|
||||
self.assertEqual(transcript.part_line("not a part", 20), "")
|
||||
self.assertEqual(transcript.part_line({"type": "tool_use", "name": "Write", "input": {"content": "Hälsa på"}}, 40), '[tool Write] {"content": "Hälsa på"}')
|
||||
self.assertEqual(transcript.part_line(entries.Text("hi"), 20), "hi")
|
||||
self.assertEqual(transcript.part_line(entries.Text(" \n"), 20), "")
|
||||
self.assertEqual(transcript.part_line(entries.ToolCall("toolu_1", "Bash", {"command": "x" * 500}), 20), '[tool Bash] {"command": "xxxxxxx')
|
||||
self.assertEqual(transcript.part_line(entries.ToolCall("toolu_1", "Write", {"content": "Hälsa på"}), 40), '[tool Write] {"content": "Hälsa på"}')
|
||||
|
||||
|
||||
class Parse(unittest.TestCase):
|
||||
def test_yields_only_json_objects(self):
|
||||
self.assertEqual(entries.parse_entry('{"type": "user"}'), {"type": "user"})
|
||||
self.assertIsNone(entries.parse_entry("not json"))
|
||||
self.assertEqual(entries.json_object('{"type": "user"}'), {"type": "user"})
|
||||
self.assertIsNone(entries.json_object("not json"))
|
||||
self.assertIsNone(entries.json_object("[1, 2]"))
|
||||
|
||||
def test_a_transcript_line_parses_into_its_kind(self):
|
||||
self.assertEqual(entries.parse_entry('{"type": "user", "message": {"content": "do it"}}'), entries.HumanTurn(text="do it"))
|
||||
self.assertIsNone(entries.parse_entry("[1, 2]"))
|
||||
|
||||
|
||||
class EntryKinds(unittest.TestCase):
|
||||
def test_names_whose_text_a_user_entry_holds(self):
|
||||
self.assertEqual([entries.entry_from(found) for found in [
|
||||
human("typed"),
|
||||
human([{"type": "text", "text": "typed "}, {"type": "text", "text": "as blocks"}]),
|
||||
{**human("<3 merge it"), "promptSource": "typed"},
|
||||
queued("typed while claude worked", "human"),
|
||||
{**human("Continue the task."), "promptSource": "sdk"},
|
||||
{**human([{"type": "text", "text": "Continue as a block."}]), "promptSource": "sdk"},
|
||||
human("<command-name>/model</command-name>"),
|
||||
{**human("peer message"), "isMeta": True},
|
||||
{**human("<task-notification>done</task-notification>"), "promptSource": "system"},
|
||||
human([{"type": "text", "text": "[Request interrupted by user]"}]),
|
||||
human([]),
|
||||
]], [
|
||||
entries.HumanTurn(text="typed"),
|
||||
entries.HumanTurn(text="typed as blocks"),
|
||||
entries.HumanTurn(text="<3 merge it"),
|
||||
entries.HumanTurn(text="typed while claude worked"),
|
||||
entries.LoopPrompt(text="Continue the task."),
|
||||
entries.InjectedText(text="Continue as a block."),
|
||||
entries.InjectedText(text="<command-name>/model</command-name>"),
|
||||
entries.InjectedText(text="peer message"),
|
||||
entries.InjectedText(text="<task-notification>done</task-notification>"),
|
||||
entries.InjectedText(text="[Request interrupted by user]"),
|
||||
entries.InjectedText(text=""),
|
||||
])
|
||||
|
||||
def test_an_assistant_turn_keeps_its_text_and_tool_calls_and_its_message(self):
|
||||
turn = {**assistant(
|
||||
{"type": "thinking", "thinking": "hm"},
|
||||
{"type": "text", "text": "Starting."},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}},
|
||||
{"type": "tool_use", "id": "toolu_2", "name": "Bash", "input": "not arguments"},
|
||||
), "uuid": "u"}
|
||||
turn["message"] |= {"id": "msg_1", "model": "claude-opus-5-5", "usage": {"input_tokens": 2}}
|
||||
self.assertEqual(entries.entry_from(turn), entries.AssistantTurn(
|
||||
uuid="u", parts=(entries.Text("Starting."), entries.ToolCall("toolu_1", "Bash", {"command": "ls"})), message_id="msg_1", model="claude-opus-5-5", usage={"input_tokens": 2},
|
||||
))
|
||||
|
||||
def test_tool_results_queued_notifications_and_other_kinds(self):
|
||||
self.assertEqual(entries.entry_from(human([{"type": "tool_result", "tool_use_id": "toolu_1", "content": "hi", "is_error": True}])), entries.ToolResults(results=(entries.ToolResult("toolu_1", "hi", True),)))
|
||||
self.assertEqual(entries.entry_from(queued("<task-notification>done</task-notification>", "task-notification")), entries.Entry())
|
||||
self.assertEqual(entries.entry_from({"type": "system", "subtype": "compact_boundary"}), entries.Entry())
|
||||
cost_state = {"type": "cost-state", "totalCostUSD": 41.1, "modelUsage": {"claude-opus-5-5": {"costUSD": 40.1, "outputTokens": 334504}, "claude-haiku-5-5": {"costUSD": 1}, "odd": {"costUSD": "1"}, "odder": 1}}
|
||||
self.assertEqual(entries.entry_from(cost_state), entries.CostState(costs={"claude-opus-5-5": 40.1, "claude-haiku-5-5": 1}))
|
||||
self.assertEqual(entries.entry_from({"type": "cost-state", "modelUsage": []}), entries.CostState(costs={}))
|
||||
|
||||
def test_every_kind_carries_where_and_when_it_was_written(self):
|
||||
self.assertEqual(
|
||||
entries.entry_from({"type": "summary", "uuid": "b", "parentUuid": "a", "timestamp": "2026-10-15T00:00:00.123Z", "cwd": "/r", "gitBranch": "item-1"}),
|
||||
entries.Entry(uuid="b", parent_uuid="a", timestamp="2026-10-15T00:00:00.123Z", cwd="/r", git_branch="item-1"),
|
||||
)
|
||||
self.assertEqual(entries.entry_from({"type": "summary", "uuid": 1, "parentUuid": None}), entries.Entry())
|
||||
|
||||
|
||||
class ClaudeReply(unittest.TestCase):
|
||||
@staticmethod
|
||||
def claude(*results):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
import typing
|
||||
|
||||
from claude_code.entries import assistant_parts, human_text, loop_text, parse_entry, part_text, tool_call
|
||||
from claude_code.entries import AssistantTurn, Entry, HumanTurn, LoopPrompt, parse_entry, Text, ToolCall
|
||||
from failure import Failure
|
||||
|
||||
JUDGED_TOOL_INPUT = 400
|
||||
@@ -13,7 +13,7 @@ class Typed(typing.NamedTuple):
|
||||
answer: str
|
||||
|
||||
|
||||
def read_transcript(path: str) -> list[dict] | Failure:
|
||||
def read_transcript(path: str) -> list[Entry] | Failure:
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
return [entry for raw in handle if (entry := parse_entry(raw))]
|
||||
@@ -21,51 +21,54 @@ def read_transcript(path: str) -> list[dict] | Failure:
|
||||
return Failure(f"could not read the transcript {path}: {failure.strerror}")
|
||||
|
||||
|
||||
def current_thread(entries: list[dict]) -> list[dict]:
|
||||
def current_thread(entries: list[Entry]) -> list[Entry]:
|
||||
"""The entries of the transcript's current thread since its last compaction, oldest first: the newest entry with a uuid and its ancestors by parentUuid.
|
||||
|
||||
That leaves out a branch the session rewound, and the results of parallel tool calls that Claude Code chains on a side branch."""
|
||||
chained = [entry for entry in entries if isinstance(entry.get("uuid"), str)]
|
||||
by_uuid = {entry["uuid"]: entry for entry in chained}
|
||||
found: list[dict] = []
|
||||
chained = [entry for entry in entries if entry.uuid]
|
||||
by_uuid = {entry.uuid: entry for entry in chained}
|
||||
found: list[Entry] = []
|
||||
entry = chained[-1] if chained else None
|
||||
# A compaction writes {"type": "system", "subtype": "compact_boundary", "parentUuid": null, ...}, so the thread starts there; the length check stops a parent cycle.
|
||||
while entry is not None and len(found) < len(chained):
|
||||
found.append(entry)
|
||||
entry = by_uuid.get(entry.get("parentUuid"))
|
||||
entry = by_uuid.get(entry.parent_uuid)
|
||||
return found[::-1]
|
||||
|
||||
|
||||
# The lines before it, such as the session's title, are the same for every chunk of a project.
|
||||
def first_timestamped_line(raw: bytes) -> bytes:
|
||||
lines = raw.splitlines()
|
||||
return next((line for line in lines if "timestamp" in (parse_entry(line) or {})), lines[0] if lines else b"")
|
||||
return next((line for line in lines if (entry := parse_entry(line)) and entry.timestamp), lines[0] if lines else b"")
|
||||
|
||||
|
||||
def human_turns(entries: list[dict]) -> int:
|
||||
return sum(1 for entry in entries if human_text(entry) is not None)
|
||||
def human_turns(entries: list[Entry]) -> int:
|
||||
return sum(1 for entry in entries if isinstance(entry, HumanTurn))
|
||||
|
||||
|
||||
def render(entries: list[dict], typed_answer: typing.Callable[[str], Typed | None]) -> str:
|
||||
def render(entries: list[Entry], typed_answer: typing.Callable[[str], Typed | None]) -> str:
|
||||
"""The transcript as the judge reads it. typed_answer splits the answer the human typed out of a -p prompt of loop-claude's, so that answer reads as the human's, like one typed in claude."""
|
||||
lines = []
|
||||
for entry in entries:
|
||||
lines += [line for part in assistant_parts(entry) if (line := part_line(part, JUDGED_TOOL_INPUT))]
|
||||
if (text := human_text(entry)) is not None:
|
||||
lines.append("[human] " + text)
|
||||
elif (text := loop_text(entry)) is not None:
|
||||
typed = typed_answer(text)
|
||||
lines += ["[loop] " + text] if typed is None else ["[loop] " + typed.loop_words, "[human] " + typed.answer]
|
||||
if isinstance(entry, AssistantTurn):
|
||||
lines += assistant_lines(entry)
|
||||
elif isinstance(entry, HumanTurn):
|
||||
lines.append("[human] " + entry.text)
|
||||
elif isinstance(entry, LoopPrompt):
|
||||
typed = typed_answer(entry.text)
|
||||
lines += ["[loop] " + entry.text] if typed is None else ["[loop] " + typed.loop_words, "[human] " + typed.answer]
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
def part_line(part: object, tool_input_chars: int) -> str:
|
||||
if (text := part_text(part)) and text.strip():
|
||||
return text
|
||||
if call := tool_call(part):
|
||||
name, arguments = call
|
||||
return f"[tool {name}] {json.dumps(arguments, ensure_ascii=False)[:tool_input_chars]}"
|
||||
return ""
|
||||
def assistant_lines(turn: AssistantTurn) -> list[str]:
|
||||
"""The turn's text and tool calls as the judge reads them."""
|
||||
return [line for part in turn.parts if (line := part_line(part, JUDGED_TOOL_INPUT))]
|
||||
|
||||
|
||||
def part_line(part: Text | ToolCall, tool_input_chars: int) -> str:
|
||||
if isinstance(part, ToolCall):
|
||||
return f"[tool {part.name}] {json.dumps(part.input, ensure_ascii=False)[:tool_input_chars]}"
|
||||
return part.text if part.text.strip() else ""
|
||||
|
||||
|
||||
def verbatim(quote: str, text: str) -> bool:
|
||||
|
||||
@@ -6,6 +6,7 @@ import typing
|
||||
from loop_claude.cycle.prompts import ANSWER_DESTINATIONS, ANSWERS_RECORDED, typed_answer
|
||||
from claude_code.reply import claude_reply, Limited
|
||||
from loop_claude.effects import Effects
|
||||
from claude_code.entries import Entry, structured_output
|
||||
from loop_claude import escalation_mark
|
||||
from failure import Failure
|
||||
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, TakenIn, Unfinished, Verdict
|
||||
@@ -15,7 +16,6 @@ from claude_code.transcript import plain, verbatim
|
||||
from loop_claude.cycle.retry import retried
|
||||
from model_routing.ladder import Rung
|
||||
from loop_claude.stop import Stop
|
||||
from claude_code.entries import structured_output
|
||||
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
JUDGE_PROMPT_CHARS = 400000
|
||||
@@ -108,8 +108,8 @@ class Grounds(typing.NamedTuple):
|
||||
"""The texts quotes_checked checks the judge's quotes against. A quote must appear in its text as written, since a quote reworded or stitched together can say what its source does not; only the evidence may join passages with an ellipsis, each passage in order."""
|
||||
# The transcript as the judge saw it.
|
||||
shown: str
|
||||
# The worker's own lines after the escalation mark, as escalation_mark.since_step renders them.
|
||||
since_step: str
|
||||
# The worker's own lines after the escalation mark, as escalation_mark.after_mark renders them, however much of the transcript shown holds.
|
||||
after_mark: str
|
||||
# README.md's premises and the records.
|
||||
quotable: str
|
||||
|
||||
@@ -119,7 +119,7 @@ class Brief(typing.NamedTuple):
|
||||
task: str
|
||||
premises: str
|
||||
records: str
|
||||
entries: list[dict]
|
||||
entries: list[Entry]
|
||||
escalated_at: int
|
||||
|
||||
|
||||
@@ -217,7 +217,7 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
|
||||
| an interrupt's answered_before | grounds.shown | the interrupt stands, as if no earlier answer was found |
|
||||
| its left_open | answered_before and the question | the interrupt becomes an Unfinished that quotes answered_before back to the worker |
|
||||
| a quoted answer | grounds.quotable | the question goes to the human as an interrupt, which keeps the escalation the answer carried |
|
||||
| the escalation | grounds.shown and grounds.since_step | the escalation is dropped, so the worker's rung holds, as escalation_mark.checked says |
|
||||
| the escalation | grounds.shown and grounds.after_mark | the escalation is dropped, so the worker's rung holds, as escalation_mark.checked says |
|
||||
|
||||
Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead.
|
||||
"""
|
||||
@@ -225,7 +225,7 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
|
||||
return misquoted_evidence(verdict)
|
||||
verdict = re_ask_checked(verdict, grounds.shown)
|
||||
verdict = answer_checked(verdict, grounds.quotable)
|
||||
return escalation_mark.checked(verdict, since=grounds.since_step, shown=grounds.shown)
|
||||
return escalation_mark.checked(verdict, after=grounds.after_mark, shown=grounds.shown)
|
||||
|
||||
|
||||
def re_ask_checked(verdict: Verdict, shown: str) -> Verdict:
|
||||
@@ -251,18 +251,22 @@ def answer_checked(verdict: Verdict, quotable: str) -> Verdict:
|
||||
|
||||
def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
|
||||
"""The prompt, and what the judge's quotes are checked against."""
|
||||
task, premises, records, entries, escalated_at = brief
|
||||
transcript = escalation_mark.transcript(entries, escalated_at, typed_answer)
|
||||
head = (
|
||||
f"From README.md:\n{premises}\n\n"
|
||||
f"The records: the project's working rules and decisions, then the human's rules for every project:\n{records or 'none'}\n\n"
|
||||
f"The task the worker was given:\n{task}\n\n"
|
||||
f"From README.md:\n{brief.premises}\n\n"
|
||||
f"The records: the project's working rules and decisions, then the human's rules for every project:\n{brief.records or 'none'}\n\n"
|
||||
f"The task the worker was given:\n{brief.task}\n\n"
|
||||
"What the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\n"
|
||||
)
|
||||
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2)
|
||||
transcript = escalation_mark.transcript(brief.entries, brief.escalated_at, typed_answer)
|
||||
# The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle.
|
||||
shown = transcript[max(0, len(transcript) - room):]
|
||||
return f"{head}{shown}\n\n{JUDGE_RULES}", Grounds(shown, escalation_mark.since_step(entries, escalated_at), f"{premises}\n\n{records}")
|
||||
grounds = Grounds(
|
||||
shown=shown,
|
||||
after_mark=escalation_mark.after_mark(brief.entries, brief.escalated_at),
|
||||
quotable=f"{brief.premises}\n\n{brief.records}",
|
||||
)
|
||||
return f"{head}{shown}\n\n{JUDGE_RULES}", grounds
|
||||
|
||||
|
||||
def grounded(evidence: str, rendered_transcript: str) -> bool:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from claude_code.entries import assistant_parts, ends_stream, hit_limit, limit_said, parse_entry, rejected_until
|
||||
from claude_code.entries import AssistantTurn, ends_stream, entry_from, hit_limit, json_object, limit_said, rejected_until
|
||||
from loop_claude.effects import Effects, say
|
||||
from failure import Failure
|
||||
from claude_code.reply import Limited
|
||||
@@ -24,14 +24,15 @@ def run_worker(root: str, args: list[str], env: dict[str, str], effects: Effects
|
||||
result: dict | None = None
|
||||
resets: list[int] = []
|
||||
for raw in proc.lines():
|
||||
event = parse_entry(raw) or {}
|
||||
event = json_object(raw) or {}
|
||||
if ends_stream(event):
|
||||
result = event
|
||||
if (resets_at := rejected_until(event)) is not None:
|
||||
resets.append(resets_at)
|
||||
for part in assistant_parts(event):
|
||||
if line := part_line(part, PRINTED_TOOL_INPUT):
|
||||
say(line, effects, stdout=True)
|
||||
if isinstance(turn := entry_from(event), AssistantTurn):
|
||||
for part in turn.parts:
|
||||
if line := part_line(part, PRINTED_TOOL_INPUT):
|
||||
say(line, effects, stdout=True)
|
||||
return worker_outcome(proc.wait(), result, resets)
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@ A worker whose run ends asking loop-claude for a higher rung cites the trigger t
|
||||
- Stamp. stamped() writes on the judge's verdict how many entries the judge read.
|
||||
- Step. routing.next_route() steps the worker up on an escalation and moves the mark to moved_to(verdict).
|
||||
- Show. transcript() puts STEP_LINE after the mark, telling the judge that the triggers before that line are spent. QUOTE_AFTER_STEP, in the judge's schema, tells the judge to quote only the worker's words after that line.
|
||||
- Check. checked() keeps an escalation only where it quotes both the transcript the judge was shown and the worker's own lines after the mark, which since_step() renders, so a passage that bought one step cannot buy another.
|
||||
- Check. checked() keeps an escalation only where it quotes both the transcript the judge was shown and the worker's own lines after the mark, which after_mark() renders, so a passage that bought one step cannot buy another.
|
||||
"""
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from claude_code.transcript import assistant_lines, render, Typed, verbatim
|
||||
from claude_code.entries import AssistantTurn, Entry
|
||||
from loop_claude.verdicts import Escalating, Verdict
|
||||
from claude_code.entries import human_text, loop_text
|
||||
from claude_code.transcript import render, Typed, verbatim
|
||||
|
||||
# Only the judge reads the step line, so it is not a [loop] line.
|
||||
STEP_TAG = "[step]"
|
||||
@@ -31,19 +31,18 @@ def moved_to(verdict: Escalating) -> int | None:
|
||||
return verdict.entries_read if verdict.escalation else None
|
||||
|
||||
|
||||
def transcript(entries: list[dict], at: int, typed_answer: typing.Callable[[str], Typed | None]) -> str:
|
||||
def transcript(entries: list[Entry], at: int, typed_answer: typing.Callable[[str], Typed | None]) -> str:
|
||||
"""The transcript as the judge sees it. Where the mark, at, is above 0, STEP_LINE comes after that many entries."""
|
||||
return "\n\n".join(part for part in (render(entries[:at], typed_answer), STEP_LINE if at else "", render(entries[at:], typed_answer)) if part)
|
||||
|
||||
|
||||
def since_step(entries: list[dict], at: int) -> str:
|
||||
"""The worker's own lines after the mark, all of them, however much of the transcript the judge is shown. loop-claude's prompts and the human's lines are left out, since they repeat earlier passages, such as the judge's evidence or the worker's question."""
|
||||
# No loop-claude prompt is left to carry an answer the human typed.
|
||||
return render([entry for entry in entries[at:] if human_text(entry) is None and loop_text(entry) is None], lambda prompt: None)
|
||||
def after_mark(entries: list[Entry], at: int) -> str:
|
||||
"""The worker's own lines after the mark, its assistant turns, all of them, however much of the transcript the judge is shown. loop-claude's prompts and the human's lines are left out, since they repeat earlier passages, such as the judge's evidence or the worker's question."""
|
||||
return "\n\n".join(line for entry in entries[at:] if isinstance(entry, AssistantTurn) for line in assistant_lines(entry))
|
||||
|
||||
|
||||
def checked(verdict: Verdict, *, since: str, shown: str) -> Verdict:
|
||||
"""The verdict without its escalation unless the escalation quotes both texts: since, the worker's lines after the mark as since_step renders them, and shown, the transcript the judge was shown. A misquote, or a quote of a passage before the mark, drops the escalation, so the worker's rung holds."""
|
||||
if isinstance(verdict, Escalating) and verdict.escalation and not (verbatim(verdict.escalation, since) and verbatim(verdict.escalation, shown)):
|
||||
def checked(verdict: Verdict, *, after: str, shown: str) -> Verdict:
|
||||
"""The verdict without its escalation unless the escalation quotes both texts: after, the worker's lines after the mark as after_mark renders them, and shown, the transcript the judge was shown. A misquote, or a quote of a passage before the mark, drops the escalation, so the worker's rung holds."""
|
||||
if isinstance(verdict, Escalating) and verdict.escalation and not (verbatim(verdict.escalation, after) and verbatim(verdict.escalation, shown)):
|
||||
return dataclasses.replace(verdict, escalation="")
|
||||
return verdict
|
||||
|
||||
@@ -69,6 +69,10 @@ def assistant(*parts):
|
||||
return {"type": "assistant", "message": {"content": list(parts)}}
|
||||
|
||||
|
||||
def kinds(*found):
|
||||
return [entries.entry_from(entry) for entry in found]
|
||||
|
||||
|
||||
def human(content):
|
||||
return {"type": "user", "message": {"content": content}}
|
||||
|
||||
@@ -1610,16 +1614,16 @@ class Records(unittest.TestCase):
|
||||
class JudgePrompt(unittest.TestCase):
|
||||
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
||||
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 0))
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 0))
|
||||
self.assertTrue(grounds.shown.endswith("\n\nnew"))
|
||||
self.assertNotIn("old", grounds.shown)
|
||||
self.assertIn(grounds.shown, prompt)
|
||||
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
||||
self.assertEqual(grounds.since_step, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
||||
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
||||
self.assertEqual(grounds.quotable, "## Goals\n\n")
|
||||
|
||||
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
|
||||
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", [assistant({"type": "text", "text": "done"})], 1))
|
||||
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(assistant({"type": "text", "text": "done"})), 1))
|
||||
self.assertNotRegex(prompt + json.dumps(judge.JUDGE_SCHEMA), r"\b[Aa]gent")
|
||||
|
||||
def test_the_judge_prompt_names_every_rule_the_judge_reads_from_the_human(self):
|
||||
@@ -1627,24 +1631,24 @@ class JudgePrompt(unittest.TestCase):
|
||||
|
||||
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
|
||||
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
||||
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
|
||||
self.assertEqual(grounds.since_step, "round 3: 🔴")
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 0))
|
||||
self.assertEqual(grounds.after_mark, "round 3: 🔴")
|
||||
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 0))
|
||||
self.assertNotIn("stepped the worker up", prompt)
|
||||
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
|
||||
self.assertEqual(grounds.since_step, "round 2: 🔴\n\nround 3: 🔴")
|
||||
self.assertEqual(grounds.after_mark, "round 2: 🔴\n\nround 3: 🔴")
|
||||
|
||||
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
|
||||
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
||||
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
|
||||
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
||||
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
|
||||
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
|
||||
|
||||
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
|
||||
typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
|
||||
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
|
||||
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
|
||||
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
||||
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
|
||||
|
||||
|
||||
@@ -1771,7 +1775,7 @@ class Verdict(unittest.TestCase):
|
||||
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
|
||||
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
|
||||
def judged_on(mark, escalation):
|
||||
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, mark), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
|
||||
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), mark), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
|
||||
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
|
||||
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
|
||||
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
|
||||
@@ -1780,12 +1784,12 @@ class Verdict(unittest.TestCase):
|
||||
|
||||
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
|
||||
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
|
||||
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
|
||||
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
|
||||
self.assertEqual(verdict, Unfinished("working", entries_read=4))
|
||||
|
||||
def test_takes_evidence_from_before_the_workers_last_step_up(self):
|
||||
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
|
||||
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
|
||||
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
|
||||
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
|
||||
|
||||
|
||||
|
||||
+13
-24
@@ -2,7 +2,7 @@ import collections
|
||||
import dataclasses
|
||||
import re
|
||||
|
||||
from claude_code.entries import assistant_parts, parse_entry, tool_call, usage_count
|
||||
from claude_code.entries import AssistantTurn, Entry, parse_entry, tool_calls, ToolCall, usage_count
|
||||
from git_cli.git_command import commits
|
||||
from reading_cost.shell_reads import Reads, shell_reads
|
||||
from command_lines.shell_command import resolved, simple_commands
|
||||
@@ -30,43 +30,32 @@ def measure(threads: list[bytes], checkout: Checkout, home: str) -> Reading | No
|
||||
commit = commit_time(worker, home)
|
||||
if commit is None:
|
||||
return None
|
||||
before = [[entry for entry in thread if entry.get("type") == "assistant" and str(entry.get("timestamp", "")) < commit] for thread in (worker, *subagents)]
|
||||
reads = [reads_of(part, str(entry.get("cwd") or "/"), home) for thread in before for entry in thread for part in assistant_parts(entry)]
|
||||
before = [[entry for entry in thread if isinstance(entry, AssistantTurn) and entry.timestamp < commit] for thread in (worker, *subagents)]
|
||||
reads = [reads_of(call, turn.cwd or "/", home) for thread in before for turn, call in tool_calls(thread)]
|
||||
roots = [checkout.main_root, *(added for found in reads for added in found.added)]
|
||||
files = collections.Counter(file for found in reads for path in found.read if (file := relative(in_main_checkout(path, checkout.main_root), roots)) in checkout.tracked)
|
||||
return Reading(sum(map(tokens_in, before)), files)
|
||||
|
||||
|
||||
def entries_of(raw: bytes) -> list[dict]:
|
||||
def entries_of(raw: bytes) -> list[Entry]:
|
||||
return [entry for line in raw.splitlines() if (entry := parse_entry(line))]
|
||||
|
||||
|
||||
def commit_time(worker: list[dict], home: str) -> str | None:
|
||||
for entry in worker:
|
||||
for part in assistant_parts(entry):
|
||||
name, arguments = call_of(part)
|
||||
if name == "Bash" and commits(simple_commands(str(arguments.get("command", "")), str(entry.get("cwd") or "/"), home), home):
|
||||
return str(entry.get("timestamp", ""))
|
||||
return None
|
||||
def commit_time(worker: list[Entry], home: str) -> str | None:
|
||||
return next((turn.timestamp for turn, call in tool_calls(worker) if call.name == "Bash" and commits(simple_commands(str(call.input.get("command", "")), turn.cwd or "/", home), home)), None)
|
||||
|
||||
|
||||
def tokens_in(entries: list[dict]) -> int:
|
||||
# Claude Code writes one entry per content block of a message, each with the message's usage.
|
||||
usages = {str(message.get("id") or entry.get("uuid")): message.get("usage") for entry in entries if isinstance(message := entry.get("message"), dict)}
|
||||
return sum(usage_count(usage, "input_tokens") + usage_count(usage, "cache_creation_input_tokens") for usage in usages.values() if isinstance(usage, dict))
|
||||
def tokens_in(turns: list[AssistantTurn]) -> int:
|
||||
# Each turn is one content block of a message, and carries the message's usage.
|
||||
usages = {turn.message_id or turn.uuid: turn.usage for turn in turns}
|
||||
return sum(usage_count(usage, "input_tokens") + usage_count(usage, "cache_creation_input_tokens") for usage in usages.values())
|
||||
|
||||
|
||||
def call_of(part: object) -> tuple[object, dict]:
|
||||
call = tool_call(part)
|
||||
return (call[0], call[1]) if call and isinstance(call[1], dict) else (None, {})
|
||||
|
||||
|
||||
def reads_of(part: object, cwd: str, home: str) -> Reads:
|
||||
name, arguments = call_of(part)
|
||||
path = arguments.get("file_path") if name == "Read" else arguments.get("path") if name == "Grep" else None
|
||||
def reads_of(call: ToolCall, cwd: str, home: str) -> Reads:
|
||||
path = call.input.get("file_path") if call.name == "Read" else call.input.get("path") if call.name == "Grep" else None
|
||||
if path:
|
||||
return Reads([resolved(cwd, str(path), home)], [])
|
||||
return shell_reads(str(arguments.get("command", "")), cwd, home) if name == "Bash" else Reads([], [])
|
||||
return shell_reads(str(call.input.get("command", "")), cwd, home) if call.name == "Bash" else Reads([], [])
|
||||
|
||||
|
||||
# chunk-preflight adds each item's worktree beside the checkout as <checkout>-item-<ID>, which no transcript command names.
|
||||
|
||||
@@ -13,6 +13,7 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from transcript_archive.write import archive
|
||||
from claude_code.entries import AssistantTurn
|
||||
from reading_cost.measure import Checkout, Reading
|
||||
from failure import Failure
|
||||
from reading_cost import figures, main, measure
|
||||
@@ -77,7 +78,7 @@ class Measure(unittest.TestCase):
|
||||
self.assertIsNotNone(measure.measure([jsonl([said("1", "m1", command="git -c user.name=t commit -m x")])], Checkout("/c/p", "/c/p", frozenset()), "/h"))
|
||||
|
||||
def test_skips_a_usage_that_is_not_a_count(self):
|
||||
self.assertEqual(measure.tokens_in([{"message": {"id": "m", "usage": {"input_tokens": "7", "cache_creation_input_tokens": 2}}}]), 2)
|
||||
self.assertEqual(measure.tokens_in([AssistantTurn(message_id="m", usage={"input_tokens": "7", "cache_creation_input_tokens": 2})]), 2)
|
||||
|
||||
|
||||
class Figures(unittest.TestCase):
|
||||
|
||||
@@ -134,7 +134,7 @@ def archived_rounds(reviewer: str, model: str, root: pathlib.Path, home: pathlib
|
||||
def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Round]:
|
||||
copies = sorted(chunk.glob("*.jsonl"))
|
||||
began = parse_entry(first_timestamped_line(copies[0].read_bytes())) if copies else None
|
||||
if not began or not isinstance(began.get("timestamp"), str):
|
||||
if not began or not began.timestamp:
|
||||
return []
|
||||
found = []
|
||||
for meta in sorted(chunk.glob("session/*/subagents/*.meta.json")):
|
||||
@@ -142,7 +142,7 @@ def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Roun
|
||||
if not isinstance(described, dict) or described.get("agentType") != reviewer:
|
||||
continue
|
||||
entries = read_transcript(str(meta.with_name(meta.name.removesuffix(".meta.json") + ".jsonl")))
|
||||
if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began["timestamp"]), entries)):
|
||||
if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began.timestamp), entries)):
|
||||
found.append(one)
|
||||
return found
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import pathlib
|
||||
import re
|
||||
import typing
|
||||
|
||||
from claude_code.entries import assistant_parts, content, part_text, parts
|
||||
from claude_code.entries import AssistantTurn, Entry, Text, UserText
|
||||
from rung_replay.search import Replay
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
@@ -34,20 +34,19 @@ class Round:
|
||||
return stamp_seconds(self.at)
|
||||
|
||||
|
||||
def round_from(reviewer: str, root: str, began: float, entries: list[dict]) -> Round | None:
|
||||
def round_from(reviewer: str, root: str, began: float, entries: list[Entry]) -> Round | None:
|
||||
"""The round a reviewer's transcript holds: its first entry's prompt, and its last text. None without a 🔴 or 🟡 finding to hold a rung to."""
|
||||
reports = [text for found in entries for part in assistant_parts(found) if (text := part_text(part))]
|
||||
first = entries[0] if entries else {}
|
||||
reports = [part.text for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, Text) and part.text]
|
||||
first = entries[0] if entries else Entry()
|
||||
if not stamped(first) or not reports or not (findings := recorded_findings(reports[-1])):
|
||||
return None
|
||||
prompt = content(first)
|
||||
prompt = prompt if isinstance(prompt, str) else "".join(part_text(part) or "" for part in parts(first))
|
||||
return Round(reviewer, root, first.get("gitBranch", ""), first["timestamp"], began, prompt, tuple(findings))
|
||||
prompt = first.text if isinstance(first, UserText) else ""
|
||||
return Round(reviewer, root, first.git_branch, first.timestamp, began, prompt, tuple(findings))
|
||||
|
||||
|
||||
def stamped(entry: dict) -> bool:
|
||||
def stamped(entry: Entry) -> bool:
|
||||
try:
|
||||
return isinstance(entry.get("timestamp"), str) and bool(stamp_seconds(entry["timestamp"]))
|
||||
return bool(entry.timestamp) and bool(stamp_seconds(entry.timestamp))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from review_replay.rounds import authored_by, lost_inputs, recorded_findings, report, reviewed_path, Round, round_from, sample, stamp_seconds
|
||||
from loop_claude.effects import Effects, system_effects
|
||||
from claude_code.entries import entry_from
|
||||
from failure import Failure
|
||||
from review_replay import main
|
||||
from rung_replay.search import Miss, Replay, Tried
|
||||
@@ -32,6 +33,10 @@ def entry(kind, text, **fields):
|
||||
return {"type": kind, "message": {"content": [{"type": "text", "text": text}]}, **fields}
|
||||
|
||||
|
||||
def kinds(found):
|
||||
return [entry_from(one) for one in found]
|
||||
|
||||
|
||||
def a_round(at, findings=("🔴 a",)):
|
||||
return Round("stability-reviewer", "/code/p", "x", f"2026-10-14T10:00:0{at}Z", 0, "Review it.", tuple(findings))
|
||||
|
||||
@@ -52,19 +57,19 @@ class RoundFrom(unittest.TestCase):
|
||||
entry("assistant", "Reading."),
|
||||
entry("assistant", "BLOCK\n\n🔴 a.py:1 breaks"),
|
||||
]
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, entries), Round("stability-reviewer", "/code/p", "x", "2026-10-14T10:00:00.500Z", 5.0, "Review branch x in /code/p-x.", ("🔴 a.py:1 breaks",)))
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, entries).seconds, 1791972000.5)
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)), Round("stability-reviewer", "/code/p", "x", "2026-10-14T10:00:00.500Z", 5.0, "Review branch x in /code/p-x.", ("🔴 a.py:1 breaks",)))
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)).seconds, 1791972000.5)
|
||||
|
||||
def test_reads_a_prompt_given_as_a_string(self):
|
||||
start = {"type": "user", "message": {"content": "Review branch x in /code/p-x."}, "timestamp": "2026-10-14T10:00:00Z", "gitBranch": "x"}
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, [start, entry("assistant", "🔴 a")]).prompt, "Review branch x in /code/p-x.")
|
||||
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "🔴 a")])).prompt, "Review branch x in /code/p-x.")
|
||||
|
||||
def test_skips_a_round_with_no_red_or_yellow_finding_or_no_report(self):
|
||||
start = entry("user", "Review.", timestamp="2026-10-14T10:00:00Z", cwd="/code/p", gitBranch="x")
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, [start, entry("assistant", "APPROVE\n\n🟢 nit")]))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, [start]))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, []))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, [{**start, "timestamp": "yesterday"}, entry("assistant", "🔴 a")]))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "APPROVE\n\n🟢 nit")])))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start])))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([])))
|
||||
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([{**start, "timestamp": "yesterday"}, entry("assistant", "🔴 a")])))
|
||||
|
||||
|
||||
class ReviewedPath(unittest.TestCase):
|
||||
|
||||
@@ -13,8 +13,8 @@ from branch_chunks.cut import BranchChunk
|
||||
from transcript_archive.project_key import checkout_of
|
||||
from routing_figures.chunk import Chunk, chunk_figures, ChunkFigures, Subagent
|
||||
from git_cli.git_repo import default_ref, Git
|
||||
from claude_code.entries import Entry, parse_entry
|
||||
from failure import Failure
|
||||
from claude_code.entries import parse_entry
|
||||
from records.file_writes import replace_file
|
||||
from transcript_archive.outcomes import VERDICT_AT
|
||||
|
||||
@@ -104,7 +104,7 @@ def branch_history(checkout: str, git: Git) -> list[BranchChunk] | Failure:
|
||||
return ref if isinstance(ref, Failure) else branch_chunks(ref, functools.partial(git, cwd=checkout, timeout=GIT_TIMEOUT, guarded=True))
|
||||
|
||||
|
||||
def entries(path: pathlib.Path) -> list[dict]:
|
||||
def entries(path: pathlib.Path) -> list[Entry]:
|
||||
with open(path, "rb") as handle:
|
||||
return [entry for line in handle if (entry := parse_entry(line))]
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import collections
|
||||
import dataclasses
|
||||
import re
|
||||
|
||||
from claude_code.entries import assistant_parts, parts, tool_call, tool_result
|
||||
from claude_code.entries import AssistantTurn, Entry, tool_calls, ToolResults
|
||||
from branch_chunks.cut import BranchChunk
|
||||
from git_cli.git_command import commits
|
||||
from routing_figures.dollars import costed, Dollars, Message, messages, recorded_costs, Spent
|
||||
@@ -21,7 +21,7 @@ class Subagent:
|
||||
"""kind is the agentType and launched_by the toolUseId of the launching Agent call, as the subagent's .meta.json names them, each empty where it names none; entries are its transcript's entries."""
|
||||
kind: str
|
||||
launched_by: str
|
||||
entries: list[dict]
|
||||
entries: list[Entry]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -29,7 +29,7 @@ class Chunk:
|
||||
"""outcomes holds each outcomes file archived for the chunk, one per transcript copy, each opening with its task; worker is its finished transcript's entries."""
|
||||
project: str
|
||||
outcomes: list[str]
|
||||
worker: list[dict]
|
||||
worker: list[Entry]
|
||||
subagents: list[Subagent]
|
||||
|
||||
|
||||
@@ -137,15 +137,9 @@ def escalations(launches: dict[str, tuple[str, dict]]) -> collections.Counter[in
|
||||
return collections.Counter(int(found[1]) for found in opened if found)
|
||||
|
||||
|
||||
def agent_calls(worker: list[dict]) -> dict[str, tuple[str, dict]]:
|
||||
def agent_calls(worker: list[Entry]) -> dict[str, tuple[str, dict]]:
|
||||
"""The timestamp and the arguments of each Agent call, by the call's id."""
|
||||
found = {}
|
||||
for entry in worker:
|
||||
for part in assistant_parts(entry):
|
||||
call = tool_call(part)
|
||||
if call and call[0] == "Agent" and isinstance(part, dict) and isinstance(call[1], dict):
|
||||
found[str(part.get("id"))] = (str(entry.get("timestamp", "")), call[1])
|
||||
return found
|
||||
return {call.id: (turn.timestamp, call.input) for turn, call in tool_calls(worker) if call.name == "Agent"}
|
||||
|
||||
|
||||
def rung_at(cycles: list[tuple[str, str]], timestamp: str) -> Rung | str:
|
||||
@@ -160,23 +154,22 @@ def to_the_second(timestamp: str) -> str:
|
||||
return timestamp[:19] + "Z"
|
||||
|
||||
|
||||
def subagent_rung(entries: list[dict], arguments: dict, worker: Rung | str) -> Rung:
|
||||
def subagent_rung(entries: list[Entry], arguments: dict, worker: Rung | str) -> Rung:
|
||||
"""The model the subagent ran, at the effort its Agent call set, or else at the worker's."""
|
||||
model = next((message["model"] for entry in entries if entry.get("type") == "assistant" and isinstance(message := entry.get("message"), dict) and isinstance(message.get("model"), str)), str(arguments.get("model") or "unknown"))
|
||||
model = next((entry.model for entry in entries if isinstance(entry, AssistantTurn) and entry.model), str(arguments.get("model") or "unknown"))
|
||||
effort = arguments.get("effort") or (worker.effort if isinstance(worker, Rung) else "")
|
||||
return Rung(model, str(effort))
|
||||
|
||||
|
||||
def results(entries: list[dict]) -> dict[str, str]:
|
||||
def results(entries: list[Entry]) -> dict[str, str]:
|
||||
"""Each tool result's text by the id of the call it answers."""
|
||||
return {result.tool_use_id: result.text for entry in entries for part in parts(entry) if (result := tool_result(part))}
|
||||
return {result.tool_use_id: result.text for entry in entries if isinstance(entry, ToolResults) for result in entry.results}
|
||||
|
||||
|
||||
def commit_titles(worker: list[dict], home: str) -> list[str]:
|
||||
def commit_titles(worker: list[Entry], home: str) -> list[str]:
|
||||
"""The title of each commit the worker's Bash calls made, in order, each once: a commit retried after a failed hook repeats its title."""
|
||||
return list(dict.fromkeys(
|
||||
commit.messages[0].split("\n", 1)[0]
|
||||
for entry in worker for part in assistant_parts(entry)
|
||||
if (found := tool_call(part)) and found[0] == "Bash" and isinstance(found[1], dict)
|
||||
for commit in commits(simple_commands(str(found[1].get("command", "")), str(entry.get("cwd") or "/"), home), home) if commit.messages
|
||||
for turn, call in tool_calls(worker) if call.name == "Bash"
|
||||
for commit in commits(simple_commands(str(call.input.get("command", "")), turn.cwd or "/", home), home) if commit.messages
|
||||
))
|
||||
|
||||
@@ -3,7 +3,7 @@ import collections
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from claude_code.entries import assistant_parts, tool_call
|
||||
from claude_code.entries import AssistantTurn, CostState, Entry, ToolCall
|
||||
from routing_figures.prices import listed
|
||||
|
||||
DIMENSIONS = ("project", "subagent type", "rung", "tool")
|
||||
@@ -31,31 +31,25 @@ class Spent(typing.NamedTuple):
|
||||
message: Message
|
||||
|
||||
|
||||
def messages(entries: list[dict]) -> list[Message]:
|
||||
def messages(entries: list[Entry]) -> list[Message]:
|
||||
"""The assistant messages, counted once each, each with the tools whose results it took in, or `prompt` where it took none in."""
|
||||
# Claude Code writes one entry per content block of a message, each with the message's usage. The usage is the same in every entry (none of 1,681 multi-entry messages in the archive of 2026-10-28 differed), so the first is kept, where reading_cost's tokens_in keeps the last.
|
||||
kept: dict[str, tuple[str, dict, list[str]]] = {}
|
||||
kept: dict[str, tuple[AssistantTurn, list[str]]] = {}
|
||||
for entry in entries:
|
||||
if entry.get("type") != "assistant" or not isinstance(message := entry.get("message"), dict):
|
||||
continue
|
||||
found = kept.setdefault(str(message.get("id") or entry.get("uuid")), (str(entry.get("timestamp", "")), message, []))
|
||||
found[2].extend(name for part in assistant_parts(entry) if (call := tool_call(part)) and isinstance(name := call[0], str))
|
||||
if isinstance(entry, AssistantTurn):
|
||||
found = kept.setdefault(entry.message_id or entry.uuid, (entry, []))
|
||||
found[1].extend(part.name for part in entry.parts if isinstance(part, ToolCall))
|
||||
counted = []
|
||||
before = [PROMPT]
|
||||
for at, message, called in kept.values():
|
||||
model = str(message.get("model") or "")
|
||||
counted.append(Message(at, before, model, listed(model, message.get("usage"))))
|
||||
for turn, called in kept.values():
|
||||
counted.append(Message(turn.timestamp, before, turn.model, listed(turn.model, turn.usage)))
|
||||
before = called or [PROMPT]
|
||||
return counted
|
||||
|
||||
|
||||
# {"type": "cost-state", "totalCostUSD": 40.1, "modelUsage": {"claude-opus-5-5": {"costUSD": 40.1, "outputTokens": 334504, ...}}, ...}
|
||||
def recorded_costs(worker: list[dict]) -> dict[str, float]:
|
||||
"""The cost the worker transcript's last cost-state entry records for each model. Claude Code counts every call of the session in it, the subagents' and the compactions' too."""
|
||||
for entry in reversed(worker):
|
||||
if entry.get("type") == "cost-state" and isinstance(by_model := entry.get("modelUsage"), dict):
|
||||
return {model: cost for model, usage in by_model.items() if isinstance(usage, dict) and isinstance(cost := usage.get("costUSD"), (int, float))}
|
||||
return {}
|
||||
def recorded_costs(worker: list[Entry]) -> dict[str, float]:
|
||||
"""The cost the worker transcript's last cost-state entry records for each model."""
|
||||
return next((entry.costs for entry in reversed(worker) if isinstance(entry, CostState)), {})
|
||||
|
||||
|
||||
def costed(spent: list[Spent], judged: list[tuple[str, float]], project: str, recorded: dict[str, float]) -> tuple[Dollars, float]:
|
||||
|
||||
@@ -19,6 +19,7 @@ from branch_chunks.cut import BranchChunk
|
||||
from routing_figures.calls import Call, calls, totals
|
||||
from routing_figures.chunk import Chunk, chunk_figures, ChunkFigures, Reports, Subagent
|
||||
from routing_figures.state import due
|
||||
from claude_code.entries import entry_from
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from test_env import isolated_env
|
||||
@@ -42,6 +43,14 @@ def cost_state(**costs):
|
||||
return {"type": "cost-state", "totalCostUSD": sum(costs.values()), "modelUsage": {model.replace("_", "-"): {"costUSD": cost} for model, cost in costs.items()}}
|
||||
|
||||
|
||||
def chunk_of(project, outcomes, worker, subagents):
|
||||
return Chunk(project, outcomes, [entry_from(entry) for entry in worker], subagents)
|
||||
|
||||
|
||||
def subagent_of(kind, launched_by, entries):
|
||||
return Subagent(kind, launched_by, [entry_from(entry) for entry in entries])
|
||||
|
||||
|
||||
def call(name, id_="t1", **arguments):
|
||||
return {"type": "tool_use", "id": id_, "name": name, "input": arguments}
|
||||
|
||||
@@ -80,7 +89,7 @@ class Measured(unittest.TestCase):
|
||||
message("m3", "2026-10-14T11:00:00.000Z", {"type": "text", "text": "on"}),
|
||||
]
|
||||
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T10:30:00Z", outcomes.stepped_up(Route(HIGH).worker_line())) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
|
||||
found = chunk_figures(Chunk("log", [kept], worker, []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [kept], worker, []), "/h")
|
||||
self.assertEqual(found.dollars["project"], {"log": 6.0})
|
||||
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0})
|
||||
self.assertEqual(found.dollars["rung"], {str(LOW): 4.0, str(HIGH): 2.0})
|
||||
@@ -89,7 +98,7 @@ class Measured(unittest.TestCase):
|
||||
|
||||
def test_counts_the_judges_recorded_cost_under_the_judge_and_the_rung_it_ran_on(self):
|
||||
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "judge-rung", "", judge_cost=0.5)) + stamped("2026-10-14T12:00:00Z", outcomes.finished(LOW_ROUTE, "m", "judge-rung", judge_cost=0.25)) + stamped("2026-10-14T12:00:00Z", outcomes.rejected("why", "{}", "other-rung", judge_cost=1.0))
|
||||
found = chunk_figures(Chunk("log", [kept], [message("m1", "2026-10-14T10:00:00Z")], []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T10:00:00Z")], []), "/h")
|
||||
self.assertEqual(found.dollars["project"], {"log": 3.75})
|
||||
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "judge": 1.75})
|
||||
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, "judge-rung": 0.75, "other-rung": 1.0})
|
||||
@@ -102,10 +111,10 @@ class Measured(unittest.TestCase):
|
||||
result("a2", "y" * 5),
|
||||
]
|
||||
subagents = [
|
||||
Subagent("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z", model="claude-sonnet-5-5"), result("r", "z" * 20)]),
|
||||
Subagent("", "a2", [message("s2", "2026-10-14T10:00:01Z")]),
|
||||
subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z", model="claude-sonnet-5-5"), result("r", "z" * 20)]),
|
||||
subagent_of("", "a2", [message("s2", "2026-10-14T10:00:01Z")]),
|
||||
]
|
||||
found = chunk_figures(Chunk("log", [stamped("2026-10-14T11:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j"))], worker, subagents), "/h")
|
||||
found = chunk_figures(chunk_of("log", [stamped("2026-10-14T11:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j"))], worker, subagents), "/h")
|
||||
self.assertEqual(found.delegated, {"claude-sonnet-5-5 at high effort": 1, str(LOW): 1})
|
||||
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "Explore": 3.0})
|
||||
self.assertEqual(found.reports, {"Explore": Reports(2, 35, 20)})
|
||||
@@ -115,16 +124,16 @@ class Measured(unittest.TestCase):
|
||||
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", prompt="Escalation trigger 2: the test failed twice"), call("Agent", "a2", prompt="Escalation trigger 2: again"), call("Agent", "a3", prompt="Escalation trigger 3: a 🔴 in round 2")),
|
||||
message("m2", "2026-10-14T10:01:00Z", call("Agent", "a4", prompt="Find the callers. Escalation trigger 1: not at the start"), call("Agent", "a5", prompt="Escalation trigger x: no number")),
|
||||
]
|
||||
self.assertEqual(chunk_figures(Chunk("log", [], worker, []), "/h").escalated, {2: 2, 3: 1})
|
||||
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h").escalated, {2: 2, 3: 1})
|
||||
|
||||
def test_counts_review_rounds_past_each_passs_first_once_per_title(self):
|
||||
commit = lambda id_, title: message(id_, "2026-10-14T10:00:00Z", call("Bash", command=f"git commit -m '{title}'"))
|
||||
worker = [commit("m1", "Answer stability round 1: 1 🔴, 0 🟡"), commit("m2", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m3", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m4", "Answer prose round 1: 0 🔴, 1 🟡")]
|
||||
self.assertEqual(chunk_figures(Chunk("log", [], worker, []), "/h").rounds_past_first, 1)
|
||||
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h").rounds_past_first, 1)
|
||||
|
||||
def fixed_later(self, history, route=LOW_ROUTE):
|
||||
"""The later fixes measured counts for a chunk that route started, which LOW_ROUTE does by item 7, on a project whose default branch holds history, None where it could not be read."""
|
||||
return chunk_figures(Chunk("log", [stamped("2026-10-14T10:00:00Z", outcomes.finished(route, "m", "j"))], [], []), "/h", history).fixed_later
|
||||
return chunk_figures(chunk_of("log", [stamped("2026-10-14T10:00:00Z", outcomes.finished(route, "m", "j"))], [], []), "/h", history).fixed_later
|
||||
|
||||
def test_counts_the_later_fixes_of_the_chunk_that_shipped_the_item_its_route_names(self):
|
||||
self.assertEqual(self.fixed_later([shipping(6, fixed_by=("c", "d", "e")), shipping(5, 7, fixed_by=("c", "d"))]), 2)
|
||||
@@ -144,7 +153,7 @@ class Measured(unittest.TestCase):
|
||||
|
||||
def test_starts_no_route_for_a_chunk_no_item_routed(self):
|
||||
kept = stamped("2026-10-14T10:00:00Z", outcomes.finished(Route(LOW).worker_line(), "m", "j"))
|
||||
found = chunk_figures(Chunk("log", [kept], [message("m1", "2026-10-14T09:00:00Z")], []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T09:00:00Z")], []), "/h")
|
||||
self.assertIsNone(found.start)
|
||||
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0})
|
||||
|
||||
@@ -155,15 +164,15 @@ class Measured(unittest.TestCase):
|
||||
message("m2", "2026-10-14T10:30:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet")),
|
||||
message("m3", "2026-10-14T10:31:00Z", call("Agent", "a2", subagent_type="Explore")),
|
||||
]
|
||||
subagents = [Subagent("Explore", "a1", [result("r", "x")]), Subagent("Explore", "a2", [message("s2", "2026-10-14T10:31:01Z", model="claude-opus-5-5")])]
|
||||
found = chunk_figures(Chunk("log", [kept], worker, subagents), "/h")
|
||||
subagents = [subagent_of("Explore", "a1", [result("r", "x")]), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:31:01Z", model="claude-opus-5-5")])]
|
||||
found = chunk_figures(chunk_of("log", [kept], worker, subagents), "/h")
|
||||
self.assertEqual(found.delegated, {"sonnet at high effort": 1, str(HIGH): 1})
|
||||
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, str(HIGH): 6.0})
|
||||
|
||||
def test_names_a_subagent_by_its_meta_or_else_its_call_or_else_unknown(self):
|
||||
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Plan"))]
|
||||
subagents = [Subagent("stability-reviewer", "a1", []), Subagent("", "a2", []), Subagent("", "", [])]
|
||||
self.assertEqual(chunk_figures(Chunk("log", [], worker, subagents), "/h").reports, {"stability-reviewer": Reports(1, 0, 0), "Plan": Reports(1, 0, 0), "unknown": Reports(1, 0, 0)})
|
||||
subagents = [subagent_of("stability-reviewer", "a1", []), subagent_of("", "a2", []), subagent_of("", "", [])]
|
||||
self.assertEqual(chunk_figures(chunk_of("log", [], worker, subagents), "/h").reports, {"stability-reviewer": Reports(1, 0, 0), "Plan": Reports(1, 0, 0), "unknown": Reports(1, 0, 0)})
|
||||
|
||||
def test_counts_a_message_without_usage_or_a_priced_model_as_nothing_and_splits_a_cost_evenly_across_tools(self):
|
||||
worker = [
|
||||
@@ -173,20 +182,20 @@ class Measured(unittest.TestCase):
|
||||
{"type": "assistant", "uuid": "u3", "message": {"model": "claude-opus-5-5", "content": []}},
|
||||
{"type": "assistant", "uuid": "u4", "message": {"model": "claude-unheard-of", "content": [], "usage": {"input_tokens": 9_000_000}}},
|
||||
]
|
||||
found = chunk_figures(Chunk("log", [], worker, []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [], worker, []), "/h")
|
||||
self.assertEqual(found.dollars["tool"], {"prompt": 4.0, "Read": 4.0, "Grep": 4.0})
|
||||
self.assertEqual(found.dollars["project"], {"log": 12.0})
|
||||
|
||||
def test_prices_input_cache_writes_by_their_lifetime_cache_reads_and_output_at_the_models_list_prices(self):
|
||||
usage = {"input_tokens": 1_000_000, "cache_creation_input_tokens": 2_000_000, "cache_creation": {"ephemeral_5m_input_tokens": 1_000_000, "ephemeral_1h_input_tokens": 1_000_000}, "cache_read_input_tokens": 1_000_000, "output_tokens": 1_000_000}
|
||||
opus = chunk_figures(Chunk("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h")
|
||||
opus = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h")
|
||||
self.assertAlmostEqual(opus.dollars["project"]["log"], 4 + 5 + 8 + 0.2 + 20)
|
||||
sonnet = chunk_figures(Chunk("log", [], [message("m1", "2026-10-14T10:00:00Z", model="claude-sonnet-5-5", usage=usage)], []), "/h")
|
||||
sonnet = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", model="claude-sonnet-5-5", usage=usage)], []), "/h")
|
||||
self.assertAlmostEqual(sonnet.dollars["project"]["log"], 2 + 2.5 + 4 + 0.2 + 10)
|
||||
|
||||
def test_counts_a_cache_write_without_a_lifetime_as_a_five_minute_one(self):
|
||||
usage = {"cache_creation_input_tokens": 1_000_000}
|
||||
found = chunk_figures(Chunk("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h")
|
||||
self.assertAlmostEqual(found.dollars["project"]["log"], 5)
|
||||
|
||||
def test_splits_the_cost_a_chunks_last_cost_state_records_for_each_model_across_its_messages_by_list_price(self):
|
||||
@@ -196,32 +205,32 @@ class Measured(unittest.TestCase):
|
||||
cost_state(claude_opus_5_5=1.0),
|
||||
cost_state(claude_opus_5_5=9.0, claude_haiku_5_5=0.5),
|
||||
]
|
||||
subagents = [Subagent("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]
|
||||
found = chunk_figures(Chunk("log", [], worker, subagents), "/h")
|
||||
subagents = [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]
|
||||
found = chunk_figures(chunk_of("log", [], worker, subagents), "/h")
|
||||
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0, "Explore": 3.0})
|
||||
self.assertEqual(found.dollars["project"], {"log": 9.0})
|
||||
self.assertEqual(found.unattributed, 0.5)
|
||||
|
||||
def test_counts_the_subagents_the_worker_launched_that_have_no_transcript(self):
|
||||
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Explore"), call("Read", "r1", file_path="/c/a"))]
|
||||
found = chunk_figures(Chunk("log", [], worker, [Subagent("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]), "/h")
|
||||
found = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]), "/h")
|
||||
self.assertEqual(found.lacking, 1)
|
||||
unnamed = chunk_figures(Chunk("log", [], worker, [Subagent("", "", [message("s1", "2026-10-14T10:00:01Z")]), Subagent("", "", [message("s2", "2026-10-14T10:00:02Z")])]), "/h")
|
||||
unnamed = chunk_figures(chunk_of("log", [], worker, [subagent_of("", "", [message("s1", "2026-10-14T10:00:01Z")]), subagent_of("", "", [message("s2", "2026-10-14T10:00:02Z")])]), "/h")
|
||||
self.assertEqual(unnamed.lacking, 0)
|
||||
nested = [message("s1", "2026-10-14T10:00:01Z", call("Agent", "n1", subagent_type="Explore"))]
|
||||
launched = chunk_figures(Chunk("log", [], worker, [Subagent("Explore", "a1", nested), Subagent("Explore", "a2", [message("s2", "2026-10-14T10:00:02Z")]), Subagent("Explore", "n1", [message("s3", "2026-10-14T10:00:03Z")])]), "/h")
|
||||
launched = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:00:02Z")]), subagent_of("Explore", "n1", [message("s3", "2026-10-14T10:00:03Z")])]), "/h")
|
||||
self.assertEqual(launched.lacking, 0)
|
||||
self.assertEqual(chunk_figures(Chunk("log", [], worker, [Subagent("Explore", "a1", nested), Subagent("Explore", "a2", [])]), "/h").lacking, 1)
|
||||
self.assertEqual(chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [])]), "/h").lacking, 1)
|
||||
|
||||
def test_charges_a_message_without_a_timestamp_to_the_rung_of_the_last_outcome(self):
|
||||
undated = message("m1", "")
|
||||
del undated["timestamp"]
|
||||
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
|
||||
self.assertEqual(chunk_figures(Chunk("log", [kept], [undated], []), "/h").dollars["rung"], {str(HIGH): 2.0})
|
||||
self.assertEqual(chunk_figures(chunk_of("log", [kept], [undated], []), "/h").dollars["rung"], {str(HIGH): 2.0})
|
||||
|
||||
def test_keeps_list_prices_for_a_model_the_cost_state_does_not_name(self):
|
||||
worker = [message("m1", "2026-10-14T10:00:00Z"), message("m2", "2026-10-14T10:01:00Z", model="claude-sonnet-5-5"), cost_state(claude_opus_5_5=3.0)]
|
||||
found = chunk_figures(Chunk("log", [], worker, []), "/h")
|
||||
found = chunk_figures(chunk_of("log", [], worker, []), "/h")
|
||||
self.assertEqual(found.dollars["project"], {"log": 4.0})
|
||||
self.assertEqual(found.unattributed, 0.0)
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ import stat
|
||||
from rule_hook.rules import Breach
|
||||
from rule_hook.call import Call, FILE_TOOLS, text
|
||||
from claude_code.transcript import current_thread, read_transcript
|
||||
from claude_code.entries import Entry, tool_calls, ToolResults
|
||||
from failure import Failure
|
||||
from claude_code.entries import parts, tool_call, tool_result, tool_use_id
|
||||
from command_lines.shell_command import resolved, simple_commands
|
||||
|
||||
# The lines a Read without a limit returns.
|
||||
@@ -116,7 +116,7 @@ def shell_asked(command: str, cwd: str, home: str) -> Asked | None:
|
||||
return None
|
||||
|
||||
|
||||
def shown_lines(thread: list[dict], asker: str, path: str, changed: str, home: str) -> dict[int, str]:
|
||||
def shown_lines(thread: list[Entry], asker: str, path: str, changed: str, home: str) -> dict[int, str]:
|
||||
"""Each line number of path that a successful read in the thread showed, with the line its latest read showed; none against a Read when the file changed after the agent last read it with Read or wrote it with Edit, MultiEdit or Write, or when it never did.
|
||||
|
||||
changed is when the file last changed, stamped as the transcript stamps its entries, so the two compare as text."""
|
||||
@@ -131,20 +131,14 @@ def shown_lines(thread: list[dict], asker: str, path: str, changed: str, home: s
|
||||
return shown
|
||||
|
||||
|
||||
def results_on(thread: list[dict], path: str, home: str) -> list[Used]:
|
||||
def results_on(thread: list[Entry], path: str, home: str) -> list[Used]:
|
||||
"""Each successful result in the thread of a call that read lines of path or wrote it."""
|
||||
calls: dict[str, tuple[object, Asked | None]] = {}
|
||||
found: list[Used] = []
|
||||
for entry in thread:
|
||||
cwd = str(entry.get("cwd", ""))
|
||||
for part in parts(entry):
|
||||
if (call := tool_call(part)) and (called := tool_use_id(part)):
|
||||
asked = asked_lines(*call, cwd, home)
|
||||
if (asked.path if asked else written_path(*call, cwd, home)) == path:
|
||||
calls[called] = (call[0], asked)
|
||||
elif (result := tool_result(part)) and not result.failed and (use := calls.get(result.tool_use_id)):
|
||||
found.append(Used(use[0], use[1], result.text, text(entry, "timestamp")))
|
||||
return found
|
||||
calls: dict[str, tuple[str, Asked | None]] = {}
|
||||
for turn, call in tool_calls(thread):
|
||||
asked = asked_lines(call.name, call.input, turn.cwd, home)
|
||||
if call.id and (asked.path if asked else written_path(call.name, call.input, turn.cwd, home)) == path:
|
||||
calls[call.id] = (call.name, asked)
|
||||
return [Used(*calls[result.tool_use_id], result.text, entry.timestamp) for entry in thread if isinstance(entry, ToolResults) for result in entry.results if not result.failed and result.tool_use_id in calls]
|
||||
|
||||
|
||||
def written_path(tool: object, given: object, cwd: str, home: str) -> str:
|
||||
|
||||
@@ -3,7 +3,6 @@ import typing
|
||||
|
||||
from rule_hook.rules import Breach
|
||||
from rule_hook.call import Call, text
|
||||
from claude_code.entries import content
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from rule_hook.git import guarded_git
|
||||
@@ -11,6 +10,7 @@ from model_routing.worker_rungs import handed
|
||||
from model_routing.ladder import position, review_rung, ReviewKind, Rung
|
||||
from review_rules.review_passes import prose, REVIEWERS
|
||||
from claude_code.transcript import read_transcript
|
||||
from claude_code.entries import UserText
|
||||
|
||||
# The pass each agent under agents/ reviews.
|
||||
PASS_OF = {agent: kind for kind, agent in REVIEWERS.items()}
|
||||
@@ -57,9 +57,8 @@ def skill_breaches(call: Call, environ: typing.Mapping[str, str]) -> list[Breach
|
||||
def opening_prompt(transcript: str) -> str:
|
||||
"""The prompt a transcript opens with, such as a subagent's from its Agent call; empty where it cannot be read."""
|
||||
entries = read_transcript(transcript) if transcript else []
|
||||
first = entries[0] if isinstance(entries, list) and entries else {}
|
||||
prompt = content(first) if first.get("type") == "user" else None
|
||||
return prompt.lstrip() if isinstance(prompt, str) else ""
|
||||
first = entries[0] if isinstance(entries, list) and entries else None
|
||||
return first.text.lstrip() if isinstance(first, UserText) else ""
|
||||
|
||||
|
||||
def review_kind(given: dict[str, typing.Any]) -> ReviewKind | None:
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 277 | 0.7.0 | | question | **Map the size of the code an item names onto S, on a scale the human picks, and have `score-outcomes` print that S for the item being scored and for each finished item it lists.** | 3 | 2 | 2 | 5 | 4 | 1.1 by 4, 1.4 by 3 | 2.4 |
|
||||
| 1 | 0.7.0 | | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 1 | 2 | 1 | 2 | 1 | | 0.0 |
|
||||
| 221 | 0.7.0 | | | **Parse each transcript entry once, where `bin/claude_code/transcript.py` reads it, into a typed value naming its kind, so no consumer works out an entry's kind from a raw `dict`.** | 4 | 1 | 3 | 6 | 6 | code health by 4 | 3.0 |
|
||||
| 105 | 0.7.0 | | decision | **Compare review rounds per pass and reading cost for the five chunks before and after the `worker:` decision moved the principles out of the worker's context, and file an item if either rose.** | 1 | 2 | 1 | 2 | 2 | 1.4 by 6 | 2.7 |
|
||||
| 175 | 0.7.0 | | | **Compare the first five chunks loop-claude starts with `chunk-preflight` against the five before them, and file an item where the preflight did not pay off.** | 1 | 2 | 1 | 2 | 2 | 1.4 by 3, 1.5 by 3 | 2.6 |
|
||||
| 112 | 0.7.0 | | | **Count an item as shipped only where its chunk built it, so an item a chunk dropped from the plan gets no outcome.** | 3 | 2 | 2 | 5 | 3 | 1.1 by 6 | 2.5 |
|
||||
@@ -84,10 +83,6 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost
|
||||
|
||||
The README line covers the bench's transcripts in `workflow-bench/logs/` too (goal 3).
|
||||
|
||||
### 221. Parse each transcript entry once, where `bin/claude_code/transcript.py` reads it, into a typed value naming its kind, so no consumer works out an entry's kind from a raw `dict`.
|
||||
|
||||
"Validate data and build DTOs as early as possible", in `principles/technical-principles.md`, raised 🟢 by the principles audit of item 210's PR. `read_transcript` (`bin/claude_code/transcript.py:16`) returns raw `dict`s, so each consumer, such as `render` beside it and `since_step` in `bin/loop_claude/escalation_mark.py`, calls the helpers in `bin/claude_code/entries.py` (`human_text`, `loop_text`, `assistant_parts`, `tool_call`) to work out an entry's kind again. Parse each line once into types such as a human turn, a loop-claude prompt and an assistant turn, and have every consumer of `claude_code.entries` under `bin/` take those types. The change touches the escalation mark, which the comprehension seats found hard at `bin/loop_claude/cycle/judge.py:153` `escalation_mark` before item 210 moved it: make `since_step` in `bin/loop_claude/escalation_mark.py`, and `Grounds.since_step` and `judge_prompt` in `bin/loop_claude/cycle/judge.py`, easier to read.
|
||||
|
||||
### 105. Compare review rounds per pass and reading cost for the five chunks before and after the `worker:` decision moved the principles out of the worker's context, and file an item if either rose.
|
||||
|
||||
The `worker:` decision is valid while review rounds per pass do not rise. Count the rounds from each PR's `Answer <pass> round <n>:` commits, and take reading cost from `docs/reading-cost.md`. Run it once five chunks have merged after the one that recorded the decision.
|
||||
|
||||
Reference in New Issue
Block a user