Files

177 lines
9.4 KiB
Python

"""The figures of one finished chunk: its dollars, the rung it started on, its review rounds and unfinished verdicts, its subagents' rungs and reports, its escalations and its later fixes."""
import collections
import dataclasses
import re
from claude_code.entries import AssistantTurn, Entry, tool_calls, ToolResults
from branch_chunks.cut import BranchChunk
from git_cli.git_command import commits
from routing_figures.dollars import costed, Dollars, Message, messages, recorded_costs, Spent
from model_routing.route import ESCALATION_MARKER, Route, route_from
from transcript_archive.outcomes import judge_costs, UNFINISHED_AT, workers
from review_rules.round_titles import rounds_past_first
from model_routing.ladder import Rung
from command_lines.shell_command import simple_commands
from project_config.templates import Titles
ESCALATION = re.compile(rf"{ESCALATION_MARKER} ([0-9]+):")
@dataclasses.dataclass(frozen=True)
class Subagent:
"""kind is the agentType and launched_by the toolUseId of the launching Agent call, as the subagent's .meta.json names them, each empty where it names none; entries are its transcript's entries."""
kind: str
launched_by: str
entries: list[Entry]
@dataclasses.dataclass(frozen=True)
class Chunk:
"""outcomes holds each outcomes file archived for the chunk, one per transcript copy, each opening with its task; worker is its finished transcript's entries."""
project: str
outcomes: list[str]
worker: list[Entry]
subagents: list[Subagent]
@dataclasses.dataclass(frozen=True)
class Reports:
"""How many subagents of a type reported, the characters of their reports, and the characters of the tool results they read."""
subagents: int
report_chars: int
read_chars: int
def __add__(self, other: "Reports") -> "Reports":
return Reports(self.subagents + other.subagents, self.report_chars + other.report_chars, self.read_chars + other.read_chars)
@dataclasses.dataclass(frozen=True)
class ChunkFigures:
"""dollars maps each of DIMENSIONS to its labels' dollars; start is the rung a routed chunk started on, None where no item routed it; escalated counts the worker's escalations by trigger; unattributed is the recorded cost of a model with no priced message in the chunk; lacking counts the Agent calls, the worker's and the subagents', that have no subagent transcript in the archive; fixed_later counts the later chunks that fixed code it shipped, None where no item routed it, its project's default branch could not be read, or no chunk on that branch shipped its item."""
dollars: Dollars
start: Rung | None
rounds_past_first: int
unfinished: int
delegated: collections.Counter[str]
reports: dict[str, Reports]
escalated: collections.Counter[int]
unattributed: float
lacking: int = 0
fixed_later: int | None = None
def chunk_figures(chunk: Chunk, home: str, titles: Titles, history: list[BranchChunk] | None = None) -> ChunkFigures:
"""titles holds the project's title templates, which each commit title is read through. history is the chunks on the project's default branch, None where it could not be read."""
cycles = sorted((cycle for kept in chunk.outcomes for cycle in workers(kept)), key=lambda cycle: cycle[0])
launches = agent_calls(chunk.worker)
reported = results(chunk.worker)
delegated = [delegation(subagent, launches, reported, cycles) for subagent in chunk.subagents]
dollars, unattributed = chunk_dollars(chunk, cycles, delegated)
start = route_from(cycles[0][1]) if cycles else None
return ChunkFigures(
dollars=dollars,
start=start.rung if start and start.item else None,
rounds_past_first=rounds_past_first([titles.summary(title) for title in commit_titles(chunk.worker, home)]),
unfinished=sum(len(UNFINISHED_AT.findall(kept)) for kept in chunk.outcomes),
delegated=collections.Counter(one.rung for one in delegated),
reports=reports_by_type(delegated),
escalated=escalations(launches),
unattributed=unattributed,
lacking=transcripts_lacking(launches, chunk.subagents),
fixed_later=later_fixes_of(start, history),
)
def chunk_dollars(chunk: Chunk, cycles: list[tuple[str, str]], delegated: list[Delegation]) -> tuple[Dollars, float]:
"""What costed makes of the messages of the worker, each on the rung of the cycle it ran in, and of its subagents, and of the chunk's judge runs."""
spent = [Spent("worker", str(rung_at(cycles, message.at)), message) for message in messages(chunk.worker)]
spent += [Spent(one.kind, one.rung, message) for one in delegated for message in one.messages]
judged = [judged for kept in chunk.outcomes for judged in judge_costs(kept)]
return costed(spent, judged, chunk.project, recorded_costs(chunk.worker))
def reports_by_type(delegated: list[Delegation]) -> dict[str, Reports]:
reports: dict[str, Reports] = {}
for one in delegated:
reports[one.kind] = reports.get(one.kind, Reports(0, 0, 0)) + Reports(1, one.report_chars, one.read_chars)
return reports
@dataclasses.dataclass(frozen=True)
class Delegation:
"""One subagent as the figures count it: its type, the rung it ran on, the characters of its report and of the tool results it read, and its messages."""
kind: str
rung: str
report_chars: int
read_chars: int
messages: list[Message]
def delegation(subagent: Subagent, launches: dict[str, tuple[str, dict]], reported: dict[str, str], cycles: list[tuple[str, str]]) -> Delegation:
"""reported holds the worker's tool results by call id, so the Agent call's result is the subagent's report."""
launched_at, arguments = launches.get(subagent.launched_by, ("", {}))
return Delegation(
kind=subagent.kind or str(arguments.get("subagent_type") or "unknown"),
rung=str(subagent_rung(subagent.entries, arguments, rung_at(cycles, launched_at))),
report_chars=len(reported.get(subagent.launched_by, "")),
read_chars=sum(map(len, results(subagent.entries).values())),
messages=messages(subagent.entries),
)
def transcripts_lacking(launches: dict[str, tuple[str, dict]], subagents: list[Subagent]) -> int:
"""How many Agent calls, the worker's and its subagents', have no subagent transcript in the archive. Each transcript answers one call, so they are the calls less the transcripts."""
return max(0, len(launches) + sum(len(agent_calls(subagent.entries)) for subagent in subagents) - len(subagents))
def later_fixes_of(start: Route | None, history: list[BranchChunk] | None) -> int | None:
"""How many later chunks fixed code shipped by the chunk in history that shipped the item start names. None where no item routed the chunk, history could not be read, or no chunk in it shipped the item."""
# lilleman chose on 2026-10-28 to find a routed chunk's chunk on the default branch by the item its route names, not by commit titles: it needs no title matching, and works where the worker's transcript doesn't show the titles; a chunk whose PR didn't remove its item reads as unknown.
if start is None or not start.item or history is None:
return None
return next((len(chunk.fixed_by) for chunk in history if any(shipped.id == start.item for shipped in chunk.shipped)), None)
def escalations(launches: dict[str, tuple[str, dict]]) -> collections.Counter[int]:
"""How many of the worker's Agent calls open with each escalation trigger's number."""
opened = (ESCALATION.match(str(arguments.get("prompt") or "")) for _, arguments in launches.values())
return collections.Counter(int(found[1]) for found in opened if found)
def agent_calls(worker: list[Entry]) -> dict[str, tuple[str, dict]]:
"""The timestamp and the arguments of each Agent call, by the call's id."""
return {call.id: (turn.timestamp, call.input) for turn, call in tool_calls(worker) if call.name == "Agent"}
def rung_at(cycles: list[tuple[str, str]], timestamp: str) -> Rung | str:
"""The rung of the cycle running at timestamp: the worker named by the first outcome stamped at or after it, or else by the last outcome. A worker line route_from cannot read is returned as its text, and "unrouted" where no outcome names a worker."""
found = next((rung for stamp, rung in cycles if stamp >= to_the_second(timestamp)), cycles[-1][1] if cycles else "unrouted")
route = route_from(found)
return route.rung if route else found
def to_the_second(timestamp: str) -> str:
"""A message's stamp, such as 2026-10-15T00:00:00.123Z, cut to the second, as an outcome is stamped: 2026-10-15T00:00:00Z. Stamps in that one form sort as text in time order."""
return timestamp[:19] + "Z"
def subagent_rung(entries: list[Entry], arguments: dict, worker: Rung | str) -> Rung:
"""The model the subagent ran, at the effort its Agent call set, or else at the worker's."""
model = next((entry.model for entry in entries if isinstance(entry, AssistantTurn) and entry.model), str(arguments.get("model") or "unknown"))
effort = arguments.get("effort") or (worker.effort if isinstance(worker, Rung) else "")
return Rung(model, str(effort))
def results(entries: list[Entry]) -> dict[str, str]:
"""Each tool result's text by the id of the call it answers."""
return {result.tool_use_id: result.text for entry in entries if isinstance(entry, ToolResults) for result in entry.results}
def commit_titles(worker: list[Entry], home: str) -> list[str]:
"""The title of each commit the worker's Bash calls made, in order, each once: a commit retried after a failed hook repeats its title."""
return list(dict.fromkeys(
commit.messages[0].split("\n", 1)[0]
for turn, call in tool_calls(worker) if call.name == "Bash"
for commit in commits(simple_commands(str(call.input.get("command", "")), turn.cwd, home), home) if commit.messages
))