Files

101 lines
6.1 KiB
Python

"""The routing figures as the markdown kept at the archive's root."""
import collections
import statistics
import typing
from routing_figures.calls import Call, Figures, mean, summed
from routing_figures.chunk import ChunkFigures
from routing_figures.dollars import DIMENSIONS
from model_routing.ladder import Ladder
def render(ladder: Ladder, figures: Figures, found: list[Call], at: str, cycle: int, since: str) -> str:
"""The figures as markdown, for the human to read. ladder orders the rungs, and since is the latest reset's stamp, empty for none."""
sections = [
"# Routing figures",
f"Measured {at}, at cycle {cycle} across loop-claude's archive, from {figures.chunks} finished chunk{'' if figures.chunks == 1 else 's'}{f' that finished after the reset of {since}' if since else ''}. {basis(figures)}",
"## Calls for a routing change",
bullets([call.sentence() for call in found]) or "None.",
*[part for dimension in DIMENSIONS for part in (f"## Dollars by {dimension}", dollars_table(figures, dimension))],
"## Routed chunks by the rung they started on",
fixes_basis(figures),
routed_table(ladder, figures),
"## Escalations by trigger across every chunk, routed or not",
f"{triggers(figures.escalated) or 'None'}.",
"## Subagent reports against the tool results they read, in characters",
table(("Subagent type", "Subagents", "Reports", "Tool results read"), [(kind, str(reports.subagents), str(reports.report_chars), str(reports.read_chars)) for kind, reports in sorted(figures.reports.items())]),
*(["## Chunks skipped, unreadable", bullets(figures.skipped)] if figures.skipped else []),
*(["## Default branches unread", bullets(figures.unread)] if figures.unread else []),
]
return "\n\n".join(sections) + "\n"
def dollars_table(figures: Figures, dimension: str) -> str:
"""The dimension's labels and their dollars, largest first."""
return table((dimension.capitalize(), "Dollars"), [(label, money(dollars)) for label, dollars in sorted(figures.dollars[dimension].items(), key=lambda spent: -spent[1])])
def routed_table(ladder: Ladder, figures: Figures) -> str:
"""One row per rung the routed chunks started on, from the bottom of the ladder up, after the rungs off it."""
return table(("Rung", "Chunks", "Review rounds past each pass's first, mean", "Unfinished verdicts, mean", "Later fixes, mean", "Subagent rungs", "Escalations by trigger"), [
(
str(rung),
str(len(chunks)),
f"{mean(chunks, lambda chunk: chunk.rounds_past_first):.1f}",
f"{mean(chunks, lambda chunk: chunk.unfinished):.1f}",
fixes_per_chunk(chunks),
subagent_rungs(chunks),
triggers(summed(chunks, lambda chunk: chunk.escalated)),
)
for rung, chunks in sorted(figures.routed.items(), key=lambda routed: ladder.position(routed[0]))
])
def bullets(lines: typing.Iterable[str]) -> str:
return "\n".join(f"- {line}" for line in lines)
def basis(figures: Figures) -> str:
"""What the dollars are, what they leave out, and where the archive falls short of the recorded cost."""
text = "Dollars are what each worker transcript's last `cost-state` entry records for each model, cache reads included. A model's dollars are split across its messages in the archive in proportion to their cost at list prices, so the cost of a compaction, or of a subagent transcript the archive lacks, lands on the archived messages of the model that ran it. A chunk without the entry, or a model it leaves out, keeps the list prices, which may be a little off. A message's dollars count towards the tools whose results it took in, split evenly; a message that took in no tool result counts towards `prompt`. The judge's cost counts from the verdicts and rejections that record it, so a chunk archived before they did, and a judge run that failed, add none."
if figures.unattributed:
text += f" {money(figures.unattributed)} of the recorded cost is left out, since it belongs to a model with no list price or with no priced message in the archive."
if figures.lacking:
text += f" The archive holds no subagent transcript for {figures.lacking} of the chunks' Agent calls."
return text
def fixes_basis(figures: Figures) -> str:
"""What the later fixes column counts, and how many routed chunks it can't count."""
text = "A routed chunk's later fixes are the chunks on its project's default branch, as last fetched, that fixed code it shipped, counted as `score-outcomes` counts them. A recent chunk has had less time to gather any. The routed chunk is matched to the chunk on that branch that removed the item its route names from todo.md. The column gives the mean over a rung's chunks whose later fixes are known, and how many those are."
if figures.unmatched:
text += f" The later fixes of {figures.unmatched} of the routed chunks are unknown, since no chunk on the branch removed their item from todo.md, or the branch could not be read."
return text
def fixes_per_chunk(chunks: list[ChunkFigures]) -> str:
"""The mean later fixes of the chunks whose later fixes are known, and how many those are, as `1.0 (3 known)`; `unknown` where none is."""
# A mean, not a median: most chunks gather no later fix, so the median reads 0 until over half do, and can't show the rise per finished chunk that goal 1's counter-signal watches.
known = [chunk.fixed_later for chunk in chunks if chunk.fixed_later is not None]
return f"{statistics.mean(known):.1f} ({len(known)} known)" if known else "unknown"
def subagent_rungs(chunks: list[ChunkFigures]) -> str:
"""The rungs the chunks' subagents ran on, most used first, as `claude-opus-5-5 at high effort 2, …`."""
return ", ".join(f"{label} {times}" for label, times in summed(chunks, lambda chunk: chunk.delegated).most_common())
def triggers(escalated: collections.Counter[int]) -> str:
"""Escalations as `trigger 2: 3, trigger 3: 1`."""
return ", ".join(f"trigger {trigger}: {times}" for trigger, times in sorted(escalated.items()))
def money(dollars: float) -> str:
return f"${dollars:,.2f}"
def table(header: tuple[str, ...], rows: list[tuple[str, ...]]) -> str:
if not rows:
return "None."
return "\n".join(f"| {' | '.join(row)} |" for row in [header, tuple("---" for _ in header), *rows])