Files
claude-config/bin/transcript_archive/outcomes.py
T

163 lines
8.8 KiB
Python

import re
import typing
from model_routing.prices import PRICES
from text_files.markdown_text import quoted
from text_files.utc_stamp import STAMPED
from claude_code.tokens import Tokens
from model_routing.route import WorkerLine
TASK = "Task: "
FINISHED = "Verdict: finished"
UNFINISHED = "Verdict: unfinished"
INTERRUPTED = "Verdict: interrupt"
ANSWERED = "Verdict: quoted answer"
REJECTED = "Judge reply rejected"
# The file opens with the task, then a blank line, then the first stamped outcome.
TASK_AT = re.compile(rf"\A{re.escape(TASK)}(.*?)\n\n{STAMPED} ", re.DOTALL)
ANY_VERDICT_AT = re.compile(rf"^({STAMPED}) ({'|'.join(re.escape(verdict) for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED))})$", re.MULTILINE)
FINISHED_AT = re.compile(rf"^({STAMPED}) {re.escape(FINISHED)}$", re.MULTILINE)
UNFINISHED_AT = re.compile(rf"^{STAMPED} {re.escape(UNFINISHED)}$", re.MULTILINE)
# A verdict closes each cycle the judge ruled on; an interrupt or a stop closes none.
VERDICT_AT = re.compile(rf"^{STAMPED} (?:{re.escape(FINISHED)}|{re.escape(UNFINISHED)})$", re.MULTILINE)
# An outcome opens with its UTC stamp; a verdict that closes a cycle, an interrupt, a quoted answer, a stop, or a step up ends with the worker line.
OUTCOME_AT = re.compile(rf"^({STAMPED}) ", re.MULTILINE)
NAMES_A_WORKER = (*(f"{verdict}\n" for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED)), "Stopped: ", "Stepped up\n")
WORKER = "Worker: "
# The rung that judged a verdict.
JUDGE = "Judge: "
# What the judge's run cost, in dollars, on the line after JUDGE; absent where the verdict recorded none.
JUDGE_COST = "Judge cost: "
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
# The claude-config release and commit that ran the cycle, on the line before JUDGE, such as `Config: v0.8.0 (<sha>)`.
CONFIG = "Config: "
# Opens the block of what the work since the previous outcome that recorded a spend spent; an outcome that didn't spend anything has none.
SPENT = "Spent since "
KINDS = "input, cache write 5m, cache write 1h, cache read, output"
# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on.
Spender = tuple[str, str]
# Opens the quote of a typed answer of the human's that reversed a choice the worker took on its own recommendation.
REVERSED = "Reversed:"
REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTILINE)
class Annotation(typing.NamedTuple):
"""What a verdict, an interrupt or a quoted answer records beside its judge lines, so a change to the setup can be judged before and after: docs/decisions.md → `loop-claude: each verdict records the claude-config release and commit that ran it…`. Each part is left out where empty; the cycle always gives config."""
config: str = ""
# The block spent_block renders.
spent: str = ""
# The human's whole typed answer that holds the judge's quote of the reversal, as loop-claude checked it.
reversed: str = ""
def annotation_lines(annotation: Annotation) -> str:
reversed_block = f"{REVERSED}\n{quoted(annotation.reversed)}\n\n" if annotation.reversed else ""
spent_block = f"{annotation.spent}\n\n" if annotation.spent else ""
config_line = f"{CONFIG}{annotation.config}\n" if annotation.config else ""
return f"{reversed_block}{spent_block}{config_line}"
# Nothing reads a Spent block yet: the first tool that derives dollars from one adds its parser here, beside this writer.
def spent_block(since: str, found: dict[Spender, Tokens]) -> str:
"""The archive's block for what the stretch spent, then the list prices in force for each model in it that has one; empty where nothing was spent."""
if not found:
return ""
spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()]
priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))]
prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else []
return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices])
def reversals(outcomes: str) -> list[str]:
"""Each reversing answer the outcomes quote, unquoted, in the order they were written."""
return [unquoted(found[1]) for found in REVERSED_AT.finditer(outcomes)]
def unquoted(block: str) -> str:
return "\n".join(line.removeprefix("> ") for line in block.split("\n"))
def spent_stamps(outcomes: str) -> list[str]:
"""The UTC stamp of each outcome that records a spend, in the order they were written."""
return [stamp for stamp, outcome in each_outcome(outcomes) if f"\n{SPENT}" in outcome]
def judge_lines(judge: str, judge_cost: float) -> str:
"""The judge's rung, and what its run cost where that is known."""
return f"{JUDGE}{judge}" + (f"\n{JUDGE_COST}{judge_cost:.4f}" if judge_cost else "")
def judge_costs(outcomes: str) -> list[tuple[str, float]]:
"""The rung and cost of each judge run the outcomes annotation a cost for."""
return [(found[1], float(found[2])) for found in JUDGE_COST_AT.finditer(outcomes)]
def escalation_block(escalation: str) -> str:
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
escalated = escalation_block(escalation)
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
escalated = escalation_block(escalation)
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
def answered(*, question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
"""escalation and worker are as on interrupted. Every parameter is keyword-only, since a swap of two str ones would still type-check."""
escalated = escalation_block(escalation)
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0) -> str:
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost)}"
def stepped_up(worker: WorkerLine) -> str:
"""The rung the next cycle steps up to, archived as it steps, so a rerun after any exit resumes on it."""
return f"Stepped up\n\n{WORKER}{worker}"
def stopped(worker: WorkerLine, why: str) -> str:
return f"Stopped: {why}\n\n{WORKER}{worker}"
def workers(outcomes: str) -> list[tuple[str, str]]:
"""The UTC stamp and the worker of each outcome that names one, in the order they were written."""
found = []
for stamp, outcome in each_outcome(outcomes):
last = outcome.rsplit("\n", 1)[-1]
if outcome.startswith(NAMES_A_WORKER) and last.startswith(WORKER):
found.append((stamp, last.removeprefix(WORKER)))
return found
def each_outcome(outcomes: str) -> list[tuple[str, str]]:
"""The UTC stamp and the text after it of each outcome, in the order they were written."""
starts = list(OUTCOME_AT.finditer(outcomes))
return [(start[1], outcomes[start.end():after.start() if after else len(outcomes)].rstrip("\n")) for start, after in zip(starts, [*starts[1:], None])]
def task_of(outcomes: str) -> str | None:
"""The task an outcomes file opens with; None when it opens with none."""
found = TASK_AT.match(outcomes)
return found[1] if found else None
def first_verdict(outcomes: str) -> tuple[str, str] | None:
"""The UTC stamp and the verdict, such as FINISHED, of the first verdict in an outcomes file; None when it holds none."""
found = ANY_VERDICT_AT.search(outcomes)
return (found[1], found[2]) if found else None