163 lines
8.8 KiB
Python
163 lines
8.8 KiB
Python
import re
|
|
import typing
|
|
|
|
from model_routing.prices import PRICES
|
|
from records.markdown_text import quoted
|
|
from records.utc_stamp import STAMPED
|
|
from claude_code.tokens import Tokens
|
|
from model_routing.route import WorkerLine
|
|
|
|
TASK = "Task: "
|
|
FINISHED = "Verdict: finished"
|
|
UNFINISHED = "Verdict: unfinished"
|
|
INTERRUPTED = "Verdict: interrupt"
|
|
ANSWERED = "Verdict: quoted answer"
|
|
REJECTED = "Judge reply rejected"
|
|
# The file opens with the task, then a blank line, then the first stamped outcome.
|
|
TASK_AT = re.compile(rf"\A{re.escape(TASK)}(.*?)\n\n{STAMPED} ", re.DOTALL)
|
|
ANY_VERDICT_AT = re.compile(rf"^({STAMPED}) ({'|'.join(re.escape(verdict) for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED))})$", re.MULTILINE)
|
|
FINISHED_AT = re.compile(rf"^({STAMPED}) {re.escape(FINISHED)}$", re.MULTILINE)
|
|
UNFINISHED_AT = re.compile(rf"^{STAMPED} {re.escape(UNFINISHED)}$", re.MULTILINE)
|
|
# A verdict closes each cycle the judge ruled on; an interrupt or a stop closes none.
|
|
VERDICT_AT = re.compile(rf"^{STAMPED} (?:{re.escape(FINISHED)}|{re.escape(UNFINISHED)})$", re.MULTILINE)
|
|
# An outcome opens with its UTC stamp; a verdict that closes a cycle, an interrupt, a quoted answer, a stop, or a step up ends with the worker line.
|
|
OUTCOME_AT = re.compile(rf"^({STAMPED}) ", re.MULTILINE)
|
|
NAMES_A_WORKER = (*(f"{verdict}\n" for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED)), "Stopped: ", "Stepped up\n")
|
|
WORKER = "Worker: "
|
|
# The rung that judged a verdict.
|
|
JUDGE = "Judge: "
|
|
# What the judge's run cost, in dollars, on the line after JUDGE; absent where the verdict recorded none.
|
|
JUDGE_COST = "Judge cost: "
|
|
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
|
|
|
|
|
|
# The claude-config release and commit that ran the cycle, on the line before JUDGE, such as `Config: v0.8.0 (<sha>)`.
|
|
CONFIG = "Config: "
|
|
# Opens the block of what the work since the previous outcome that recorded a spend spent; an outcome that spent nothing has none.
|
|
SPENT = "Spent since "
|
|
KINDS = "input, cache write 5m, cache write 1h, cache read, output"
|
|
# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on.
|
|
Spender = tuple[str, str]
|
|
# Opens the quote of a typed answer of the human's that reversed a choice the worker took on its own recommendation.
|
|
REVERSED = "Reversed:"
|
|
REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTILINE)
|
|
|
|
|
|
class Annotation(typing.NamedTuple):
|
|
"""What a verdict, an interrupt or a quoted answer records beside its judge lines, so a change to the setup can be judged before and after: docs/decisions.md → `loop-claude: each verdict records the claude-config release and commit that ran it…`. Each part is left out where empty; the cycle always gives config."""
|
|
config: str = ""
|
|
# The block spent_block renders.
|
|
spent: str = ""
|
|
# The human's whole typed answer that holds the judge's quote of the reversal, as loop-claude checked it.
|
|
reversed: str = ""
|
|
|
|
|
|
def annotation_lines(annotation: Annotation) -> str:
|
|
reversed_block = f"{REVERSED}\n{quoted(annotation.reversed)}\n\n" if annotation.reversed else ""
|
|
spent_block = f"{annotation.spent}\n\n" if annotation.spent else ""
|
|
config_line = f"{CONFIG}{annotation.config}\n" if annotation.config else ""
|
|
return f"{reversed_block}{spent_block}{config_line}"
|
|
|
|
|
|
# Nothing reads a Spent block yet: the first tool that derives dollars from one adds its parser here, beside this writer.
|
|
def spent_block(since: str, found: dict[Spender, Tokens]) -> str:
|
|
"""The archive's block for what the stretch spent, then the list prices in force for each model in it; empty where nothing was spent."""
|
|
if not found:
|
|
return ""
|
|
spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()]
|
|
priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))]
|
|
prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else []
|
|
return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices])
|
|
|
|
|
|
def reversals(outcomes: str) -> list[str]:
|
|
"""Each reversing answer the outcomes quote, unquoted, in the order they were written."""
|
|
return [unquoted(found[1]) for found in REVERSED_AT.finditer(outcomes)]
|
|
|
|
|
|
def unquoted(block: str) -> str:
|
|
return "\n".join(line.removeprefix("> ") for line in block.split("\n"))
|
|
|
|
|
|
def spent_stamps(outcomes: str) -> list[str]:
|
|
"""The UTC stamp of each outcome that records a spend, in the order they were written."""
|
|
return [stamp for stamp, outcome in each_outcome(outcomes) if f"\n{SPENT}" in outcome]
|
|
|
|
|
|
def judge_lines(judge: str, judge_cost: float) -> str:
|
|
"""The judge's rung, and what its run cost where that is known."""
|
|
return f"{JUDGE}{judge}" + (f"\n{JUDGE_COST}{judge_cost:.4f}" if judge_cost else "")
|
|
|
|
|
|
def judge_costs(outcomes: str) -> list[tuple[str, float]]:
|
|
"""The rung and cost of each judge run the outcomes annotation a cost for."""
|
|
return [(found[1], float(found[2])) for found in JUDGE_COST_AT.finditer(outcomes)]
|
|
|
|
|
|
def escalation_block(escalation: str) -> str:
|
|
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
|
|
|
|
|
|
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
|
|
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
|
|
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
|
|
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def answered(*, question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
|
|
"""escalation and worker are as on interrupted. Every parameter is keyword-only, since a swap of two str ones would still type-check."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0) -> str:
|
|
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
|
|
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost)}"
|
|
|
|
|
|
def stepped_up(worker: WorkerLine) -> str:
|
|
"""The rung the next cycle steps up to, archived as it steps, so a rerun after any exit resumes on it."""
|
|
return f"Stepped up\n\n{WORKER}{worker}"
|
|
|
|
|
|
def stopped(worker: WorkerLine, why: str) -> str:
|
|
return f"Stopped: {why}\n\n{WORKER}{worker}"
|
|
|
|
|
|
def workers(outcomes: str) -> list[tuple[str, str]]:
|
|
"""The UTC stamp and the worker of each outcome that names one, in the order they were written."""
|
|
found = []
|
|
for stamp, outcome in each_outcome(outcomes):
|
|
last = outcome.rsplit("\n", 1)[-1]
|
|
if outcome.startswith(NAMES_A_WORKER) and last.startswith(WORKER):
|
|
found.append((stamp, last.removeprefix(WORKER)))
|
|
return found
|
|
|
|
|
|
def each_outcome(outcomes: str) -> list[tuple[str, str]]:
|
|
"""The UTC stamp and the text after it of each outcome, in the order they were written."""
|
|
starts = list(OUTCOME_AT.finditer(outcomes))
|
|
return [(start[1], outcomes[start.end():after.start() if after else len(outcomes)].rstrip("\n")) for start, after in zip(starts, [*starts[1:], None])]
|
|
|
|
|
|
def task_of(outcomes: str) -> str | None:
|
|
"""The task an outcomes file opens with; None when it opens with none."""
|
|
found = TASK_AT.match(outcomes)
|
|
return found[1] if found else None
|
|
|
|
|
|
def first_verdict(outcomes: str) -> tuple[str, str] | None:
|
|
"""The UTC stamp and the verdict, such as FINISHED, of the first verdict in an outcomes file; None when it holds none."""
|
|
found = ANY_VERDICT_AT.search(outcomes)
|
|
return (found[1], found[2]) if found else None
|