103 lines
5.2 KiB
Python
103 lines
5.2 KiB
Python
import re
|
|
|
|
from records.markdown_text import quoted
|
|
from records.utc_stamp import STAMPED
|
|
from model_routing.route import WorkerLine
|
|
|
|
TASK = "Task: "
|
|
FINISHED = "Verdict: finished"
|
|
UNFINISHED = "Verdict: unfinished"
|
|
INTERRUPTED = "Verdict: interrupt"
|
|
ANSWERED = "Verdict: quoted answer"
|
|
REJECTED = "Judge reply rejected"
|
|
# The file opens with the task, then a blank line, then the first stamped outcome.
|
|
TASK_AT = re.compile(rf"\A{re.escape(TASK)}(.*?)\n\n{STAMPED} ", re.DOTALL)
|
|
ANY_VERDICT_AT = re.compile(rf"^({STAMPED}) ({'|'.join(re.escape(verdict) for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED))})$", re.MULTILINE)
|
|
FINISHED_AT = re.compile(rf"^({STAMPED}) {re.escape(FINISHED)}$", re.MULTILINE)
|
|
UNFINISHED_AT = re.compile(rf"^{STAMPED} {re.escape(UNFINISHED)}$", re.MULTILINE)
|
|
# A verdict closes each cycle the judge ruled on; an interrupt or a stop closes none.
|
|
VERDICT_AT = re.compile(rf"^{STAMPED} (?:{re.escape(FINISHED)}|{re.escape(UNFINISHED)})$", re.MULTILINE)
|
|
# An outcome opens with its UTC stamp; a verdict that closes a cycle, an interrupt, a quoted answer, a stop, or a step up ends with the worker line.
|
|
OUTCOME_AT = re.compile(rf"^({STAMPED}) ", re.MULTILINE)
|
|
NAMES_A_WORKER = (*(f"{verdict}\n" for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED)), "Stopped: ", "Stepped up\n")
|
|
WORKER = "Worker: "
|
|
# The rung that judged a verdict.
|
|
JUDGE = "Judge: "
|
|
# What the judge's run cost, in dollars, on the line after JUDGE; absent where the verdict recorded none.
|
|
JUDGE_COST = "Judge cost: "
|
|
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
|
|
|
|
|
|
def judge_lines(judge: str, judge_cost: float) -> str:
|
|
"""The judge's rung, and what its run cost where that is known."""
|
|
return f"{JUDGE}{judge}" + (f"\n{JUDGE_COST}{judge_cost:.4f}" if judge_cost else "")
|
|
|
|
|
|
def judge_costs(outcomes: str) -> list[tuple[str, float]]:
|
|
"""The rung and cost of each judge run the outcomes record a cost for."""
|
|
return [(found[1], float(found[2])) for found in JUDGE_COST_AT.finditer(outcomes)]
|
|
|
|
|
|
def escalation_block(escalation: str) -> str:
|
|
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
|
|
|
|
|
|
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0) -> str:
|
|
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0) -> str:
|
|
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0) -> str:
|
|
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def answered(*, question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine, judge_cost: float = 0.0) -> str:
|
|
"""escalation and worker are as on interrupted. Every parameter is keyword-only, since a swap of two str ones would still type-check."""
|
|
escalated = escalation_block(escalation)
|
|
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
|
|
|
|
|
|
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0) -> str:
|
|
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
|
|
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost)}"
|
|
|
|
|
|
def stepped_up(worker: WorkerLine) -> str:
|
|
"""The rung the next cycle steps up to, archived as it steps, so a rerun after any exit resumes on it."""
|
|
return f"Stepped up\n\n{WORKER}{worker}"
|
|
|
|
|
|
def stopped(worker: WorkerLine, why: str) -> str:
|
|
return f"Stopped: {why}\n\n{WORKER}{worker}"
|
|
|
|
|
|
def workers(outcomes: str) -> list[tuple[str, str]]:
|
|
"""The UTC stamp and the worker of each outcome that names one, in the order they were written."""
|
|
starts = list(OUTCOME_AT.finditer(outcomes))
|
|
found = []
|
|
for start, after in zip(starts, [*starts[1:], None]):
|
|
outcome = outcomes[start.end():after.start() if after else len(outcomes)].rstrip("\n")
|
|
last = outcome.rsplit("\n", 1)[-1]
|
|
if outcome.startswith(NAMES_A_WORKER) and last.startswith(WORKER):
|
|
found.append((start[1], last.removeprefix(WORKER)))
|
|
return found
|
|
|
|
|
|
def task_of(outcomes: str) -> str | None:
|
|
"""The task an outcomes file opens with; None when it opens with none."""
|
|
found = TASK_AT.match(outcomes)
|
|
return found[1] if found else None
|
|
|
|
|
|
def first_verdict(outcomes: str) -> tuple[str, str] | None:
|
|
"""The UTC stamp and the verdict, such as FINISHED, of the first verdict in an outcomes file; None when it holds none."""
|
|
found = ANY_VERDICT_AT.search(outcomes)
|
|
return (found[1], found[2]) if found else None
|