Files
claude-config/bin/routing_figures/test_routing_figures.py
T

533 lines
36 KiB
Python

import collections
import contextlib
import dataclasses
import io
import json
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from routing_figures import archive, state
from test_env import assert_not_root, isolated_env
from project_config.templates import BARE, Titles
from branch_chunks.cut import BranchChunk
from routing_figures.calls import Call, calls, totals
from routing_figures.chunk import Chunk, chunk_figures, ChunkFigures, Reports, Subagent
from text_files.doubles import compress
from project_config.doubles import configured, write_config
from routing_figures.state import due
from claude_code.entries import entry_from
from failure import Failure
from git_cli import git_repo
from plan.todo_table import Item
from transcript_archive import outcomes, write
from model_routing.ladder_file import read_ladder, SHIPPED
from routing_figures.render import render
from text_files.file_writes import replace_file
from model_routing.route import Route
from model_routing.ladder import Rung
LADDER = read_ladder(SHIPPED)
LOW = Rung("claude-opus-5-5", "medium")
HIGH = Rung("claude-fable-5-2", "low")
LOW_ROUTE = Route(LOW, 7, 3, 4).worker_line()
def message(id_, at, *parts, model="claude-opus-5-5", usage=None):
"""Writes 100k output tokens: $2 on Opus 5.5, $1 on Sonnet 5.5 and $4 on Fable 5.2, at $20, $10 and $40 per million."""
return {"type": "assistant", "timestamp": at, "cwd": "/c", "message": {"id": id_, "model": model, "content": list(parts), "usage": usage or {"output_tokens": 100_000}}}
def cost_state(**costs):
return {"type": "cost-state", "totalCostUSD": sum(costs.values()), "modelUsage": {model.replace("_", "-"): {"costUSD": cost} for model, cost in costs.items()}}
def chunk_of(project, outcomes, worker, subagents):
return Chunk(project, outcomes, [entry_from(entry) for entry in worker], subagents)
def subagent_of(kind, launched_by, entries):
return Subagent(kind, launched_by, [entry_from(entry) for entry in entries])
def call(name, id_="t1", **arguments):
return {"type": "tool_use", "id": id_, "name": name, "input": arguments}
def result(id_, text):
return {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": id_, "content": [{"type": "text", "text": text}]}]}}
def stamped(at, outcome):
return f"{at} {outcome}\n"
def figures(start=LOW, rounds=0, unfinished=0, reports=None, escalated=None):
return ChunkFigures({}, start, rounds, unfinished, collections.Counter(), reports or {}, collections.Counter(escalated or {}), 0.0)
def shipping(*items, fixed_by=()):
"""A chunk on a default branch that shipped the items with these IDs."""
return BranchChunk("before", "tip", "abc1234", "merge", "2026-10-14T11:00:00Z", (), tuple(Item(id_, (0, 1, 0), "", f"**Item {id_}.**", {}, (), ()) for id_ in items), fixed_by)
def no_checkout(*args, **options):
return Failure("no checkout")
# Three routed chunks whose later fixes are known: 0, 0 and 3, so their median and mean differ.
MATCHED = [dataclasses.replace(figures(), fixed_later=fixed) for fixed in (0, 0, 3)]
class Measured(unittest.TestCase):
def test_prices_each_message_once_by_project_type_rung_and_the_tool_it_followed(self):
worker = [
message("m1", "2026-10-14T10:00:00.100Z", {"type": "text", "text": "reading"}),
message("m1", "2026-10-14T10:00:00.200Z", call("Read", file_path="/c/a")),
message("m2", "2026-10-14T10:05:00.000Z", {"type": "text", "text": "done"}),
message("m3", "2026-10-14T11:00:00.000Z", {"type": "text", "text": "on"}),
]
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T10:30:00Z", outcomes.stepped_up(Route(HIGH).worker_line())) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
found = chunk_figures(chunk_of("log", [kept], worker, []), "/h", BARE)
self.assertEqual(found.dollars["project"], {"log": 6.0})
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0})
self.assertEqual(found.dollars["rung"], {str(LOW): 4.0, str(HIGH): 2.0})
self.assertEqual(found.dollars["tool"], {"prompt": 4.0, "Read": 2.0})
self.assertEqual((found.start, found.unfinished), (LOW, 1))
def test_counts_the_judges_recorded_cost_under_the_judge_and_the_rung_it_ran_on(self):
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "judge-rung", "", judge_cost=0.5)) + stamped("2026-10-14T12:00:00Z", outcomes.finished(LOW_ROUTE, "m", "judge-rung", judge_cost=0.25)) + stamped("2026-10-14T12:00:00Z", outcomes.rejected("why", "{}", "other-rung", judge_cost=1.0))
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T10:00:00Z")], []), "/h", BARE)
self.assertEqual(found.dollars["project"], {"log": 3.75})
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "judge": 1.75})
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, "judge-rung": 0.75, "other-rung": 1.0})
self.assertEqual(found.dollars["tool"], {"prompt": 3.75})
def test_reads_a_subagents_rung_from_its_model_and_the_effort_its_call_set_or_else_the_workers(self):
worker = [
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet", effort="high", prompt="find"), call("Agent", "a2", subagent_type="Explore", prompt="find")),
result("a1", "x" * 30),
result("a2", "y" * 5),
]
subagents = [
subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z", model="claude-sonnet-5-5"), result("r", "z" * 20)]),
subagent_of("", "a2", [message("s2", "2026-10-14T10:00:01Z")]),
]
found = chunk_figures(chunk_of("log", [stamped("2026-10-14T11:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j"))], worker, subagents), "/h", BARE)
self.assertEqual(found.delegated, {"claude-sonnet-5-5 at high effort": 1, str(LOW): 1})
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "Explore": 3.0})
self.assertEqual(found.reports, {"Explore": Reports(2, 35, 20)})
def test_counts_the_workers_escalations_by_the_trigger_each_subagents_prompt_opens_with(self):
worker = [
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", prompt="Escalation trigger 2: the test failed twice"), call("Agent", "a2", prompt="Escalation trigger 2: again"), call("Agent", "a3", prompt="Escalation trigger 3: a 🔴 in round 2")),
message("m2", "2026-10-14T10:01:00Z", call("Agent", "a4", prompt="Find the callers. Escalation trigger 1: not at the start"), call("Agent", "a5", prompt="Escalation trigger x: no number")),
]
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).escalated, {2: 2, 3: 1})
def test_counts_review_rounds_past_each_passs_first_once_per_title(self):
commit = lambda id_, title: message(id_, "2026-10-14T10:00:00Z", call("Bash", command=f"git commit -m '{title}'"))
worker = [commit("m1", "Answer stability round 1: 1 🔴, 0 🟡"), commit("m2", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m3", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m4", "Answer prose round 1: 0 🔴, 1 🟡")]
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).rounds_past_first, 1)
def test_reads_each_round_title_through_the_projects_title_templates(self):
commit = lambda id_, title: message(id_, "2026-10-14T10:00:00Z", call("Bash", command=f"git commit -m '{title}'"))
worker = [commit("m1", "PAY-7 - Answer stability round 1: 1 🔴, 0 🟡"), commit("m2", "PAY-7 - Answer stability round 2: 0 🔴, 0 🟡")]
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", Titles("PAY-{id} - {summary}", "{summary}")).rounds_past_first, 1)
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).rounds_past_first, 0)
def fixed_later(self, history, route=LOW_ROUTE):
"""The later fixes measured counts for a chunk that route started, which LOW_ROUTE does by item 7, on a project whose default branch holds history, None where it could not be read."""
return chunk_figures(chunk_of("log", [stamped("2026-10-14T10:00:00Z", outcomes.finished(route, "m", "j"))], [], []), "/h", BARE, history).fixed_later
def test_counts_the_later_fixes_of_the_chunk_that_shipped_the_item_its_route_names(self):
self.assertEqual(self.fixed_later([shipping(6, fixed_by=("c", "d", "e")), shipping(5, 7, fixed_by=("c", "d"))]), 2)
def test_counts_no_later_fix_where_no_later_chunk_fixed_the_chunk_that_shipped_the_item(self):
self.assertEqual(self.fixed_later([shipping(7)]), 0)
def test_leaves_the_later_fixes_unknown_where_no_chunk_shipped_the_item(self):
self.assertIsNone(self.fixed_later([shipping(8, fixed_by=("c",))]))
self.assertIsNone(self.fixed_later([]))
def test_leaves_the_later_fixes_unknown_where_the_history_could_not_be_read(self):
self.assertIsNone(self.fixed_later(None))
def test_leaves_the_later_fixes_unknown_where_the_route_names_no_item(self):
self.assertIsNone(self.fixed_later([shipping(7, fixed_by=("c",))], Route(LOW).worker_line()))
def test_starts_no_route_for_a_chunk_no_item_routed(self):
kept = stamped("2026-10-14T10:00:00Z", outcomes.finished(Route(LOW).worker_line(), "m", "j"))
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T09:00:00Z")], []), "/h", BARE)
self.assertIsNone(found.start)
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0})
def test_puts_a_subagent_on_the_model_it_ran_at_the_effort_of_the_cycle_that_launched_it(self):
kept = stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T11:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "m", "j"))
worker = [
message("m1", "2026-10-14T10:00:00.900Z"),
message("m2", "2026-10-14T10:30:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet")),
message("m3", "2026-10-14T10:31:00Z", call("Agent", "a2", subagent_type="Explore")),
]
subagents = [subagent_of("Explore", "a1", [result("r", "x")]), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:31:01Z", model="claude-fable-5-2")])]
found = chunk_figures(chunk_of("log", [kept], worker, subagents), "/h", BARE)
self.assertEqual(found.delegated, {"sonnet at low effort": 1, str(HIGH): 1})
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, str(HIGH): 8.0})
def test_names_a_subagent_by_its_meta_or_else_its_call_or_else_unknown(self):
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Plan"))]
subagents = [subagent_of("stability-reviewer", "a1", []), subagent_of("", "a2", []), subagent_of("", "", [])]
self.assertEqual(chunk_figures(chunk_of("log", [], worker, subagents), "/h", BARE).reports, {"stability-reviewer": Reports(1, 0, 0), "Plan": Reports(1, 0, 0), "unknown": Reports(1, 0, 0)})
def test_counts_a_message_without_usage_or_a_priced_model_as_nothing_and_splits_a_cost_evenly_across_tools(self):
worker = [
{"type": "assistant", "uuid": "u1", "message": {"model": "claude-opus-5-5", "content": [call("Read"), call("Grep")], "usage": {"input_tokens": 1_000_000, "output_tokens": "many"}}},
{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t1", "content": "plain"}, "odd"]}},
{"type": "assistant", "uuid": "u2", "message": {"model": "claude-opus-5-5", "content": [], "usage": {"input_tokens": 2_000_000}}},
{"type": "assistant", "uuid": "u3", "message": {"model": "claude-opus-5-5", "content": []}},
{"type": "assistant", "uuid": "u4", "message": {"model": "claude-unheard-of", "content": [], "usage": {"input_tokens": 9_000_000}}},
]
found = chunk_figures(chunk_of("log", [], worker, []), "/h", BARE)
self.assertEqual(found.dollars["tool"], {"prompt": 4.0, "Read": 4.0, "Grep": 4.0})
self.assertEqual(found.dollars["project"], {"log": 12.0})
def test_prices_input_cache_writes_by_their_lifetime_cache_reads_and_output_at_the_models_list_prices(self):
usage = {"input_tokens": 1_000_000, "cache_creation_input_tokens": 2_000_000, "cache_creation": {"ephemeral_5m_input_tokens": 1_000_000, "ephemeral_1h_input_tokens": 1_000_000}, "cache_read_input_tokens": 1_000_000, "output_tokens": 1_000_000}
opus = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h", BARE)
self.assertAlmostEqual(opus.dollars["project"]["log"], 4 + 5 + 8 + 0.2 + 20)
sonnet = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", model="claude-sonnet-5-5", usage=usage)], []), "/h", BARE)
self.assertAlmostEqual(sonnet.dollars["project"]["log"], 2 + 2.5 + 4 + 0.2 + 10)
def test_counts_a_cache_write_without_a_lifetime_as_a_five_minute_one(self):
usage = {"cache_creation_input_tokens": 1_000_000}
found = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h", BARE)
self.assertAlmostEqual(found.dollars["project"]["log"], 5)
def test_splits_the_cost_a_chunks_last_cost_state_records_for_each_model_across_its_messages_by_list_price(self):
worker = [
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore")),
message("m2", "2026-10-14T10:01:00Z"),
cost_state(claude_opus_5_5=1.0),
cost_state(claude_opus_5_5=9.0, claude_haiku_5_5=0.5),
]
subagents = [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]
found = chunk_figures(chunk_of("log", [], worker, subagents), "/h", BARE)
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0, "Explore": 3.0})
self.assertEqual(found.dollars["project"], {"log": 9.0})
self.assertEqual(found.unattributed, 0.5)
def test_counts_the_subagents_the_worker_launched_that_have_no_transcript(self):
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Explore"), call("Read", "r1", file_path="/c/a"))]
found = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]), "/h", BARE)
self.assertEqual(found.lacking, 1)
unnamed = chunk_figures(chunk_of("log", [], worker, [subagent_of("", "", [message("s1", "2026-10-14T10:00:01Z")]), subagent_of("", "", [message("s2", "2026-10-14T10:00:02Z")])]), "/h", BARE)
self.assertEqual(unnamed.lacking, 0)
nested = [message("s1", "2026-10-14T10:00:01Z", call("Agent", "n1", subagent_type="Explore"))]
launched = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:00:02Z")]), subagent_of("Explore", "n1", [message("s3", "2026-10-14T10:00:03Z")])]), "/h", BARE)
self.assertEqual(launched.lacking, 0)
self.assertEqual(chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [])]), "/h", BARE).lacking, 1)
def test_charges_a_message_without_a_timestamp_to_the_rung_of_the_last_outcome(self):
undated = message("m1", "")
del undated["timestamp"]
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
self.assertEqual(chunk_figures(chunk_of("log", [kept], [undated], []), "/h", BARE).dollars["rung"], {str(HIGH): 2.0})
def test_keeps_list_prices_for_a_model_the_cost_state_does_not_name(self):
worker = [message("m1", "2026-10-14T10:00:00Z"), message("m2", "2026-10-14T10:01:00Z", model="claude-sonnet-5-5"), cost_state(claude_opus_5_5=3.0)]
found = chunk_figures(chunk_of("log", [], worker, []), "/h", BARE)
self.assertEqual(found.dollars["project"], {"log": 4.0})
self.assertEqual(found.unattributed, 0.0)
class Calls(unittest.TestCase):
def test_calls_when_chunks_on_a_rung_took_more_rounds_or_unfinished_verdicts_than_the_rung_above(self):
chunks = [figures(LOW, rounds=2)] * 5 + [figures(HIGH, unfinished=1)] * 5
self.assertEqual(calls(LADDER, totals(chunks)), [Call("rounds", (str(LOW), str(HIGH)))])
self.assertEqual(calls(LADDER, totals(chunks))[0].sentence(), f"Chunks routed onto {LOW} took more review rounds past each pass's first than chunks routed onto {HIGH}.")
[unfinished] = calls(LADDER, totals([figures(LOW, unfinished=1)] * 5 + [figures(HIGH)] * 5))
self.assertEqual(unfinished, Call("unfinished", (str(LOW), str(HIGH))))
self.assertEqual(unfinished.sentence(), f"Chunks routed onto {LOW} took more unfinished verdicts than chunks routed onto {HIGH}.")
def test_compares_the_bottom_rungs_and_calls_nothing_on_a_tie(self):
self.assertEqual(calls(LADDER, totals([figures(LADDER.rungs[0], rounds=1)] * 5 + [figures(LADDER.rungs[1])] * 5)), [Call("rounds", (str(LADDER.rungs[0]), str(LADDER.rungs[1])))])
self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=1)] * 5 + [figures(HIGH, rounds=1)] * 5)), [])
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 20)})] * 5)), [])
def test_compares_only_rungs_with_five_chunks_each(self):
self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=2)] * 5 + [figures(HIGH)] * 4)), [])
def test_calls_when_a_subagent_types_reports_outgrow_the_searches_they_ran(self):
[call] = calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 5))
self.assertEqual(call, Call("reports", ("Explore",)))
self.assertEqual(call.sentence(), "Explore subagents' reports took more of the worker's context than their searches would have.")
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 4)), [])
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 30)})] * 5)), [])
def test_renders_each_dimension_largest_first_and_the_calls(self):
chunk = ChunkFigures({"project": collections.Counter({"a": 2.5, "b": 1234.5})}, LOW, 1, 0, collections.Counter({str(HIGH): 2}), {"Explore": Reports(2, 3, 4)}, collections.Counter(), 0.0)
text = render(LADDER, totals([chunk]), [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", 40, "")
self.assertIn("Measured 2026-10-15T00:00:00Z, at cycle 40 across loop-claude's archive, from 1 finished chunk.", text)
self.assertIn("## Calls for a routing change\n\n- Explore subagents' reports took more of the worker's context than their searches would have.\n", text)
self.assertIn("## Dollars by project", text)
self.assertIn("| b | $1,234.50 |\n| a | $2.50 |", text)
self.assertIn("cache reads included", text)
self.assertIn("The judge's cost counts from the verdicts and rejections that record it, so a chunk archived before they did, and a judge run that failed, add none.", text)
self.assertNotIn("left out", text)
self.assertNotIn("no transcript", text)
self.assertIn("The archive holds no subagent transcript for 2 of the chunks' Agent calls.", render(LADDER, totals([dataclasses.replace(chunk, lacking=1)] * 2), [], "2026-10-15T00:00:00Z", 40, ""))
self.assertIn("$0.50 of the recorded cost is left out", render(LADDER, totals([ChunkFigures({}, LOW, 0, 0, collections.Counter(), {}, collections.Counter(), 0.5), chunk]), [], "2026-10-15T00:00:00Z", 40, ""))
self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |", text)
two = ChunkFigures({}, HIGH, 0, 1, collections.Counter({str(HIGH): 2, str(LOW): 1}), {}, collections.Counter(), 0.0)
self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |\n| {HIGH} | 1 | 0.0 | 1.0 | unknown | {HIGH} 2, {LOW} 1 | |", render(LADDER, totals([two, chunk]), [], "2026-10-15T00:00:00Z", 40, ""))
self.assertIn("| Explore | 2 | 3 | 4 |", text)
escalated = render(LADDER, totals([figures(escalated={2: 2}), figures(escalated={2: 1, 10: 1}), figures(start=None, escalated={3: 1})]), [], "2026-10-15T00:00:00Z", 40, "")
self.assertIn(f"| {LOW} | 2 | 0.0 | 0.0 | unknown | | trigger 2: 3, trigger 10: 1 |", escalated)
self.assertIn("## Escalations by trigger across every chunk, routed or not\n\ntrigger 2: 3, trigger 3: 1, trigger 10: 1.\n", escalated)
self.assertIn("## Escalations by trigger across every chunk, routed or not\n\nNone.\n", text)
self.assertIn("## Calls for a routing change\n\nNone.\n", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, ""))
self.assertIn("from 0 finished chunks that finished after the reset of 2026-10-14T00:00:00Z.", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, "2026-10-14T00:00:00Z"))
def test_renders_the_mean_later_fixes_of_the_matched_chunks_and_how_many_they_are(self):
text = render(LADDER, totals([*MATCHED, figures()]), [], "2026-10-15T00:00:00Z", 40, "")
self.assertIn(f"| {LOW} | 4 | 0.0 | 0.0 | 1.0 (3 known) | | |", text)
self.assertIn("## Routed chunks by the rung they started on\n\nA routed chunk's later fixes are", text)
self.assertNotIn("later fixes", text.split("## Calls for a routing change")[0])
def test_counts_the_routed_chunks_whose_later_fixes_are_unknown(self):
self.assertIn("The later fixes of 2 of the routed chunks are unknown", render(LADDER, totals([*MATCHED, figures(), figures(), figures(start=None)]), [], "2026-10-15T00:00:00Z", 40, ""))
self.assertNotIn("are unknown", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, ""))
def test_names_each_default_branch_unread(self):
self.assertIn("## Default branches unread\n\n- log: no main branch\n", render(LADDER, totals(MATCHED, unread=("log: no main branch",)), [], "2026-10-15T00:00:00Z", 40, ""))
self.assertNotIn("branches unread", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, ""))
def test_measures_once_the_count_runs_20_past_the_last_measurement(self):
self.assertEqual([due(cycle, 20) for cycle in (39, 40, 41)], [False, True, True])
self.assertTrue(due(20, 0))
class BranchHistory(unittest.TestCase):
def test_counts_no_chunk_of_a_project_whose_plan_is_outside_the_checkout(self):
checkout = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())).resolve()
folder = configured(self, checkout)
config = folder / "config.toml"
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
found = archive.branch_history(str(checkout), folder.parents[1], BARE, git_repo.git)
assert isinstance(found, Failure), found
self.assertEqual(found.reason, f"its chunks' items and later fixes are not counted: the project's config puts {folder}/todo.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
class Archive(unittest.TestCase):
def setUp(self):
self.root = pathlib.Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.root)
def chunk(self, key, name, kept, worker, subagent=None):
folder = self.root / key / name
(folder / "session" / "s" / "subagents").mkdir(parents=True)
(folder / "copy.outcomes.md").write_text(f"Task: t\n\n{kept}")
(folder / "copy.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in worker))
if subagent:
meta, entries = subagent
(folder / "session" / "s" / "subagents" / "agent-a.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in entries))
(folder / "session" / "s" / "subagents" / "agent-a.meta.json").write_text(json.dumps(meta))
def test_counts_the_verdicts_across_every_projects_chunks(self):
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.unfinished("w", "e", "j", "")) + stamped("2026-10-14T10:01:00Z", outcomes.stopped("w", "Ctrl-C")) + stamped("2026-10-14T10:02:00Z", outcomes.finished("w", "e", "j")), [])
self.chunk("log", "c2", stamped("2026-10-14T10:00:00Z", outcomes.interrupted("q", "e", "j", "", "w")), [])
self.assertEqual(archive.cycles(self.root), 2)
self.assertEqual(archive.cycles(self.root / "missing"), 0)
def test_measures_each_finished_chunk_keyed_by_its_project(self):
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z", call("Agent", "a1", subagent_type="Explore"))], ({"agentType": "Explore", "toolUseId": "a1"}, [message("s1", "2026-10-14T09:00:01Z")]))
self.chunk("log", "c2", stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, "/h", "", no_checkout)
self.assertEqual([chunk.dollars["project"] for chunk in found.chunks], [{"home/x": 4.0}])
self.assertEqual(found.chunks[0].dollars["subagent type"], {"worker": 2.0, "Explore": 2.0})
self.assertEqual(archive.measure_archive(self.root, "/h", "2026-10-14T10:00:00Z", no_checkout), archive.ArchiveChunks([], ()))
def test_measures_a_chunk_whose_transcripts_are_compressed(self):
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z", call("Agent", "a1", subagent_type="Explore"))], ({"agentType": "Explore", "toolUseId": "a1"}, [message("s1", "2026-10-14T09:00:01Z")]))
for copy in self.root.rglob("*.jsonl"):
self.assertIsNone(compress(copy))
found = archive.measure_archive(self.root, "/h", "", no_checkout)
self.assertEqual(found.chunks[0].dollars["subagent type"], {"worker": 2.0, "Explore": 2.0})
def test_counts_each_chunks_later_fixes_from_its_projects_checkout(self):
home = self.root / "home"
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
self.ship_then_fix(home / "code" / "log", env)
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
self.assertEqual([(chunk.dollars["project"].keys(), chunk.fixed_later) for chunk in found.chunks], [({"log": 0}.keys(), 1)])
def test_reads_a_projects_round_titles_and_its_branchs_history_through_its_configs_templates(self):
home = self.root / "home"
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
round_title = "PAY-8 - Answer stability round {}: 1 🔴, 0 🟡 (1 misread); fix"
self.ship_then_fix(home / "code" / "log", env, round_title.format(1), ships_fix=False)
config = write_config(home, home / "code" / "log") / "config.toml"
worker = [message(f"m{n}", "2026-10-14T09:00:00Z", call("Bash", command=f"git commit -m '{round_title.format(n)}'")) for n in (1, 2)]
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), worker)
bare = archive.measure_archive(self.root, str(home), "", git_repo.git)
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\n')
templated = archive.measure_archive(self.root, str(home), "", git_repo.git)
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in bare.chunks], [(0, 0)])
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in templated.chunks], [(1, 1)])
def test_measures_no_chunk_of_a_project_while_a_config_cannot_be_read_and_names_it(self):
home = self.root / "home"
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
self.ship_then_fix(home / "code" / "log", env)
config = write_config(home, home / "code" / "log") / "config.toml"
config.write_text(config.read_text() + 'commit-title = "PAY-{id}"\n')
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
self.assertEqual(found.chunks, [])
self.assertEqual(len(found.unread), 1)
self.assertRegex(found.unread[0], r"^log: none of its chunks is measured: .*`commit-title` is 'PAY-\{id\}'")
def test_reads_each_projects_history_guarded_and_each_read_of_its_branch_time_limited(self):
home = self.root / "home"
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
self.ship_then_fix(home / "code" / "log", env)
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
made = []
def recording(*args, **options):
made.append((args[0], options))
return git_repo.git(*args, **options)
archive.measure_archive(self.root, str(home), "", recording)
self.assertTrue(all(options.get("guarded") is True for _, options in made))
read = [options.get("timeout") for command, options in made if command in ("log", "ls-tree", "show", "diff", "blame")]
self.assertEqual(set(read), {archive.GIT_TIMEOUT})
self.assertIn("blame", [command for command, _ in made])
def test_names_a_project_whose_checkout_has_no_default_branch(self):
home = self.root / "home"
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
(home / "code" / "log").mkdir(parents=True)
subprocess.run(["git", "init", "-q"], cwd=home / "code" / "log", check=True, env=env, capture_output=True)
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
self.assertEqual(found.unread, ("log: no origin remote, and no main or master branch to take as the default branch",))
def test_names_each_project_whose_history_it_cannot_read(self):
self.chunk("gone", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, "/h", "", no_checkout)
self.assertEqual(([chunk.fixed_later for chunk in found.chunks], found.unread), ([None], ("gone: no checkout",)))
def ship_then_fix(self, checkout, env, fix_title="Fix six", ships_fix=True):
"""Makes a checkout whose main branch merges the chunk that ships item 7 of LOW_ROUTE, then one titled fix_title that rewrites a line item 7 shipped, and ships a defect item where ships_fix."""
checkout.mkdir(parents=True)
git = lambda *args: subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=checkout, check=True, env=env, capture_output=True)
plan = lambda rows: "# todo\n\n## Items\n\n| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n" + rows
def merge(branch, title, files):
git("switch", "-q", "-c", branch)
for name, text in files.items():
(checkout / name).write_text(text)
git("add", *files)
git("commit", "-q", "-m", title)
git("switch", "-q", "main")
git("merge", "-q", "--no-ff", "-m", f"Merge {branch}", branch)
fix = "| 6 | 0.1.0 | | defect | **Fix six.** | 1 | 1 | 1 | 1 | 2 | 1.1 by 3 | 1.0 |\n"
git("init", "-q", "-b", "main")
(checkout / "todo.md").write_text(plan("| 7 | 0.1.0 | | | **Ship seven.** | 1 | 1 | 1 | 1 | 2 | 1.1 by 3 | 1.0 |\n" + fix))
(checkout / "a.py").write_text("1\n")
git("add", "todo.md", "a.py")
git("commit", "-q", "-m", "Start")
merge("ship", "Ship seven", {"a.py": "1\nx\n", "todo.md": plan(fix)})
merge("fix", fix_title, {"a.py": "1\ny\n", "todo.md": plan("" if ships_fix else fix)})
def test_reads_each_cycles_route_from_its_own_outcomes_file(self):
transcript = self.root / "t.jsonl"
kept = self.root / "archive"
first = message("m1", "2026-09-21T00:00:00Z")
transcript.write_text(json.dumps(first) + "\n")
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.unfinished(LOW_ROUTE, "e", "j", ""), 1790000000, replace=replace_file))
transcript.write_text(json.dumps(first) + "\n" + json.dumps(message("m2", "2099-01-01T00:00:00Z")) + "\n")
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.finished(Route(HIGH).worker_line(), "m", "j"), 4070908800, replace=replace_file))
(found,) = archive.measure_archive(kept, "/h", "", no_checkout).chunks
self.assertEqual((found.start, found.unfinished), (LOW, 1))
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, str(HIGH): 2.0})
def test_skips_a_chunk_it_cannot_read_and_names_it(self):
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [])
(self.root / "log" / "c1" / "copy.jsonl").unlink()
self.chunk("other", "c2", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
self.chunk("broken", "c3", "", [])
(self.root / "broken" / "c3" / "copy.outcomes.md").write_bytes(b"\xff")
found = archive.measure_archive(self.root, "/h", "", no_checkout)
self.assertEqual((len(found.chunks), found.skipped), (1, ("broken", "log/c1")))
self.assertIn("## Chunks skipped, unreadable\n\n- log/c1\n", render(LADDER, totals([], ("log/c1",)), [], "2026-10-15T00:00:00Z", 20, ""))
def test_keeps_the_cycle_it_measured_at_and_reads_it_back(self):
self.assertEqual(state.read_state(self.root), state.State(0, ()))
self.assertIsNone(state.mark_measured(self.root, 40))
self.assertEqual(state.read_state(self.root), state.State(40, ()))
self.assertIsNone(archive.write_figures(self.root, "# Routing figures\n"))
self.assertEqual(state.read_state(self.root).measured_at, 40, "the rendered figures are for the human, and never read back")
def test_reads_the_calls_raised_since_the_latest_reset(self):
a, b, c = Call("reports", ("Explore",)), Call("rounds", ("x", "y")), Call("unfinished", ("x", "y"))
self.assertEqual(state.raised(self.root), state.Raised("", frozenset()))
self.assertIsNone(state.mark_raised(self.root, [a], "2026-10-15T00:00:00Z", replace_file))
self.assertIsNone(state.mark_raised(self.root, [b, c], "2026-10-15T01:00:00Z", replace_file))
self.assertEqual((self.root / state.RAISED).read_text(), f"# Routing calls raised\n\n- 2026-10-15T00:00:00Z {a.sentence()}\n- 2026-10-15T01:00:00Z {b.sentence()}\n- 2026-10-15T01:00:00Z {c.sentence()}\n")
self.assertEqual(state.raised(self.root), state.Raised("", frozenset({a, b, c})))
with open(self.root / state.RAISED, "a") as handle:
handle.write("- 2026-10-15T01:00:00Z Reset\n")
self.assertEqual(state.raised(self.root), state.Raised("2026-10-15T01:00:00Z", frozenset()), "a call stamped at the reset's second was raised before it")
self.assertIsNone(state.mark_raised(self.root, [a], "2026-10-15T02:00:00Z", replace_file))
self.assertEqual(state.raised(self.root), state.Raised("2026-10-15T01:00:00Z", frozenset({a})))
def test_fails_when_it_cannot_keep_the_state(self):
assert_not_root(self)
self.root.chmod(0o555)
self.addCleanup(self.root.chmod, 0o755)
self.assertIsInstance(state.mark_measured(self.root, 40), Failure)
self.assertIsInstance(state.mark_raised(self.root, [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", replace_file), Failure)
def test_fails_when_it_cannot_list_a_raised_call_for_the_human(self):
self.assertIsNone(state.mark_measured(self.root, 40))
self.assertEqual(state.mark_raised(self.root, [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", replace=lambda path, data: Failure("full")), Failure(f"could not write {self.root / state.RAISED}: full"))
def test_knows_a_raised_call_by_its_fields_whatever_its_line_reads(self):
call = Call("reports", ("Explore",))
self.assertIsNone(state.mark_raised(self.root, [call], "2026-10-15T00:00:00Z", replace_file))
(self.root / state.RAISED).write_text("# Routing calls raised\n\n- 2026-10-15T00:00:00Z Reworded.\n")
self.assertEqual(state.raised(self.root).calls, frozenset({call}))
def test_fails_on_a_state_it_cannot_read(self):
for text in ("not json", "[]", '{"measured_at": "40", "raised": []}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "nope", "subjects": []}]}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "reports", "subjects": ["a", "b"]}]}', '{"measured_at": true, "raised": []}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "reports", "subjects": "a"}]}'):
with self.subTest(text):
(self.root / state.STATE).write_text(text)
self.assertIsInstance(state.read_state(self.root), Failure)
self.assertIsInstance(state.raised(self.root), Failure)
(self.root / state.STATE).unlink()
(self.root / state.STATE).mkdir()
self.assertIsInstance(state.read_state(self.root), Failure)
def test_holds_the_lock_while_no_other_loop_does(self):
with archive.locked(self.root) as held, archive.locked(self.root) as other:
self.assertEqual((held, other), (True, False))
with archive.locked(self.root) as again:
self.assertTrue(again)
def test_fails_on_an_archive_it_cannot_read(self):
(self.root / state.RAISED).mkdir()
self.assertIsInstance(state.raised(self.root), Failure)
if __name__ == "__main__":
unittest.main()