533 lines
36 KiB
Python
533 lines
36 KiB
Python
import collections
|
|
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from routing_figures import archive, state
|
|
from test_env import assert_not_root, isolated_env
|
|
from project_config.templates import BARE, Titles
|
|
from branch_chunks.cut import BranchChunk
|
|
from routing_figures.calls import Call, calls, totals
|
|
from routing_figures.chunk import Chunk, chunk_figures, ChunkFigures, Reports, Subagent
|
|
from text_files.doubles import compress
|
|
from project_config.doubles import configured, write_config
|
|
from routing_figures.state import due
|
|
from claude_code.entries import entry_from
|
|
from failure import Failure
|
|
from git_cli import git_repo
|
|
from plan.todo_table import Item
|
|
from transcript_archive import outcomes, write
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from routing_figures.render import render
|
|
from text_files.file_writes import replace_file
|
|
from model_routing.route import Route
|
|
from model_routing.ladder import Rung
|
|
|
|
LADDER = read_ladder(SHIPPED)
|
|
LOW = Rung("claude-opus-5-5", "medium")
|
|
HIGH = Rung("claude-fable-5-2", "low")
|
|
LOW_ROUTE = Route(LOW, 7, 3, 4).worker_line()
|
|
|
|
|
|
def message(id_, at, *parts, model="claude-opus-5-5", usage=None):
|
|
"""Writes 100k output tokens: $2 on Opus 5.5, $1 on Sonnet 5.5 and $4 on Fable 5.2, at $20, $10 and $40 per million."""
|
|
return {"type": "assistant", "timestamp": at, "cwd": "/c", "message": {"id": id_, "model": model, "content": list(parts), "usage": usage or {"output_tokens": 100_000}}}
|
|
|
|
|
|
def cost_state(**costs):
|
|
return {"type": "cost-state", "totalCostUSD": sum(costs.values()), "modelUsage": {model.replace("_", "-"): {"costUSD": cost} for model, cost in costs.items()}}
|
|
|
|
|
|
def chunk_of(project, outcomes, worker, subagents):
|
|
return Chunk(project, outcomes, [entry_from(entry) for entry in worker], subagents)
|
|
|
|
|
|
def subagent_of(kind, launched_by, entries):
|
|
return Subagent(kind, launched_by, [entry_from(entry) for entry in entries])
|
|
|
|
|
|
def call(name, id_="t1", **arguments):
|
|
return {"type": "tool_use", "id": id_, "name": name, "input": arguments}
|
|
|
|
|
|
def result(id_, text):
|
|
return {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": id_, "content": [{"type": "text", "text": text}]}]}}
|
|
|
|
|
|
def stamped(at, outcome):
|
|
return f"{at} {outcome}\n"
|
|
|
|
|
|
def figures(start=LOW, rounds=0, unfinished=0, reports=None, escalated=None):
|
|
return ChunkFigures({}, start, rounds, unfinished, collections.Counter(), reports or {}, collections.Counter(escalated or {}), 0.0)
|
|
|
|
|
|
def shipping(*items, fixed_by=()):
|
|
"""A chunk on a default branch that shipped the items with these IDs."""
|
|
return BranchChunk("before", "tip", "abc1234", "merge", "2026-10-14T11:00:00Z", (), tuple(Item(id_, (0, 1, 0), "", f"**Item {id_}.**", {}, (), ()) for id_ in items), fixed_by)
|
|
|
|
|
|
def no_checkout(*args, **options):
|
|
return Failure("no checkout")
|
|
|
|
|
|
# Three routed chunks whose later fixes are known: 0, 0 and 3, so their median and mean differ.
|
|
MATCHED = [dataclasses.replace(figures(), fixed_later=fixed) for fixed in (0, 0, 3)]
|
|
|
|
|
|
class Measured(unittest.TestCase):
|
|
def test_prices_each_message_once_by_project_type_rung_and_the_tool_it_followed(self):
|
|
worker = [
|
|
message("m1", "2026-10-14T10:00:00.100Z", {"type": "text", "text": "reading"}),
|
|
message("m1", "2026-10-14T10:00:00.200Z", call("Read", file_path="/c/a")),
|
|
message("m2", "2026-10-14T10:05:00.000Z", {"type": "text", "text": "done"}),
|
|
message("m3", "2026-10-14T11:00:00.000Z", {"type": "text", "text": "on"}),
|
|
]
|
|
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T10:30:00Z", outcomes.stepped_up(Route(HIGH).worker_line())) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
|
|
found = chunk_figures(chunk_of("log", [kept], worker, []), "/h", BARE)
|
|
self.assertEqual(found.dollars["project"], {"log": 6.0})
|
|
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0})
|
|
self.assertEqual(found.dollars["rung"], {str(LOW): 4.0, str(HIGH): 2.0})
|
|
self.assertEqual(found.dollars["tool"], {"prompt": 4.0, "Read": 2.0})
|
|
self.assertEqual((found.start, found.unfinished), (LOW, 1))
|
|
|
|
def test_counts_the_judges_recorded_cost_under_the_judge_and_the_rung_it_ran_on(self):
|
|
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "judge-rung", "", judge_cost=0.5)) + stamped("2026-10-14T12:00:00Z", outcomes.finished(LOW_ROUTE, "m", "judge-rung", judge_cost=0.25)) + stamped("2026-10-14T12:00:00Z", outcomes.rejected("why", "{}", "other-rung", judge_cost=1.0))
|
|
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T10:00:00Z")], []), "/h", BARE)
|
|
self.assertEqual(found.dollars["project"], {"log": 3.75})
|
|
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "judge": 1.75})
|
|
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, "judge-rung": 0.75, "other-rung": 1.0})
|
|
self.assertEqual(found.dollars["tool"], {"prompt": 3.75})
|
|
|
|
def test_reads_a_subagents_rung_from_its_model_and_the_effort_its_call_set_or_else_the_workers(self):
|
|
worker = [
|
|
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet", effort="high", prompt="find"), call("Agent", "a2", subagent_type="Explore", prompt="find")),
|
|
result("a1", "x" * 30),
|
|
result("a2", "y" * 5),
|
|
]
|
|
subagents = [
|
|
subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z", model="claude-sonnet-5-5"), result("r", "z" * 20)]),
|
|
subagent_of("", "a2", [message("s2", "2026-10-14T10:00:01Z")]),
|
|
]
|
|
found = chunk_figures(chunk_of("log", [stamped("2026-10-14T11:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j"))], worker, subagents), "/h", BARE)
|
|
self.assertEqual(found.delegated, {"claude-sonnet-5-5 at high effort": 1, str(LOW): 1})
|
|
self.assertEqual(found.dollars["subagent type"], {"worker": 2.0, "Explore": 3.0})
|
|
self.assertEqual(found.reports, {"Explore": Reports(2, 35, 20)})
|
|
|
|
def test_counts_the_workers_escalations_by_the_trigger_each_subagents_prompt_opens_with(self):
|
|
worker = [
|
|
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", prompt="Escalation trigger 2: the test failed twice"), call("Agent", "a2", prompt="Escalation trigger 2: again"), call("Agent", "a3", prompt="Escalation trigger 3: a 🔴 in round 2")),
|
|
message("m2", "2026-10-14T10:01:00Z", call("Agent", "a4", prompt="Find the callers. Escalation trigger 1: not at the start"), call("Agent", "a5", prompt="Escalation trigger x: no number")),
|
|
]
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).escalated, {2: 2, 3: 1})
|
|
|
|
def test_counts_review_rounds_past_each_passs_first_once_per_title(self):
|
|
commit = lambda id_, title: message(id_, "2026-10-14T10:00:00Z", call("Bash", command=f"git commit -m '{title}'"))
|
|
worker = [commit("m1", "Answer stability round 1: 1 🔴, 0 🟡"), commit("m2", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m3", "Answer stability round 2: 0 🔴, 0 🟡"), commit("m4", "Answer prose round 1: 0 🔴, 1 🟡")]
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).rounds_past_first, 1)
|
|
|
|
def test_reads_each_round_title_through_the_projects_title_templates(self):
|
|
commit = lambda id_, title: message(id_, "2026-10-14T10:00:00Z", call("Bash", command=f"git commit -m '{title}'"))
|
|
worker = [commit("m1", "PAY-7 - Answer stability round 1: 1 🔴, 0 🟡"), commit("m2", "PAY-7 - Answer stability round 2: 0 🔴, 0 🟡")]
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", Titles("PAY-{id} - {summary}", "{summary}")).rounds_past_first, 1)
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, []), "/h", BARE).rounds_past_first, 0)
|
|
|
|
def fixed_later(self, history, route=LOW_ROUTE):
|
|
"""The later fixes measured counts for a chunk that route started, which LOW_ROUTE does by item 7, on a project whose default branch holds history, None where it could not be read."""
|
|
return chunk_figures(chunk_of("log", [stamped("2026-10-14T10:00:00Z", outcomes.finished(route, "m", "j"))], [], []), "/h", BARE, history).fixed_later
|
|
|
|
def test_counts_the_later_fixes_of_the_chunk_that_shipped_the_item_its_route_names(self):
|
|
self.assertEqual(self.fixed_later([shipping(6, fixed_by=("c", "d", "e")), shipping(5, 7, fixed_by=("c", "d"))]), 2)
|
|
|
|
def test_counts_no_later_fix_where_no_later_chunk_fixed_the_chunk_that_shipped_the_item(self):
|
|
self.assertEqual(self.fixed_later([shipping(7)]), 0)
|
|
|
|
def test_leaves_the_later_fixes_unknown_where_no_chunk_shipped_the_item(self):
|
|
self.assertIsNone(self.fixed_later([shipping(8, fixed_by=("c",))]))
|
|
self.assertIsNone(self.fixed_later([]))
|
|
|
|
def test_leaves_the_later_fixes_unknown_where_the_history_could_not_be_read(self):
|
|
self.assertIsNone(self.fixed_later(None))
|
|
|
|
def test_leaves_the_later_fixes_unknown_where_the_route_names_no_item(self):
|
|
self.assertIsNone(self.fixed_later([shipping(7, fixed_by=("c",))], Route(LOW).worker_line()))
|
|
|
|
def test_starts_no_route_for_a_chunk_no_item_routed(self):
|
|
kept = stamped("2026-10-14T10:00:00Z", outcomes.finished(Route(LOW).worker_line(), "m", "j"))
|
|
found = chunk_figures(chunk_of("log", [kept], [message("m1", "2026-10-14T09:00:00Z")], []), "/h", BARE)
|
|
self.assertIsNone(found.start)
|
|
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0})
|
|
|
|
def test_puts_a_subagent_on_the_model_it_ran_at_the_effort_of_the_cycle_that_launched_it(self):
|
|
kept = stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T11:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "m", "j"))
|
|
worker = [
|
|
message("m1", "2026-10-14T10:00:00.900Z"),
|
|
message("m2", "2026-10-14T10:30:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet")),
|
|
message("m3", "2026-10-14T10:31:00Z", call("Agent", "a2", subagent_type="Explore")),
|
|
]
|
|
subagents = [subagent_of("Explore", "a1", [result("r", "x")]), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:31:01Z", model="claude-fable-5-2")])]
|
|
found = chunk_figures(chunk_of("log", [kept], worker, subagents), "/h", BARE)
|
|
self.assertEqual(found.delegated, {"sonnet at low effort": 1, str(HIGH): 1})
|
|
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, str(HIGH): 8.0})
|
|
|
|
def test_names_a_subagent_by_its_meta_or_else_its_call_or_else_unknown(self):
|
|
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Plan"))]
|
|
subagents = [subagent_of("stability-reviewer", "a1", []), subagent_of("", "a2", []), subagent_of("", "", [])]
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, subagents), "/h", BARE).reports, {"stability-reviewer": Reports(1, 0, 0), "Plan": Reports(1, 0, 0), "unknown": Reports(1, 0, 0)})
|
|
|
|
def test_counts_a_message_without_usage_or_a_priced_model_as_nothing_and_splits_a_cost_evenly_across_tools(self):
|
|
worker = [
|
|
{"type": "assistant", "uuid": "u1", "message": {"model": "claude-opus-5-5", "content": [call("Read"), call("Grep")], "usage": {"input_tokens": 1_000_000, "output_tokens": "many"}}},
|
|
{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t1", "content": "plain"}, "odd"]}},
|
|
{"type": "assistant", "uuid": "u2", "message": {"model": "claude-opus-5-5", "content": [], "usage": {"input_tokens": 2_000_000}}},
|
|
{"type": "assistant", "uuid": "u3", "message": {"model": "claude-opus-5-5", "content": []}},
|
|
{"type": "assistant", "uuid": "u4", "message": {"model": "claude-unheard-of", "content": [], "usage": {"input_tokens": 9_000_000}}},
|
|
]
|
|
found = chunk_figures(chunk_of("log", [], worker, []), "/h", BARE)
|
|
self.assertEqual(found.dollars["tool"], {"prompt": 4.0, "Read": 4.0, "Grep": 4.0})
|
|
self.assertEqual(found.dollars["project"], {"log": 12.0})
|
|
|
|
def test_prices_input_cache_writes_by_their_lifetime_cache_reads_and_output_at_the_models_list_prices(self):
|
|
usage = {"input_tokens": 1_000_000, "cache_creation_input_tokens": 2_000_000, "cache_creation": {"ephemeral_5m_input_tokens": 1_000_000, "ephemeral_1h_input_tokens": 1_000_000}, "cache_read_input_tokens": 1_000_000, "output_tokens": 1_000_000}
|
|
opus = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h", BARE)
|
|
self.assertAlmostEqual(opus.dollars["project"]["log"], 4 + 5 + 8 + 0.2 + 20)
|
|
sonnet = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", model="claude-sonnet-5-5", usage=usage)], []), "/h", BARE)
|
|
self.assertAlmostEqual(sonnet.dollars["project"]["log"], 2 + 2.5 + 4 + 0.2 + 10)
|
|
|
|
def test_counts_a_cache_write_without_a_lifetime_as_a_five_minute_one(self):
|
|
usage = {"cache_creation_input_tokens": 1_000_000}
|
|
found = chunk_figures(chunk_of("log", [], [message("m1", "2026-10-14T10:00:00Z", usage=usage)], []), "/h", BARE)
|
|
self.assertAlmostEqual(found.dollars["project"]["log"], 5)
|
|
|
|
def test_splits_the_cost_a_chunks_last_cost_state_records_for_each_model_across_its_messages_by_list_price(self):
|
|
worker = [
|
|
message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore")),
|
|
message("m2", "2026-10-14T10:01:00Z"),
|
|
cost_state(claude_opus_5_5=1.0),
|
|
cost_state(claude_opus_5_5=9.0, claude_haiku_5_5=0.5),
|
|
]
|
|
subagents = [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]
|
|
found = chunk_figures(chunk_of("log", [], worker, subagents), "/h", BARE)
|
|
self.assertEqual(found.dollars["subagent type"], {"worker": 6.0, "Explore": 3.0})
|
|
self.assertEqual(found.dollars["project"], {"log": 9.0})
|
|
self.assertEqual(found.unattributed, 0.5)
|
|
|
|
def test_counts_the_subagents_the_worker_launched_that_have_no_transcript(self):
|
|
worker = [message("m1", "2026-10-14T10:00:00Z", call("Agent", "a1", subagent_type="Explore"), call("Agent", "a2", subagent_type="Explore"), call("Read", "r1", file_path="/c/a"))]
|
|
found = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", [message("s1", "2026-10-14T10:00:01Z")])]), "/h", BARE)
|
|
self.assertEqual(found.lacking, 1)
|
|
unnamed = chunk_figures(chunk_of("log", [], worker, [subagent_of("", "", [message("s1", "2026-10-14T10:00:01Z")]), subagent_of("", "", [message("s2", "2026-10-14T10:00:02Z")])]), "/h", BARE)
|
|
self.assertEqual(unnamed.lacking, 0)
|
|
nested = [message("s1", "2026-10-14T10:00:01Z", call("Agent", "n1", subagent_type="Explore"))]
|
|
launched = chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [message("s2", "2026-10-14T10:00:02Z")]), subagent_of("Explore", "n1", [message("s3", "2026-10-14T10:00:03Z")])]), "/h", BARE)
|
|
self.assertEqual(launched.lacking, 0)
|
|
self.assertEqual(chunk_figures(chunk_of("log", [], worker, [subagent_of("Explore", "a1", nested), subagent_of("Explore", "a2", [])]), "/h", BARE).lacking, 1)
|
|
|
|
def test_charges_a_message_without_a_timestamp_to_the_rung_of_the_last_outcome(self):
|
|
undated = message("m1", "")
|
|
del undated["timestamp"]
|
|
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
|
|
self.assertEqual(chunk_figures(chunk_of("log", [kept], [undated], []), "/h", BARE).dollars["rung"], {str(HIGH): 2.0})
|
|
|
|
def test_keeps_list_prices_for_a_model_the_cost_state_does_not_name(self):
|
|
worker = [message("m1", "2026-10-14T10:00:00Z"), message("m2", "2026-10-14T10:01:00Z", model="claude-sonnet-5-5"), cost_state(claude_opus_5_5=3.0)]
|
|
found = chunk_figures(chunk_of("log", [], worker, []), "/h", BARE)
|
|
self.assertEqual(found.dollars["project"], {"log": 4.0})
|
|
self.assertEqual(found.unattributed, 0.0)
|
|
|
|
|
|
class Calls(unittest.TestCase):
|
|
def test_calls_when_chunks_on_a_rung_took_more_rounds_or_unfinished_verdicts_than_the_rung_above(self):
|
|
chunks = [figures(LOW, rounds=2)] * 5 + [figures(HIGH, unfinished=1)] * 5
|
|
self.assertEqual(calls(LADDER, totals(chunks)), [Call("rounds", (str(LOW), str(HIGH)))])
|
|
self.assertEqual(calls(LADDER, totals(chunks))[0].sentence(), f"Chunks routed onto {LOW} took more review rounds past each pass's first than chunks routed onto {HIGH}.")
|
|
[unfinished] = calls(LADDER, totals([figures(LOW, unfinished=1)] * 5 + [figures(HIGH)] * 5))
|
|
self.assertEqual(unfinished, Call("unfinished", (str(LOW), str(HIGH))))
|
|
self.assertEqual(unfinished.sentence(), f"Chunks routed onto {LOW} took more unfinished verdicts than chunks routed onto {HIGH}.")
|
|
|
|
def test_compares_the_bottom_rungs_and_calls_nothing_on_a_tie(self):
|
|
self.assertEqual(calls(LADDER, totals([figures(LADDER.rungs[0], rounds=1)] * 5 + [figures(LADDER.rungs[1])] * 5)), [Call("rounds", (str(LADDER.rungs[0]), str(LADDER.rungs[1])))])
|
|
self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=1)] * 5 + [figures(HIGH, rounds=1)] * 5)), [])
|
|
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 20)})] * 5)), [])
|
|
|
|
def test_compares_only_rungs_with_five_chunks_each(self):
|
|
self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=2)] * 5 + [figures(HIGH)] * 4)), [])
|
|
|
|
def test_calls_when_a_subagent_types_reports_outgrow_the_searches_they_ran(self):
|
|
[call] = calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 5))
|
|
self.assertEqual(call, Call("reports", ("Explore",)))
|
|
self.assertEqual(call.sentence(), "Explore subagents' reports took more of the worker's context than their searches would have.")
|
|
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 4)), [])
|
|
self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 30)})] * 5)), [])
|
|
|
|
def test_renders_each_dimension_largest_first_and_the_calls(self):
|
|
chunk = ChunkFigures({"project": collections.Counter({"a": 2.5, "b": 1234.5})}, LOW, 1, 0, collections.Counter({str(HIGH): 2}), {"Explore": Reports(2, 3, 4)}, collections.Counter(), 0.0)
|
|
text = render(LADDER, totals([chunk]), [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", 40, "")
|
|
self.assertIn("Measured 2026-10-15T00:00:00Z, at cycle 40 across loop-claude's archive, from 1 finished chunk.", text)
|
|
self.assertIn("## Calls for a routing change\n\n- Explore subagents' reports took more of the worker's context than their searches would have.\n", text)
|
|
self.assertIn("## Dollars by project", text)
|
|
self.assertIn("| b | $1,234.50 |\n| a | $2.50 |", text)
|
|
self.assertIn("cache reads included", text)
|
|
self.assertIn("The judge's cost counts from the verdicts and rejections that record it, so a chunk archived before they did, and a judge run that failed, add none.", text)
|
|
self.assertNotIn("left out", text)
|
|
self.assertNotIn("no transcript", text)
|
|
self.assertIn("The archive holds no subagent transcript for 2 of the chunks' Agent calls.", render(LADDER, totals([dataclasses.replace(chunk, lacking=1)] * 2), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
self.assertIn("$0.50 of the recorded cost is left out", render(LADDER, totals([ChunkFigures({}, LOW, 0, 0, collections.Counter(), {}, collections.Counter(), 0.5), chunk]), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |", text)
|
|
two = ChunkFigures({}, HIGH, 0, 1, collections.Counter({str(HIGH): 2, str(LOW): 1}), {}, collections.Counter(), 0.0)
|
|
self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |\n| {HIGH} | 1 | 0.0 | 1.0 | unknown | {HIGH} 2, {LOW} 1 | |", render(LADDER, totals([two, chunk]), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
self.assertIn("| Explore | 2 | 3 | 4 |", text)
|
|
escalated = render(LADDER, totals([figures(escalated={2: 2}), figures(escalated={2: 1, 10: 1}), figures(start=None, escalated={3: 1})]), [], "2026-10-15T00:00:00Z", 40, "")
|
|
self.assertIn(f"| {LOW} | 2 | 0.0 | 0.0 | unknown | | trigger 2: 3, trigger 10: 1 |", escalated)
|
|
self.assertIn("## Escalations by trigger across every chunk, routed or not\n\ntrigger 2: 3, trigger 3: 1, trigger 10: 1.\n", escalated)
|
|
self.assertIn("## Escalations by trigger across every chunk, routed or not\n\nNone.\n", text)
|
|
self.assertIn("## Calls for a routing change\n\nNone.\n", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, ""))
|
|
self.assertIn("from 0 finished chunks that finished after the reset of 2026-10-14T00:00:00Z.", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, "2026-10-14T00:00:00Z"))
|
|
|
|
def test_renders_the_mean_later_fixes_of_the_matched_chunks_and_how_many_they_are(self):
|
|
text = render(LADDER, totals([*MATCHED, figures()]), [], "2026-10-15T00:00:00Z", 40, "")
|
|
self.assertIn(f"| {LOW} | 4 | 0.0 | 0.0 | 1.0 (3 known) | | |", text)
|
|
self.assertIn("## Routed chunks by the rung they started on\n\nA routed chunk's later fixes are", text)
|
|
self.assertNotIn("later fixes", text.split("## Calls for a routing change")[0])
|
|
|
|
def test_counts_the_routed_chunks_whose_later_fixes_are_unknown(self):
|
|
self.assertIn("The later fixes of 2 of the routed chunks are unknown", render(LADDER, totals([*MATCHED, figures(), figures(), figures(start=None)]), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
self.assertNotIn("are unknown", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
|
|
def test_names_each_default_branch_unread(self):
|
|
self.assertIn("## Default branches unread\n\n- log: no main branch\n", render(LADDER, totals(MATCHED, unread=("log: no main branch",)), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
self.assertNotIn("branches unread", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, ""))
|
|
|
|
def test_measures_once_the_count_runs_20_past_the_last_measurement(self):
|
|
self.assertEqual([due(cycle, 20) for cycle in (39, 40, 41)], [False, True, True])
|
|
self.assertTrue(due(20, 0))
|
|
|
|
|
|
class BranchHistory(unittest.TestCase):
|
|
def test_counts_no_chunk_of_a_project_whose_plan_is_outside_the_checkout(self):
|
|
checkout = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())).resolve()
|
|
folder = configured(self, checkout)
|
|
config = folder / "config.toml"
|
|
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
|
|
found = archive.branch_history(str(checkout), folder.parents[1], BARE, git_repo.git)
|
|
assert isinstance(found, Failure), found
|
|
self.assertEqual(found.reason, f"its chunks' items and later fixes are not counted: the project's config puts {folder}/todo.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
|
|
|
|
|
|
class Archive(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = pathlib.Path(tempfile.mkdtemp())
|
|
self.addCleanup(shutil.rmtree, self.root)
|
|
|
|
def chunk(self, key, name, kept, worker, subagent=None):
|
|
folder = self.root / key / name
|
|
(folder / "session" / "s" / "subagents").mkdir(parents=True)
|
|
(folder / "copy.outcomes.md").write_text(f"Task: t\n\n{kept}")
|
|
(folder / "copy.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in worker))
|
|
if subagent:
|
|
meta, entries = subagent
|
|
(folder / "session" / "s" / "subagents" / "agent-a.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in entries))
|
|
(folder / "session" / "s" / "subagents" / "agent-a.meta.json").write_text(json.dumps(meta))
|
|
|
|
def test_counts_the_verdicts_across_every_projects_chunks(self):
|
|
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.unfinished("w", "e", "j", "")) + stamped("2026-10-14T10:01:00Z", outcomes.stopped("w", "Ctrl-C")) + stamped("2026-10-14T10:02:00Z", outcomes.finished("w", "e", "j")), [])
|
|
self.chunk("log", "c2", stamped("2026-10-14T10:00:00Z", outcomes.interrupted("q", "e", "j", "", "w")), [])
|
|
self.assertEqual(archive.cycles(self.root), 2)
|
|
self.assertEqual(archive.cycles(self.root / "missing"), 0)
|
|
|
|
def test_measures_each_finished_chunk_keyed_by_its_project(self):
|
|
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z", call("Agent", "a1", subagent_type="Explore"))], ({"agentType": "Explore", "toolUseId": "a1"}, [message("s1", "2026-10-14T09:00:01Z")]))
|
|
self.chunk("log", "c2", stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
found = archive.measure_archive(self.root, "/h", "", no_checkout)
|
|
self.assertEqual([chunk.dollars["project"] for chunk in found.chunks], [{"home/x": 4.0}])
|
|
self.assertEqual(found.chunks[0].dollars["subagent type"], {"worker": 2.0, "Explore": 2.0})
|
|
self.assertEqual(archive.measure_archive(self.root, "/h", "2026-10-14T10:00:00Z", no_checkout), archive.ArchiveChunks([], ()))
|
|
|
|
def test_measures_a_chunk_whose_transcripts_are_compressed(self):
|
|
self.chunk("home/x", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z", call("Agent", "a1", subagent_type="Explore"))], ({"agentType": "Explore", "toolUseId": "a1"}, [message("s1", "2026-10-14T09:00:01Z")]))
|
|
for copy in self.root.rglob("*.jsonl"):
|
|
self.assertIsNone(compress(copy))
|
|
found = archive.measure_archive(self.root, "/h", "", no_checkout)
|
|
self.assertEqual(found.chunks[0].dollars["subagent type"], {"worker": 2.0, "Explore": 2.0})
|
|
|
|
def test_counts_each_chunks_later_fixes_from_its_projects_checkout(self):
|
|
home = self.root / "home"
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
|
self.ship_then_fix(home / "code" / "log", env)
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
|
self.assertEqual([(chunk.dollars["project"].keys(), chunk.fixed_later) for chunk in found.chunks], [({"log": 0}.keys(), 1)])
|
|
|
|
def test_reads_a_projects_round_titles_and_its_branchs_history_through_its_configs_templates(self):
|
|
home = self.root / "home"
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
|
round_title = "PAY-8 - Answer stability round {}: 1 🔴, 0 🟡 (1 misread); fix"
|
|
self.ship_then_fix(home / "code" / "log", env, round_title.format(1), ships_fix=False)
|
|
config = write_config(home, home / "code" / "log") / "config.toml"
|
|
worker = [message(f"m{n}", "2026-10-14T09:00:00Z", call("Bash", command=f"git commit -m '{round_title.format(n)}'")) for n in (1, 2)]
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), worker)
|
|
bare = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
|
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\n')
|
|
templated = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
|
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in bare.chunks], [(0, 0)])
|
|
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in templated.chunks], [(1, 1)])
|
|
|
|
def test_measures_no_chunk_of_a_project_while_a_config_cannot_be_read_and_names_it(self):
|
|
home = self.root / "home"
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
|
self.ship_then_fix(home / "code" / "log", env)
|
|
config = write_config(home, home / "code" / "log") / "config.toml"
|
|
config.write_text(config.read_text() + 'commit-title = "PAY-{id}"\n')
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
|
self.assertEqual(found.chunks, [])
|
|
self.assertEqual(len(found.unread), 1)
|
|
self.assertRegex(found.unread[0], r"^log: none of its chunks is measured: .*`commit-title` is 'PAY-\{id\}'")
|
|
|
|
def test_reads_each_projects_history_guarded_and_each_read_of_its_branch_time_limited(self):
|
|
home = self.root / "home"
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
|
self.ship_then_fix(home / "code" / "log", env)
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
made = []
|
|
|
|
def recording(*args, **options):
|
|
made.append((args[0], options))
|
|
return git_repo.git(*args, **options)
|
|
archive.measure_archive(self.root, str(home), "", recording)
|
|
self.assertTrue(all(options.get("guarded") is True for _, options in made))
|
|
read = [options.get("timeout") for command, options in made if command in ("log", "ls-tree", "show", "diff", "blame")]
|
|
self.assertEqual(set(read), {archive.GIT_TIMEOUT})
|
|
self.assertIn("blame", [command for command, _ in made])
|
|
|
|
def test_names_a_project_whose_checkout_has_no_default_branch(self):
|
|
home = self.root / "home"
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
|
(home / "code" / "log").mkdir(parents=True)
|
|
subprocess.run(["git", "init", "-q"], cwd=home / "code" / "log", check=True, env=env, capture_output=True)
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
|
self.assertEqual(found.unread, ("log: no origin remote, and no main or master branch to take as the default branch",))
|
|
|
|
def test_names_each_project_whose_history_it_cannot_read(self):
|
|
self.chunk("gone", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
found = archive.measure_archive(self.root, "/h", "", no_checkout)
|
|
self.assertEqual(([chunk.fixed_later for chunk in found.chunks], found.unread), ([None], ("gone: no checkout",)))
|
|
|
|
def ship_then_fix(self, checkout, env, fix_title="Fix six", ships_fix=True):
|
|
"""Makes a checkout whose main branch merges the chunk that ships item 7 of LOW_ROUTE, then one titled fix_title that rewrites a line item 7 shipped, and ships a defect item where ships_fix."""
|
|
checkout.mkdir(parents=True)
|
|
git = lambda *args: subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=checkout, check=True, env=env, capture_output=True)
|
|
plan = lambda rows: "# todo\n\n## Items\n\n| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n" + rows
|
|
|
|
def merge(branch, title, files):
|
|
git("switch", "-q", "-c", branch)
|
|
for name, text in files.items():
|
|
(checkout / name).write_text(text)
|
|
git("add", *files)
|
|
git("commit", "-q", "-m", title)
|
|
git("switch", "-q", "main")
|
|
git("merge", "-q", "--no-ff", "-m", f"Merge {branch}", branch)
|
|
fix = "| 6 | 0.1.0 | | defect | **Fix six.** | 1 | 1 | 1 | 1 | 2 | 1.1 by 3 | 1.0 |\n"
|
|
git("init", "-q", "-b", "main")
|
|
(checkout / "todo.md").write_text(plan("| 7 | 0.1.0 | | | **Ship seven.** | 1 | 1 | 1 | 1 | 2 | 1.1 by 3 | 1.0 |\n" + fix))
|
|
(checkout / "a.py").write_text("1\n")
|
|
git("add", "todo.md", "a.py")
|
|
git("commit", "-q", "-m", "Start")
|
|
merge("ship", "Ship seven", {"a.py": "1\nx\n", "todo.md": plan(fix)})
|
|
merge("fix", fix_title, {"a.py": "1\ny\n", "todo.md": plan("" if ships_fix else fix)})
|
|
|
|
def test_reads_each_cycles_route_from_its_own_outcomes_file(self):
|
|
transcript = self.root / "t.jsonl"
|
|
kept = self.root / "archive"
|
|
first = message("m1", "2026-09-21T00:00:00Z")
|
|
transcript.write_text(json.dumps(first) + "\n")
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.unfinished(LOW_ROUTE, "e", "j", ""), 1790000000, replace=replace_file))
|
|
transcript.write_text(json.dumps(first) + "\n" + json.dumps(message("m2", "2099-01-01T00:00:00Z")) + "\n")
|
|
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.finished(Route(HIGH).worker_line(), "m", "j"), 4070908800, replace=replace_file))
|
|
(found,) = archive.measure_archive(kept, "/h", "", no_checkout).chunks
|
|
self.assertEqual((found.start, found.unfinished), (LOW, 1))
|
|
self.assertEqual(found.dollars["rung"], {str(LOW): 2.0, str(HIGH): 2.0})
|
|
|
|
def test_skips_a_chunk_it_cannot_read_and_names_it(self):
|
|
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [])
|
|
(self.root / "log" / "c1" / "copy.jsonl").unlink()
|
|
self.chunk("other", "c2", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
|
self.chunk("broken", "c3", "", [])
|
|
(self.root / "broken" / "c3" / "copy.outcomes.md").write_bytes(b"\xff")
|
|
found = archive.measure_archive(self.root, "/h", "", no_checkout)
|
|
self.assertEqual((len(found.chunks), found.skipped), (1, ("broken", "log/c1")))
|
|
self.assertIn("## Chunks skipped, unreadable\n\n- log/c1\n", render(LADDER, totals([], ("log/c1",)), [], "2026-10-15T00:00:00Z", 20, ""))
|
|
|
|
def test_keeps_the_cycle_it_measured_at_and_reads_it_back(self):
|
|
self.assertEqual(state.read_state(self.root), state.State(0, ()))
|
|
self.assertIsNone(state.mark_measured(self.root, 40))
|
|
self.assertEqual(state.read_state(self.root), state.State(40, ()))
|
|
self.assertIsNone(archive.write_figures(self.root, "# Routing figures\n"))
|
|
self.assertEqual(state.read_state(self.root).measured_at, 40, "the rendered figures are for the human, and never read back")
|
|
|
|
def test_reads_the_calls_raised_since_the_latest_reset(self):
|
|
a, b, c = Call("reports", ("Explore",)), Call("rounds", ("x", "y")), Call("unfinished", ("x", "y"))
|
|
self.assertEqual(state.raised(self.root), state.Raised("", frozenset()))
|
|
self.assertIsNone(state.mark_raised(self.root, [a], "2026-10-15T00:00:00Z", replace_file))
|
|
self.assertIsNone(state.mark_raised(self.root, [b, c], "2026-10-15T01:00:00Z", replace_file))
|
|
self.assertEqual((self.root / state.RAISED).read_text(), f"# Routing calls raised\n\n- 2026-10-15T00:00:00Z {a.sentence()}\n- 2026-10-15T01:00:00Z {b.sentence()}\n- 2026-10-15T01:00:00Z {c.sentence()}\n")
|
|
self.assertEqual(state.raised(self.root), state.Raised("", frozenset({a, b, c})))
|
|
with open(self.root / state.RAISED, "a") as handle:
|
|
handle.write("- 2026-10-15T01:00:00Z Reset\n")
|
|
self.assertEqual(state.raised(self.root), state.Raised("2026-10-15T01:00:00Z", frozenset()), "a call stamped at the reset's second was raised before it")
|
|
self.assertIsNone(state.mark_raised(self.root, [a], "2026-10-15T02:00:00Z", replace_file))
|
|
self.assertEqual(state.raised(self.root), state.Raised("2026-10-15T01:00:00Z", frozenset({a})))
|
|
|
|
def test_fails_when_it_cannot_keep_the_state(self):
|
|
assert_not_root(self)
|
|
self.root.chmod(0o555)
|
|
self.addCleanup(self.root.chmod, 0o755)
|
|
self.assertIsInstance(state.mark_measured(self.root, 40), Failure)
|
|
self.assertIsInstance(state.mark_raised(self.root, [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", replace_file), Failure)
|
|
|
|
def test_fails_when_it_cannot_list_a_raised_call_for_the_human(self):
|
|
self.assertIsNone(state.mark_measured(self.root, 40))
|
|
self.assertEqual(state.mark_raised(self.root, [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", replace=lambda path, data: Failure("full")), Failure(f"could not write {self.root / state.RAISED}: full"))
|
|
|
|
def test_knows_a_raised_call_by_its_fields_whatever_its_line_reads(self):
|
|
call = Call("reports", ("Explore",))
|
|
self.assertIsNone(state.mark_raised(self.root, [call], "2026-10-15T00:00:00Z", replace_file))
|
|
(self.root / state.RAISED).write_text("# Routing calls raised\n\n- 2026-10-15T00:00:00Z Reworded.\n")
|
|
self.assertEqual(state.raised(self.root).calls, frozenset({call}))
|
|
|
|
def test_fails_on_a_state_it_cannot_read(self):
|
|
for text in ("not json", "[]", '{"measured_at": "40", "raised": []}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "nope", "subjects": []}]}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "reports", "subjects": ["a", "b"]}]}', '{"measured_at": true, "raised": []}', '{"measured_at": 40, "raised": [{"at": "x", "kind": "reports", "subjects": "a"}]}'):
|
|
with self.subTest(text):
|
|
(self.root / state.STATE).write_text(text)
|
|
self.assertIsInstance(state.read_state(self.root), Failure)
|
|
self.assertIsInstance(state.raised(self.root), Failure)
|
|
(self.root / state.STATE).unlink()
|
|
(self.root / state.STATE).mkdir()
|
|
self.assertIsInstance(state.read_state(self.root), Failure)
|
|
|
|
def test_holds_the_lock_while_no_other_loop_does(self):
|
|
with archive.locked(self.root) as held, archive.locked(self.root) as other:
|
|
self.assertEqual((held, other), (True, False))
|
|
with archive.locked(self.root) as again:
|
|
self.assertTrue(again)
|
|
|
|
def test_fails_on_an_archive_it_cannot_read(self):
|
|
(self.root / state.RAISED).mkdir()
|
|
self.assertIsInstance(state.raised(self.root), Failure)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|