Files
claude-config/bin/reading_cost/test_reading_cost.py
T

267 lines
15 KiB
Python

import collections
import contextlib
import hashlib
import io
import json
import os
import pathlib
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from transcript_archive.write import archive
from claude_code.entries import AssistantTurn
from project_config.templates import BRANCH
from reading_cost.measure import Checkout, Reading
from text_files.doubles import compress
from failure import Failure
from reading_cost import figures, main, measure
from transcript_archive.read import finished_chunks, FinishedChunk
from record_store.held import Held
from test_env import isolated_env
from record_store.fakes import MemoryStore
from transcript_archive import outcomes
from transcript_archive.location import project_archive
from reading_cost.shell_reads import Reads, shell_reads
from text_files.file_writes import replace_file
from project_config.doubles import write_config
TOOL = pathlib.Path(__file__).resolve().parents[1] / "reading-cost"
def said(timestamp, message_id, command=None, tokens=0, cwd="/c/p", **call):
part = {"type": "tool_use", "name": "Bash", "input": {"command": command}} if command else {"type": "tool_use", **call}
return {"type": "assistant", "timestamp": timestamp, "cwd": cwd, "message": {"id": message_id, "content": [part], "usage": {"input_tokens": tokens, "cache_creation_input_tokens": 2 * tokens, "cache_read_input_tokens": 1000}}}
def jsonl(entries):
return b"".join(json.dumps(entry).encode() + b"\n" for entry in entries)
def folder_name():
return hashlib.sha256(os.urandom(8)).hexdigest()
class ShellReads(unittest.TestCase):
def test_reads_operands_past_a_cd_and_skips_patterns_redirect_targets_and_heredoc_bodies(self):
command = "cd ../w && sed -n 1,5p src/a.py | grep -n x\ngrep -rn 'def x' bin/t.py > out.txt\npython3 - <<'EOF'\ncat secret.py\nEOF\ncat ~/README.md\nsed -i s/a/b/ c.py"
self.assertEqual(shell_reads(command, "/c/p", "/h"), Reads(["/c/w/src/a.py", "/c/w/bin/t.py", "/h/README.md"], []))
def test_ends_a_dashed_heredoc_at_its_marker(self):
self.assertEqual(shell_reads("cat <<-EOF > x.md\ncat secret.py\nEOF\ncat a.py", "/c/p", "/h"), Reads(["/c/p/a.py"], []))
def test_names_the_worktree_a_git_worktree_add_makes(self):
self.assertEqual(shell_reads("git worktree add -q -b topic ../p-topic origin/main", "/c/p", "/h"), Reads([], ["/c/p-topic"]))
class Measure(unittest.TestCase):
def test_counts_each_message_once_and_the_tracked_files_read_before_the_first_commit(self):
worker = [
said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p/README.md"}),
said("2", "m1", tokens=10, name="Read", input={"file_path": "/c/p/.env"}),
said("3", "m2", command="git worktree add ../w && cd ../w && cat src/a.py && cd src && cat b.py"),
said("4", "m3", command="cd /tmp && cat x.py"),
said("5", "m4", command="git -C ../w commit -m 'x'", tokens=100),
said("6", "m5", tokens=100, name="Read", input={"file_path": "/c/p/late.py"}),
]
subagent = [said("2", "s1", tokens=1, name="Grep", input={"path": "/c/w/src/a.py"}), said("7", "s2", tokens=100, name="Read", input={"file_path": "/c/p/late.py"})]
checkout = Checkout("/c/w", "/c/p", frozenset({"README.md", "late.py", "src/a.py", "src/b.py", "x.py"}), BRANCH)
self.assertEqual(measure.measure([jsonl(worker), jsonl(subagent)], checkout, "/h"), Reading(33, collections.Counter({"src/a.py": 2, "README.md": 1, "src/b.py": 1})))
def test_counts_only_reads_and_greps_by_their_path_and_a_command_without_a_cwd_from_the_root(self):
worker = [
said("1", "m1", name="Edit", input={"file_path": "/c/p/a.py"}),
said("2", "m2", name="Glob", input={"path": "/c/p/a.py"}),
said("3", "m3", command="cat c/p/b.py", cwd=""),
said("4", "m4", command="git commit -m x"),
]
self.assertEqual(measure.measure([jsonl(worker)], Checkout("/c/p", "/c/p", frozenset({"a.py", "b.py"}), BRANCH), "/h").files, collections.Counter({"b.py": 1}))
def test_counts_a_file_read_in_the_worktree_chunk_preflight_made_as_the_checkout_s(self):
worker = [said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p-item-7/README.md"}), said("2", "m2", command="git commit -m x")]
checkout = Checkout("/c/p-item-7", "/c/p", frozenset({"README.md"}), BRANCH)
self.assertEqual(measure.measure([jsonl(worker)], checkout, "/h").files, collections.Counter({"README.md": 1}))
def test_counts_a_file_read_in_a_worktree_named_by_the_configs_branch_template(self):
worker = [said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p-feature-PAY-7/README.md"}), said("2", "m2", command="git commit -m x")]
checkout = Checkout("/c/p-feature-PAY-7", "/c/p", frozenset({"README.md"}), "feature/PAY-{id}")
self.assertEqual(measure.measure([jsonl(worker)], checkout, "/h").files, collections.Counter({"README.md": 1}))
self.assertEqual(measure.measure([jsonl(worker)], Checkout("/c/p-feature-PAY-7", "/c/p", frozenset({"README.md"}), BRANCH), "/h").files, collections.Counter())
def test_measures_nothing_for_a_worker_that_never_committed(self):
self.assertIsNone(measure.measure([jsonl([said("1", "m1", command="git log --grep commit")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
self.assertIsNone(measure.measure([jsonl([said("1", "m1", command="echo git commit")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
def test_finds_a_commit_behind_git_options(self):
self.assertIsNotNone(measure.measure([jsonl([said("1", "m1", command="git -c user.name=t commit -m x")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
def test_skips_a_usage_that_is_not_a_count(self):
self.assertEqual(measure.tokens_in([AssistantTurn(message_id="m", usage={"input_tokens": "7", "cache_creation_input_tokens": 2})]), 2)
class Figures(unittest.TestCase):
def test_rises_only_above_the_five_before_and_needs_ten(self):
flat = [Reading(100, collections.Counter({"a.py": 1}))] * 10
self.assertEqual(figures.trend(flat[:9]), "9 chunks measured since the last reset; the trend needs 10.")
self.assertTrue(figures.trend(flat).startswith("Not rising: the last 5 chunks' median is 100 new input tokens"))
self.assertTrue(figures.trend(flat[:9] + [Reading(3000000, collections.Counter())]).startswith("Not rising:"))
self.assertEqual(figures.trend(flat[:5] + [Reading(101, collections.Counter({"b.py": 2}))] * 5), "Rising: the last 5 chunks' median is 101 new input tokens before their first commit, the 5 before them 100.\n\nRead most over the last 5 (opens and searches):\n\n- `b.py` 10")
def test_a_figure_reads_back_as_the_reading_it_was_written_from_and_a_damaged_one_fails(self):
reading = Reading(7, collections.Counter({"b.py": 1, "a.py": 3}))
written = figures.figure(FinishedChunk("abcdef0123456789", "2026-10-08T00:00:00Z", pathlib.Path(), []), reading)
self.assertEqual(written, "## Reading cost, chunk abcdef012345, finished 2026-10-08T00:00:00Z\n\n7 new input tokens before the first commit.\n\nRead most (opens and searches):\n\n- `a.py` 3\n- `b.py` 1\n")
self.assertEqual(figures.readings_of(written), [reading])
self.assertEqual(figures.readings_of(written.replace("7 new", "seven new")), Failure('the figure under "## Reading cost, chunk abcdef012345, finished 2026-10-08T00:00:00Z" holds no single tokens line; restore it from git history'))
for damaged in (written.replace(", finished", ""), "## Reading cost reset, x\n"):
with self.subTest(damaged=damaged):
self.assertIsInstance(figures.readings_of(damaged), Failure)
def test_counts_only_figures_finished_after_the_last_reset_in_the_order_they_finished(self):
def written(chunk, finished, tokens):
return figures.figure(FinishedChunk(chunk, finished, pathlib.Path(), []), Reading(tokens, collections.Counter()))
history = "\n".join([written("a", "2026-10-01T00:00:00Z", 1), written("c", "2026-10-03T00:00:00Z", 3), "## Reading cost reset, 2026-10-02T00:00:00Z\n", written("b", "2026-10-04T00:00:00Z", 2)])
self.assertEqual(figures.readings_of(history), [Reading(3, collections.Counter()), Reading(2, collections.Counter())])
class ReadingCostTool(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.home = pathlib.Path(os.path.realpath(temp.name))
self.env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(self.home)}
self.checkout = self.home / "code" / "proj"
self.checkout.mkdir(parents=True)
(self.checkout / "a.py").write_text("")
self.git("init", "-q")
self.git("add", "a.py")
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
self.archive = project_archive(str(self.checkout), self.home, {})
write_config(self.home, self.checkout)
def git(self, *args):
subprocess.run(["git", *args], cwd=self.checkout, check=True, env=self.env)
def archive_chunk(self, at, outcome=outcomes.finished, worktree=None):
"""Archives a chunk whose worker read a.py in worktree, else in the checkout."""
transcript = self.home / ".claude" / "projects" / "-proj" / f"{folder_name()}.jsonl"
(transcript.with_suffix("") / "subagents").mkdir(parents=True)
read_in = worktree or self.checkout
worker = [said("1", "m1", tokens=10, cwd=str(read_in), name="Read", input={"file_path": str(read_in / "a.py")}), said("2", "m2", command="git commit -m x")]
transcript.write_bytes(jsonl([{"timestamp": str(at)}, *worker]))
(transcript.with_suffix("") / "subagents" / "agent-a.jsonl").write_bytes(jsonl([said("1", "s1", tokens=1)]))
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(archive(self.archive, str(transcript), "Do it.", outcome("m", "merged", "j"), at, replace=replace_file))
def run_tool(self, cwd, *args):
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=cwd, env=self.env, text=True)
def test_writes_the_record_where_the_config_puts_it_and_fails_without_a_config(self):
config = self.home / ".claude-projects" / "proj" / "config.toml"
config.write_text(config.read_text().replace('"checkout:docs/reading-cost.md"', '"folder:docs/reading-cost.md"'))
record = config.parent / "docs" / "reading-cost.md"
record.parent.mkdir()
ran = self.run_tool(self.checkout, "--reset")
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
self.assertIn(f"to {record}.", ran.stdout)
self.assertTrue(record.is_file())
self.assertFalse((self.checkout / "docs").exists())
config.unlink()
ran = self.run_tool(self.checkout)
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
self.assertIn("no config names the checkout", ran.stderr)
def test_measures_a_chunk_whose_archived_transcripts_are_compressed(self):
self.archive_chunk(1790000000)
for copy in self.archive.rglob("*.jsonl"):
self.assertIsNone(compress(copy))
ran = self.run_tool(self.checkout)
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertIn("33 new input tokens before the first commit.", (self.checkout / "docs" / "reading-cost.md").read_text())
def test_appends_each_finished_chunk_once_in_the_order_they_finished_from_a_worktree(self):
self.archive_chunk(1790000200)
self.archive_chunk(1790000100)
self.archive_chunk(1790000000, lambda *fields: outcomes.unfinished(*fields, ""))
chunks = finished_chunks(self.archive)
self.assertEqual([chunk.finished for chunk in chunks], ["2026-09-21T14:15:00Z", "2026-09-21T14:16:40Z"])
worktree = self.home / "code" / "proj-wt"
self.git("worktree", "add", "-q", str(worktree))
ran = self.run_tool(worktree)
self.assertEqual(ran.returncode, 0, ran.stderr)
history = worktree / "docs" / "reading-cost.md"
self.assertEqual(ran.stdout, f"Appended 2 reading-cost figures from {self.home / '.claude' / 'loop-claude-transcripts' / 'proj'} to {history}.\n2 chunks measured since the last reset; the trend needs 10.\n")
figure = "## Reading cost, chunk {}, finished {}\n\n33 new input tokens before the first commit.\n\nRead most (opens and searches):\n\n- `a.py` 1\n"
self.assertEqual(history.read_text(), "# Reading cost\n\n" + "\n".join(figure.format(chunk.folder[:12], chunk.finished) for chunk in chunks))
self.assertTrue(self.run_tool(worktree).stdout.startswith(f"Appended 0 reading-cost figures from"))
self.assertEqual(history.read_text().count("## Reading cost"), 2)
reset = self.run_tool(worktree, "--reset")
self.assertEqual(reset.stdout, f"Appended 0 reading-cost figures from {self.home / '.claude' / 'loop-claude-transcripts' / 'proj'} and a reset to {history}.\n0 chunks measured since the last reset; the trend needs 10.\n")
self.assertRegex(history.read_text(), r"\n## Reading cost reset, \d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ\n$")
def test_counts_a_file_read_in_a_worktree_the_configs_branch_template_names(self):
config = self.home / ".claude-projects" / "proj" / "config.toml"
config.write_text(config.read_text() + 'branch = "feature/PAY-{id}"\n')
self.archive_chunk(1790000100, worktree=self.home / "code" / "proj-feature-PAY-7")
ran = self.run_tool(self.checkout)
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertIn("- `a.py` 1\n", (self.checkout / "docs" / "reading-cost.md").read_text())
def test_refuses_an_option_it_does_not_know(self):
ran = self.run_tool(self.checkout, "--rest")
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
def test_exits_1_outside_a_checkout(self):
ran = self.run_tool(self.home)
self.assertEqual(ran.returncode, 1)
self.assertIn("not a git repository", ran.stderr)
class Located(unittest.TestCase):
def test_lists_the_whole_checkout_from_a_folder_inside_it(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
root = pathlib.Path(temp.name).resolve()
(root / "sub").mkdir()
(root / "a.txt").write_text("a")
(root / "sub" / "b.txt").write_text("b")
for args in (("init", "--quiet"), ("add", ".")):
subprocess.run(["git", *args], check=True, cwd=root, env=isolated_env())
with contextlib.chdir(root / "sub"):
found = main.located(BRANCH)
self.assertEqual(found, Checkout(str(root), str(root), frozenset({"a.txt", "sub/b.txt"}), BRANCH))
def test_fails_in_a_checkout_whose_path_holds_a_newline(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
root = pathlib.Path(temp.name, "a\nb")
root.mkdir()
subprocess.run(["git", "init", "--quiet"], check=True, cwd=root, env=isolated_env())
with contextlib.chdir(root):
self.assertIsInstance(main.located(BRANCH), Failure)
class Update(unittest.TestCase):
def test_writes_over_the_version_it_read(self):
record = MemoryStore("# Reading cost\n", version=2)
archive = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
self.assertNotIsInstance(main.update(record, archive, Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h", "2026-11-14T00:00:00Z"), Failure)
self.assertEqual(record.overs, ["2"])
def test_fails_where_the_record_refuses_the_write(self):
class Unwritable(MemoryStore):
def write(self, text: str, over: str | None) -> Held | Failure:
return Failure("the wiki is down")
archive = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
updated = main.update(Unwritable("# Reading cost\n"), archive, Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h", "2026-11-14T00:00:00Z")
self.assertEqual(updated, Failure("the wiki is down"))
if __name__ == "__main__":
unittest.main()