267 lines
15 KiB
Python
267 lines
15 KiB
Python
import collections
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from transcript_archive.write import archive
|
|
from claude_code.entries import AssistantTurn
|
|
from project_config.templates import BRANCH
|
|
from reading_cost.measure import Checkout, Reading
|
|
from text_files.doubles import compress
|
|
from failure import Failure
|
|
from reading_cost import figures, main, measure
|
|
from transcript_archive.read import finished_chunks, FinishedChunk
|
|
from record_store.held import Held
|
|
from test_env import isolated_env
|
|
from record_store.fakes import MemoryStore
|
|
from transcript_archive import outcomes
|
|
from transcript_archive.location import project_archive
|
|
from reading_cost.shell_reads import Reads, shell_reads
|
|
from text_files.file_writes import replace_file
|
|
from project_config.doubles import write_config
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parents[1] / "reading-cost"
|
|
|
|
|
|
def said(timestamp, message_id, command=None, tokens=0, cwd="/c/p", **call):
|
|
part = {"type": "tool_use", "name": "Bash", "input": {"command": command}} if command else {"type": "tool_use", **call}
|
|
return {"type": "assistant", "timestamp": timestamp, "cwd": cwd, "message": {"id": message_id, "content": [part], "usage": {"input_tokens": tokens, "cache_creation_input_tokens": 2 * tokens, "cache_read_input_tokens": 1000}}}
|
|
|
|
|
|
def jsonl(entries):
|
|
return b"".join(json.dumps(entry).encode() + b"\n" for entry in entries)
|
|
|
|
|
|
def folder_name():
|
|
return hashlib.sha256(os.urandom(8)).hexdigest()
|
|
|
|
|
|
class ShellReads(unittest.TestCase):
|
|
def test_reads_operands_past_a_cd_and_skips_patterns_redirect_targets_and_heredoc_bodies(self):
|
|
command = "cd ../w && sed -n 1,5p src/a.py | grep -n x\ngrep -rn 'def x' bin/t.py > out.txt\npython3 - <<'EOF'\ncat secret.py\nEOF\ncat ~/README.md\nsed -i s/a/b/ c.py"
|
|
self.assertEqual(shell_reads(command, "/c/p", "/h"), Reads(["/c/w/src/a.py", "/c/w/bin/t.py", "/h/README.md"], []))
|
|
|
|
def test_ends_a_dashed_heredoc_at_its_marker(self):
|
|
self.assertEqual(shell_reads("cat <<-EOF > x.md\ncat secret.py\nEOF\ncat a.py", "/c/p", "/h"), Reads(["/c/p/a.py"], []))
|
|
|
|
def test_names_the_worktree_a_git_worktree_add_makes(self):
|
|
self.assertEqual(shell_reads("git worktree add -q -b topic ../p-topic origin/main", "/c/p", "/h"), Reads([], ["/c/p-topic"]))
|
|
|
|
|
|
class Measure(unittest.TestCase):
|
|
def test_counts_each_message_once_and_the_tracked_files_read_before_the_first_commit(self):
|
|
worker = [
|
|
said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p/README.md"}),
|
|
said("2", "m1", tokens=10, name="Read", input={"file_path": "/c/p/.env"}),
|
|
said("3", "m2", command="git worktree add ../w && cd ../w && cat src/a.py && cd src && cat b.py"),
|
|
said("4", "m3", command="cd /tmp && cat x.py"),
|
|
said("5", "m4", command="git -C ../w commit -m 'x'", tokens=100),
|
|
said("6", "m5", tokens=100, name="Read", input={"file_path": "/c/p/late.py"}),
|
|
]
|
|
subagent = [said("2", "s1", tokens=1, name="Grep", input={"path": "/c/w/src/a.py"}), said("7", "s2", tokens=100, name="Read", input={"file_path": "/c/p/late.py"})]
|
|
checkout = Checkout("/c/w", "/c/p", frozenset({"README.md", "late.py", "src/a.py", "src/b.py", "x.py"}), BRANCH)
|
|
self.assertEqual(measure.measure([jsonl(worker), jsonl(subagent)], checkout, "/h"), Reading(33, collections.Counter({"src/a.py": 2, "README.md": 1, "src/b.py": 1})))
|
|
|
|
def test_counts_only_reads_and_greps_by_their_path_and_a_command_without_a_cwd_from_the_root(self):
|
|
worker = [
|
|
said("1", "m1", name="Edit", input={"file_path": "/c/p/a.py"}),
|
|
said("2", "m2", name="Glob", input={"path": "/c/p/a.py"}),
|
|
said("3", "m3", command="cat c/p/b.py", cwd=""),
|
|
said("4", "m4", command="git commit -m x"),
|
|
]
|
|
self.assertEqual(measure.measure([jsonl(worker)], Checkout("/c/p", "/c/p", frozenset({"a.py", "b.py"}), BRANCH), "/h").files, collections.Counter({"b.py": 1}))
|
|
|
|
def test_counts_a_file_read_in_the_worktree_chunk_preflight_made_as_the_checkout_s(self):
|
|
worker = [said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p-item-7/README.md"}), said("2", "m2", command="git commit -m x")]
|
|
checkout = Checkout("/c/p-item-7", "/c/p", frozenset({"README.md"}), BRANCH)
|
|
self.assertEqual(measure.measure([jsonl(worker)], checkout, "/h").files, collections.Counter({"README.md": 1}))
|
|
|
|
def test_counts_a_file_read_in_a_worktree_named_by_the_configs_branch_template(self):
|
|
worker = [said("1", "m1", tokens=10, name="Read", input={"file_path": "/c/p-feature-PAY-7/README.md"}), said("2", "m2", command="git commit -m x")]
|
|
checkout = Checkout("/c/p-feature-PAY-7", "/c/p", frozenset({"README.md"}), "feature/PAY-{id}")
|
|
self.assertEqual(measure.measure([jsonl(worker)], checkout, "/h").files, collections.Counter({"README.md": 1}))
|
|
self.assertEqual(measure.measure([jsonl(worker)], Checkout("/c/p-feature-PAY-7", "/c/p", frozenset({"README.md"}), BRANCH), "/h").files, collections.Counter())
|
|
|
|
def test_measures_nothing_for_a_worker_that_never_committed(self):
|
|
self.assertIsNone(measure.measure([jsonl([said("1", "m1", command="git log --grep commit")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
|
|
self.assertIsNone(measure.measure([jsonl([said("1", "m1", command="echo git commit")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
|
|
|
|
def test_finds_a_commit_behind_git_options(self):
|
|
self.assertIsNotNone(measure.measure([jsonl([said("1", "m1", command="git -c user.name=t commit -m x")])], Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h"))
|
|
|
|
def test_skips_a_usage_that_is_not_a_count(self):
|
|
self.assertEqual(measure.tokens_in([AssistantTurn(message_id="m", usage={"input_tokens": "7", "cache_creation_input_tokens": 2})]), 2)
|
|
|
|
|
|
class Figures(unittest.TestCase):
|
|
def test_rises_only_above_the_five_before_and_needs_ten(self):
|
|
flat = [Reading(100, collections.Counter({"a.py": 1}))] * 10
|
|
self.assertEqual(figures.trend(flat[:9]), "9 chunks measured since the last reset; the trend needs 10.")
|
|
self.assertTrue(figures.trend(flat).startswith("Not rising: the last 5 chunks' median is 100 new input tokens"))
|
|
self.assertTrue(figures.trend(flat[:9] + [Reading(3000000, collections.Counter())]).startswith("Not rising:"))
|
|
self.assertEqual(figures.trend(flat[:5] + [Reading(101, collections.Counter({"b.py": 2}))] * 5), "Rising: the last 5 chunks' median is 101 new input tokens before their first commit, the 5 before them 100.\n\nRead most over the last 5 (opens and searches):\n\n- `b.py` 10")
|
|
|
|
def test_a_figure_reads_back_as_the_reading_it_was_written_from_and_a_damaged_one_fails(self):
|
|
reading = Reading(7, collections.Counter({"b.py": 1, "a.py": 3}))
|
|
written = figures.figure(FinishedChunk("abcdef0123456789", "2026-10-08T00:00:00Z", pathlib.Path(), []), reading)
|
|
self.assertEqual(written, "## Reading cost, chunk abcdef012345, finished 2026-10-08T00:00:00Z\n\n7 new input tokens before the first commit.\n\nRead most (opens and searches):\n\n- `a.py` 3\n- `b.py` 1\n")
|
|
self.assertEqual(figures.readings_of(written), [reading])
|
|
self.assertEqual(figures.readings_of(written.replace("7 new", "seven new")), Failure('the figure under "## Reading cost, chunk abcdef012345, finished 2026-10-08T00:00:00Z" holds no single tokens line; restore it from git history'))
|
|
for damaged in (written.replace(", finished", ""), "## Reading cost reset, x\n"):
|
|
with self.subTest(damaged=damaged):
|
|
self.assertIsInstance(figures.readings_of(damaged), Failure)
|
|
|
|
|
|
def test_counts_only_figures_finished_after_the_last_reset_in_the_order_they_finished(self):
|
|
def written(chunk, finished, tokens):
|
|
return figures.figure(FinishedChunk(chunk, finished, pathlib.Path(), []), Reading(tokens, collections.Counter()))
|
|
history = "\n".join([written("a", "2026-10-01T00:00:00Z", 1), written("c", "2026-10-03T00:00:00Z", 3), "## Reading cost reset, 2026-10-02T00:00:00Z\n", written("b", "2026-10-04T00:00:00Z", 2)])
|
|
self.assertEqual(figures.readings_of(history), [Reading(3, collections.Counter()), Reading(2, collections.Counter())])
|
|
|
|
|
|
class ReadingCostTool(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.home = pathlib.Path(os.path.realpath(temp.name))
|
|
self.env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(self.home)}
|
|
self.checkout = self.home / "code" / "proj"
|
|
self.checkout.mkdir(parents=True)
|
|
(self.checkout / "a.py").write_text("")
|
|
self.git("init", "-q")
|
|
self.git("add", "a.py")
|
|
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
|
|
self.archive = project_archive(str(self.checkout), self.home, {})
|
|
write_config(self.home, self.checkout)
|
|
|
|
def git(self, *args):
|
|
subprocess.run(["git", *args], cwd=self.checkout, check=True, env=self.env)
|
|
|
|
def archive_chunk(self, at, outcome=outcomes.finished, worktree=None):
|
|
"""Archives a chunk whose worker read a.py in worktree, else in the checkout."""
|
|
transcript = self.home / ".claude" / "projects" / "-proj" / f"{folder_name()}.jsonl"
|
|
(transcript.with_suffix("") / "subagents").mkdir(parents=True)
|
|
read_in = worktree or self.checkout
|
|
worker = [said("1", "m1", tokens=10, cwd=str(read_in), name="Read", input={"file_path": str(read_in / "a.py")}), said("2", "m2", command="git commit -m x")]
|
|
transcript.write_bytes(jsonl([{"timestamp": str(at)}, *worker]))
|
|
(transcript.with_suffix("") / "subagents" / "agent-a.jsonl").write_bytes(jsonl([said("1", "s1", tokens=1)]))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(archive(self.archive, str(transcript), "Do it.", outcome("m", "merged", "j"), at, replace=replace_file))
|
|
|
|
def run_tool(self, cwd, *args):
|
|
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=cwd, env=self.env, text=True)
|
|
|
|
def test_writes_the_record_where_the_config_puts_it_and_fails_without_a_config(self):
|
|
config = self.home / ".claude-projects" / "proj" / "config.toml"
|
|
config.write_text(config.read_text().replace('"checkout:docs/reading-cost.md"', '"folder:docs/reading-cost.md"'))
|
|
record = config.parent / "docs" / "reading-cost.md"
|
|
record.parent.mkdir()
|
|
ran = self.run_tool(self.checkout, "--reset")
|
|
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
|
|
self.assertIn(f"to {record}.", ran.stdout)
|
|
self.assertTrue(record.is_file())
|
|
self.assertFalse((self.checkout / "docs").exists())
|
|
config.unlink()
|
|
ran = self.run_tool(self.checkout)
|
|
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
|
|
self.assertIn("no config names the checkout", ran.stderr)
|
|
|
|
def test_measures_a_chunk_whose_archived_transcripts_are_compressed(self):
|
|
self.archive_chunk(1790000000)
|
|
for copy in self.archive.rglob("*.jsonl"):
|
|
self.assertIsNone(compress(copy))
|
|
ran = self.run_tool(self.checkout)
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertIn("33 new input tokens before the first commit.", (self.checkout / "docs" / "reading-cost.md").read_text())
|
|
|
|
def test_appends_each_finished_chunk_once_in_the_order_they_finished_from_a_worktree(self):
|
|
self.archive_chunk(1790000200)
|
|
self.archive_chunk(1790000100)
|
|
self.archive_chunk(1790000000, lambda *fields: outcomes.unfinished(*fields, ""))
|
|
chunks = finished_chunks(self.archive)
|
|
self.assertEqual([chunk.finished for chunk in chunks], ["2026-09-21T14:15:00Z", "2026-09-21T14:16:40Z"])
|
|
worktree = self.home / "code" / "proj-wt"
|
|
self.git("worktree", "add", "-q", str(worktree))
|
|
ran = self.run_tool(worktree)
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
history = worktree / "docs" / "reading-cost.md"
|
|
self.assertEqual(ran.stdout, f"Appended 2 reading-cost figures from {self.home / '.claude' / 'loop-claude-transcripts' / 'proj'} to {history}.\n2 chunks measured since the last reset; the trend needs 10.\n")
|
|
figure = "## Reading cost, chunk {}, finished {}\n\n33 new input tokens before the first commit.\n\nRead most (opens and searches):\n\n- `a.py` 1\n"
|
|
self.assertEqual(history.read_text(), "# Reading cost\n\n" + "\n".join(figure.format(chunk.folder[:12], chunk.finished) for chunk in chunks))
|
|
self.assertTrue(self.run_tool(worktree).stdout.startswith(f"Appended 0 reading-cost figures from"))
|
|
self.assertEqual(history.read_text().count("## Reading cost"), 2)
|
|
reset = self.run_tool(worktree, "--reset")
|
|
self.assertEqual(reset.stdout, f"Appended 0 reading-cost figures from {self.home / '.claude' / 'loop-claude-transcripts' / 'proj'} and a reset to {history}.\n0 chunks measured since the last reset; the trend needs 10.\n")
|
|
self.assertRegex(history.read_text(), r"\n## Reading cost reset, \d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ\n$")
|
|
|
|
def test_counts_a_file_read_in_a_worktree_the_configs_branch_template_names(self):
|
|
config = self.home / ".claude-projects" / "proj" / "config.toml"
|
|
config.write_text(config.read_text() + 'branch = "feature/PAY-{id}"\n')
|
|
self.archive_chunk(1790000100, worktree=self.home / "code" / "proj-feature-PAY-7")
|
|
ran = self.run_tool(self.checkout)
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertIn("- `a.py` 1\n", (self.checkout / "docs" / "reading-cost.md").read_text())
|
|
|
|
def test_refuses_an_option_it_does_not_know(self):
|
|
ran = self.run_tool(self.checkout, "--rest")
|
|
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
|
|
|
|
def test_exits_1_outside_a_checkout(self):
|
|
ran = self.run_tool(self.home)
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("not a git repository", ran.stderr)
|
|
|
|
|
|
class Located(unittest.TestCase):
|
|
def test_lists_the_whole_checkout_from_a_folder_inside_it(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
root = pathlib.Path(temp.name).resolve()
|
|
(root / "sub").mkdir()
|
|
(root / "a.txt").write_text("a")
|
|
(root / "sub" / "b.txt").write_text("b")
|
|
for args in (("init", "--quiet"), ("add", ".")):
|
|
subprocess.run(["git", *args], check=True, cwd=root, env=isolated_env())
|
|
with contextlib.chdir(root / "sub"):
|
|
found = main.located(BRANCH)
|
|
self.assertEqual(found, Checkout(str(root), str(root), frozenset({"a.txt", "sub/b.txt"}), BRANCH))
|
|
|
|
def test_fails_in_a_checkout_whose_path_holds_a_newline(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
root = pathlib.Path(temp.name, "a\nb")
|
|
root.mkdir()
|
|
subprocess.run(["git", "init", "--quiet"], check=True, cwd=root, env=isolated_env())
|
|
with contextlib.chdir(root):
|
|
self.assertIsInstance(main.located(BRANCH), Failure)
|
|
|
|
|
|
class Update(unittest.TestCase):
|
|
def test_writes_over_the_version_it_read(self):
|
|
record = MemoryStore("# Reading cost\n", version=2)
|
|
archive = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
self.assertNotIsInstance(main.update(record, archive, Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h", "2026-11-14T00:00:00Z"), Failure)
|
|
self.assertEqual(record.overs, ["2"])
|
|
|
|
def test_fails_where_the_record_refuses_the_write(self):
|
|
class Unwritable(MemoryStore):
|
|
def write(self, text: str, over: str | None) -> Held | Failure:
|
|
return Failure("the wiki is down")
|
|
|
|
archive = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
updated = main.update(Unwritable("# Reading cost\n"), archive, Checkout("/c/p", "/c/p", frozenset(), BRANCH), "/h", "2026-11-14T00:00:00Z")
|
|
self.assertEqual(updated, Failure("the wiki is down"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|