Print a release's shell command patterns with command-patterns
This commit is contained in:
@@ -32,8 +32,11 @@
|
||||
!bin/chunk_preflight/*.py
|
||||
!bin/claude_code/
|
||||
!bin/claude_code/*.py
|
||||
!bin/command-patterns
|
||||
!bin/command_lines/
|
||||
!bin/command_lines/*.py
|
||||
!bin/command_patterns/
|
||||
!bin/command_patterns/*.py
|
||||
!bin/deploy-release
|
||||
!bin/deploy_release/
|
||||
!bin/deploy_release/*.py
|
||||
|
||||
@@ -95,6 +95,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t
|
||||
|
||||
- `chunk-pr open|ready|merge` opens a chunk's pull request as a draft, marks it ready and merges it, with `gh` on a GitHub origin and `tea` elsewhere; `chunk-pr --help` states the steps.
|
||||
- `chunk-preflight` finds the mechanical facts a chunk starts from, and with `--worktree` makes its worktree; `loop-claude` runs it with `--worktree` before it routes each chunk, and `chunk-preflight --help` lists the facts.
|
||||
- `command-patterns [<version>]` prints a table of the shell command patterns a release's archived chunks ran, such as `tea pr create`: per pattern its chunks, calls, command and result tokens and tool time, for an audit of the release; `command-patterns --help` states the measure.
|
||||
- `deploy-release [--cut] [<checkout>]` moves `~/.claude`, or the checkout named, to the newest release, as Update says; with `--cut`, run in the development clone, it first cuts the next release, as Release says.
|
||||
- `judge-replay <model>` replays loop-claude's judge on archived verdicts of the projects run on the model, and prints the lowest rung that, like each rung above it, returns them; `judge-replay --help` states the replay.
|
||||
- `larv-review <command>` holds larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's round count, the scoring run's trigger; the comprehension history's format, its latest places and its count of how often seats agree; and each reviewer's prompt header. `larv-review --help` states them.
|
||||
@@ -130,7 +131,7 @@ Then, in the copy: `uvx --with pytest==9.2.0 'mutmut>=3,<4' run && uvx --with py
|
||||
`bin/` is the import root: each launcher puts it on `sys.path`, so a module imports another by its path under `bin/`, such as `from claude_code.entries import parse_entry`. Each tool's package holds its tests. Beside the packages, `bin/`'s root holds only the launchers, the tools of one file with their tests, `test_env.py` and `failure.py`. `failure.py`'s `Failure`, which modules across `bin/` return for a failed step, belongs to no single package. `branch_chunks/`, `claude_code/`, `command_lines/`, `git_cli/`, `machine_load/`, `model_routing/`, `records/`, `release_plan/`, `review_rules/`, `rung_replay/` and `transcript_archive/` serve more than one tool.
|
||||
|
||||
- `branch_chunks/` cuts a default branch into chunks and finds, for each, the items it shipped and the later chunks that fixed its code, for `score_outcomes` and `routing_figures`. It imports `reading_cost`'s record path.
|
||||
- `command_lines/` reads a shell command: its simple commands, and a docker call's subcommand and memory limits, for `reading_cost`, `routing_figures`, `rule_hook` and `run_heavy`.
|
||||
- `command_lines/` reads a shell command: its simple commands, and a docker call's subcommand and memory limits, for `command_patterns`, `reading_cost`, `routing_figures`, `rule_hook` and `run_heavy`.
|
||||
- `chunk_pr` imports `deploy_release`'s forge calls, and `forge_of` as `deploy_release` does.
|
||||
- `deploy_release` imports `chunk_preflight`'s `forge_of`, which tells a GitHub origin from a Gitea one, and `release_plan`'s `release_standing`.
|
||||
- `git_cli/` runs git for every tool (`git_repo.py`), and reads the git calls in a shell command (`git_command.py`).
|
||||
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from command_patterns import main
|
||||
|
||||
sys.exit(main.main(sys.argv[1:]))
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Print a table of the shell command patterns a release's chunks ran, so an audit of the release reads one table.
|
||||
|
||||
usage: command-patterns [<version>]
|
||||
|
||||
Run it inside a project checkout or one of its worktrees. It reads loop-claude's archive of the
|
||||
project, as reading-cost does, and takes each chunk that finished in the release's window: after
|
||||
the tag before v<version> and up to v<version>'s own tag, by their tag dates. Without <version>, it
|
||||
takes the chunks finished since the newest v* tag, the release not yet cut. It reads the worker's
|
||||
transcript and each subagent's.
|
||||
|
||||
Each Bash call counts once, under the pattern of its first simple command other than `cd`: the
|
||||
tool's name, then up to two words that read as subcommands, as in `tea pr create`. A word that
|
||||
holds anything but lowercase letters, digits and dashes, such as an option or a path, ends the
|
||||
pattern. Per pattern the table lists the chunks that ran it, its calls, the tokens of its commands
|
||||
and of their results, estimated at four characters a token, and the seconds from each call to its
|
||||
result, most calls first.
|
||||
|
||||
Exit 1, saying why, on unknown arguments, a version no tag names, or when git cannot start or finds
|
||||
no checkout, or the archive cannot be read.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from command_patterns.tally import table, tally
|
||||
from failure import Failure
|
||||
from git_cli.git_repo import git
|
||||
from records.utc_stamp import seconds
|
||||
from transcript_archive.location import project_archive
|
||||
from transcript_archive.read import finished_chunks, read_threads
|
||||
|
||||
USAGE = "usage: command-patterns [<version>]"
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if argv in (["-h"], ["--help"]):
|
||||
print(__doc__.strip())
|
||||
return 0
|
||||
if len(argv) > 1 or argv[:1] and argv[0].startswith("-"):
|
||||
print(f"command-patterns: unknown arguments {' '.join(argv)}; {USAGE}", file=sys.stderr)
|
||||
return 1
|
||||
printed = report(argv[0] if argv else None)
|
||||
if isinstance(printed, Failure):
|
||||
print(f"command-patterns: {printed.reason}", file=sys.stderr)
|
||||
return 1
|
||||
print(printed)
|
||||
return 0
|
||||
|
||||
|
||||
def report(version: str | None) -> str | Failure:
|
||||
common = git("rev-parse", "--path-format=absolute", "--git-common-dir")
|
||||
if isinstance(common, Failure):
|
||||
return common
|
||||
# The common dir is the main checkout's .git, whichever worktree runs this, and the main checkout's path keys the archive.
|
||||
main_root = str(pathlib.Path(common.strip()).parent)
|
||||
bounds = window(version, main_root)
|
||||
if isinstance(bounds, Failure):
|
||||
return bounds
|
||||
after, until = bounds
|
||||
chunks = finished_chunks(project_archive(main_root, pathlib.Path.home(), os.environ))
|
||||
if isinstance(chunks, Failure):
|
||||
return chunks
|
||||
threads = []
|
||||
for chunk in chunks:
|
||||
if (after is None or seconds(chunk.finished) > after) and (until is None or seconds(chunk.finished) <= until):
|
||||
read = read_threads(chunk)
|
||||
if isinstance(read, Failure):
|
||||
return read
|
||||
threads.append(read)
|
||||
named = f"v{version}" if version else "the release not yet cut"
|
||||
return f"Shell command patterns of {len(threads)} chunk{'' if len(threads) == 1 else 's'} in {named}:\n\n{table(tally(threads))}"
|
||||
|
||||
|
||||
def window(version: str | None, cwd: str) -> tuple[int | None, int | None] | Failure:
|
||||
"""The release's window as tag dates in seconds since the epoch: after the tag before it, up to its own; None where it is open."""
|
||||
listed = git("-c", "versionsort.suffix=-", "tag", "--list", "v*", "--sort=-v:refname", "--format=%(refname:short) %(creatordate:unix)", cwd=cwd)
|
||||
if isinstance(listed, Failure):
|
||||
return listed
|
||||
tags = [(name, int(stamp)) for name, stamp in (line.split() for line in listed.splitlines())]
|
||||
if version is None:
|
||||
return (tags[0][1] if tags else None, None)
|
||||
names = [name for name, _ in tags]
|
||||
if f"v{version}" not in names:
|
||||
return Failure(f"no tag v{version}; name a cut release, or none for the release not yet cut")
|
||||
at = names.index(f"v{version}")
|
||||
return (tags[at + 1][1] if at + 1 < len(tags) else None, tags[at][1])
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Tally the shell calls of archived chunks by command pattern, such as `tea pr create`."""
|
||||
import dataclasses
|
||||
import datetime
|
||||
import posixpath
|
||||
import re
|
||||
|
||||
from claude_code.entries import AssistantTurn, parse_entry, ToolCall, ToolResults
|
||||
from command_lines.shell_command import segments
|
||||
|
||||
# A word that reads as a subcommand, such as `pr` in `tea pr create`; an option, a path or a number ends the pattern.
|
||||
SUBCOMMAND = re.compile(r"[a-z][a-z0-9-]*")
|
||||
# The words after the tool a pattern keeps at most.
|
||||
SUBCOMMANDS = 2
|
||||
# Tokens are estimated from characters, at about four characters a token.
|
||||
CHARACTERS_PER_TOKEN = 4
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Call:
|
||||
command: str
|
||||
result: str
|
||||
seconds: float
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Row:
|
||||
pattern: str
|
||||
chunks: int
|
||||
calls: int
|
||||
command_tokens: int
|
||||
result_tokens: int
|
||||
seconds: int
|
||||
|
||||
|
||||
def pattern(command: str) -> str:
|
||||
"""The call's first simple command other than `cd`: its tool's name, then up to two subcommand words."""
|
||||
commands = segments(command)
|
||||
for words in commands:
|
||||
if words[0] != "cd":
|
||||
kept = []
|
||||
for word in words[1:1 + SUBCOMMANDS]:
|
||||
if not SUBCOMMAND.fullmatch(word):
|
||||
break
|
||||
kept.append(word)
|
||||
return " ".join([posixpath.basename(words[0]), *kept])
|
||||
return "cd" if commands else "(unparsed)"
|
||||
|
||||
|
||||
def calls(thread: bytes) -> list[Call]:
|
||||
"""Each Bash call in a transcript, with its result's text and the seconds until the result; a call without a result has an empty one, taking no time."""
|
||||
asked: dict[str, tuple[str, str]] = {}
|
||||
answered: dict[str, tuple[str, str]] = {}
|
||||
for line in thread.splitlines():
|
||||
entry = parse_entry(line)
|
||||
if isinstance(entry, AssistantTurn):
|
||||
for part in entry.parts:
|
||||
if isinstance(part, ToolCall) and part.name == "Bash" and isinstance(command := part.input.get("command"), str):
|
||||
asked.setdefault(part.id, (command, entry.timestamp))
|
||||
elif isinstance(entry, ToolResults):
|
||||
# Claude Code logs some results twice; the first carries the time the tool ended.
|
||||
for found in entry.results:
|
||||
answered.setdefault(found.tool_use_id, (found.text, entry.timestamp))
|
||||
return [Call(command, answered.get(id, ("", ""))[0], elapsed(at, answered.get(id, ("", ""))[1])) for id, (command, at) in asked.items()]
|
||||
|
||||
|
||||
def elapsed(start: str, end: str) -> float:
|
||||
"""Seconds from one transcript timestamp to another; 0 where either is missing or unreadable."""
|
||||
try:
|
||||
return max(0.0, (datetime.datetime.fromisoformat(end) - datetime.datetime.fromisoformat(start)).total_seconds())
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def tally(chunks: list[list[bytes]]) -> list[Row]:
|
||||
"""One row per pattern over the chunks, each given as its threads, most calls first."""
|
||||
found: dict[str, list[tuple[int, Call]]] = {}
|
||||
for index, threads in enumerate(chunks):
|
||||
for thread in threads:
|
||||
for each in calls(thread):
|
||||
found.setdefault(pattern(each.command), []).append((index, each))
|
||||
rows = [Row(name, len({index for index, _ in made}), len(made), sum(len(each.command) for _, each in made) // CHARACTERS_PER_TOKEN, sum(len(each.result) for _, each in made) // CHARACTERS_PER_TOKEN, round(sum(each.seconds for _, each in made))) for name, made in found.items()]
|
||||
return sorted(rows, key=lambda row: (-row.calls, row.pattern))
|
||||
|
||||
|
||||
def table(rows: list[Row]) -> str:
|
||||
lines = ["| Pattern | Chunks | Calls | Command tokens | Result tokens | Tool seconds |", "|---|---|---|---|---|---|"]
|
||||
return "\n".join([*lines, *(f"| {row.pattern} | {row.chunks} | {row.calls} | {row.command_tokens} | {row.result_tokens} | {row.seconds} |" for row in rows)])
|
||||
@@ -0,0 +1,115 @@
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from command_patterns.tally import pattern, Row, tally
|
||||
from test_env import isolated_env
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "command-patterns"
|
||||
|
||||
|
||||
def call(id: str, command: str, at: str) -> str:
|
||||
return json.dumps({"type": "assistant", "timestamp": at, "message": {"id": f"m-{id}", "content": [{"type": "tool_use", "id": id, "name": "Bash", "input": {"command": command}}]}})
|
||||
|
||||
|
||||
def result(id: str, text: str, at: str) -> str:
|
||||
return json.dumps({"type": "user", "timestamp": at, "message": {"content": [{"type": "tool_result", "tool_use_id": id, "content": text}]}})
|
||||
|
||||
|
||||
def thread(*lines: str) -> bytes:
|
||||
return ("\n".join(lines) + "\n").encode()
|
||||
|
||||
|
||||
class Pattern(unittest.TestCase):
|
||||
def test_names_the_tool_and_its_subcommands_as_in_tea_pr_create(self):
|
||||
self.assertEqual(pattern("tea pr create --title t"), "tea pr create")
|
||||
|
||||
def test_skips_a_leading_cd(self):
|
||||
self.assertEqual(pattern("cd /x && git status --short"), "git status")
|
||||
|
||||
def test_stops_at_the_first_option_or_path(self):
|
||||
self.assertEqual(pattern("git -C /x status"), "git")
|
||||
self.assertEqual(pattern("~/.claude/bin/run-heavy ./ci.sh"), "run-heavy")
|
||||
|
||||
def test_takes_a_lone_cd_as_cd(self):
|
||||
self.assertEqual(pattern("cd /x"), "cd")
|
||||
|
||||
|
||||
class Tally(unittest.TestCase):
|
||||
def test_counts_each_patterns_chunks_calls_tokens_and_tool_time(self):
|
||||
first = [thread(call("a", "tea pr create --title t", "2026-11-01T10:00:00Z"), result("a", "x" * 400, "2026-11-01T10:00:03Z"), call("b", "git status", "2026-11-01T10:01:00Z"), result("b", "", "2026-11-01T10:01:01Z"))]
|
||||
second = [thread(call("c", "tea pr create --title u", "2026-11-02T10:00:00Z"), result("c", "y" * 40, "2026-11-02T10:00:02Z")), thread(call("d", "tea pr create", "2026-11-02T11:00:00Z"))]
|
||||
rows = tally([first, second])
|
||||
self.assertEqual(rows[0], Row("tea pr create", chunks=2, calls=3, command_tokens=(23 + 23 + 13) // 4, result_tokens=(400 + 40) // 4, seconds=5))
|
||||
self.assertEqual(rows[1], Row("git status", chunks=1, calls=1, command_tokens=10 // 4, result_tokens=0, seconds=1))
|
||||
|
||||
def test_counts_a_result_logged_twice_once(self):
|
||||
rows = tally([[thread(call("a", "ls", "2026-11-01T10:00:00Z"), result("a", "abcd", "2026-11-01T10:00:01Z"), result("a", "abcd", "2026-11-01T10:00:09Z"))]])
|
||||
self.assertEqual(rows, [Row("ls", chunks=1, calls=1, command_tokens=0, result_tokens=1, seconds=1)])
|
||||
|
||||
|
||||
class Main(unittest.TestCase):
|
||||
"""A checkout tagged v0.1.0 at 2026-11-01 and v0.2.0 at 2026-11-05, and one archived chunk finished before, between and after them."""
|
||||
|
||||
def setUp(self):
|
||||
self.home = pathlib.Path(os.path.realpath(tempfile.mkdtemp()))
|
||||
self.addCleanup(subprocess.run, ["rm", "-rf", str(self.home)])
|
||||
self.checkout = self.home / "code" / "project"
|
||||
self.checkout.mkdir(parents=True)
|
||||
self.env = {**isolated_env(), "HOME": str(self.home), "GIT_CONFIG_NOSYSTEM": "1", "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||
self.git("init", "-q", "-b", "main")
|
||||
self.git("commit", "-q", "--allow-empty", "-m", "Start")
|
||||
self.tag("v0.1.0", "2026-11-01T00:00:00Z")
|
||||
self.tag("v0.2.0", "2026-11-05T00:00:00Z")
|
||||
archive = self.home / ".claude" / "loop-claude-transcripts" / "project"
|
||||
for name, finished, command in (("before", "2026-10-30T00:00:00Z", "make old"), ("between", "2026-11-03T00:00:00Z", "tea pr create"), ("after", "2026-11-07T00:00:00Z", "gh pr view")):
|
||||
folder = archive / name
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "worker.jsonl").write_bytes(thread(call(name, command, finished), result(name, "ok", finished)))
|
||||
(folder / "worker.outcomes.md").write_text(f"Task: t\n\n{finished} Verdict: finished\n")
|
||||
|
||||
def git(self, *args: str, at: str | None = None) -> None:
|
||||
subprocess.run(["git", *args], check=True, cwd=self.checkout, env={**self.env, **({"GIT_COMMITTER_DATE": at} if at else {})}, capture_output=True)
|
||||
|
||||
def tag(self, name: str, at: str) -> None:
|
||||
self.git("tag", "-a", "-m", name, name, at=at)
|
||||
|
||||
def run_tool(self, *args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=self.checkout, env=self.env, text=True)
|
||||
|
||||
def test_reads_the_chunks_finished_since_the_newest_tag_by_default(self):
|
||||
ran = self.run_tool()
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
self.assertIn("| gh pr view | 1 | 1 |", ran.stdout)
|
||||
self.assertNotIn("tea pr create", ran.stdout)
|
||||
self.assertIn("1 chunk", ran.stdout)
|
||||
|
||||
def test_reads_the_chunks_finished_between_the_tag_before_a_release_and_its_own(self):
|
||||
ran = self.run_tool("0.2.0")
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
self.assertIn("| tea pr create | 1 | 1 |", ran.stdout)
|
||||
self.assertNotIn("make old", ran.stdout)
|
||||
self.assertNotIn("gh pr view", ran.stdout)
|
||||
|
||||
def test_reads_every_chunk_before_the_first_tag_for_the_first_release(self):
|
||||
ran = self.run_tool("0.1.0")
|
||||
self.assertIn("| make old | 1 | 1 |", ran.stdout)
|
||||
self.assertNotIn("tea pr create", ran.stdout)
|
||||
|
||||
def test_refuses_a_release_no_tag_names(self):
|
||||
ran = self.run_tool("0.9.0")
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertIn("v0.9.0", ran.stderr)
|
||||
|
||||
def test_refuses_more_than_one_argument(self):
|
||||
self.assertEqual(self.run_tool("0.1.0", "0.2.0").returncode, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user