851 lines
56 KiB
Python
851 lines
56 KiB
Python
import datetime
|
|
import itertools
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import resource
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from rule_hook.repeat_read import Asked, modified, read_breaches, result_lines
|
|
from rule_hook.rules import blocks, Breach, check, command_breaches, file_breaches, overrides
|
|
from rule_hook.main import breaches_of
|
|
from rule_hook.call import Call, parsed_call
|
|
from git_cli.git_command import commit_message, CommitCall, commits
|
|
from release_plan.test_ranking import DECISIONS, README, TODO
|
|
from rule_hook.git import guarded_git
|
|
from test_env import isolated_env
|
|
from rule_hook.override_log import LogLine, parsed_entry, this_chunks
|
|
from rule_hook.reviewer_rung import reviewer_breaches, skill_breaches
|
|
from command_lines.shell_command import segments, simple_commands
|
|
|
|
HOOK = pathlib.Path(__file__).resolve().parents[1] / "rule-hook"
|
|
# Built from parts, so this file carries no attribution of its own.
|
|
TRAILER = "Co-Authored-" + "By: Claude <noreply@anthropic.com>"
|
|
|
|
|
|
def rules(breaches: list[Breach]) -> list[str]:
|
|
return [breach.rule for breach in breaches]
|
|
|
|
|
|
def committed(command: str, cwd: str = "/r") -> list[CommitCall]:
|
|
return commits(simple_commands(command, cwd, "/h"), "/h")
|
|
|
|
|
|
def checked(command: str) -> list[str]:
|
|
return rules(command_breaches(command, simple_commands(command, "/r", "/h"), "/h"))
|
|
|
|
|
|
def hook(call: dict, cwd: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess:
|
|
return subprocess.run([str(HOOK)], capture_output=True, cwd=cwd, env=isolated_env() | (env or {}), input=json.dumps(call), text=True, timeout=10)
|
|
|
|
|
|
def git(cwd: str, *args: str) -> str:
|
|
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, check=True, cwd=cwd, env=isolated_env(), text=True).stdout
|
|
|
|
|
|
class Commits(unittest.TestCase):
|
|
def test_cd_and_dash_capital_c_move_the_commit(self):
|
|
self.assertEqual(committed("cd a && git -C ~/b commit -am 'x'"), [CommitCall("/h/b", ["x"], False, False)])
|
|
self.assertEqual(committed("cd a && git -c x=y -C b commit -m x")[0].directory, "/r/a/b")
|
|
|
|
def test_each_message_spelling_is_read(self):
|
|
self.assertEqual(commit_message(["--message=a", "-m", "b=c", "--message", "c", "-mz", "-am", "d"]), (["a", "b=c", "c", "z", "d"], False, False))
|
|
|
|
def test_an_amend_is_seen(self):
|
|
self.assertEqual([commit.amends for commit in committed("git commit --amend -m x && git commit -m y")], [True, False])
|
|
self.assertEqual(commit_message(["--", "--amend"]), ([], False, False), "after --, --amend names a path")
|
|
|
|
def test_a_file_message_is_seen(self):
|
|
self.assertEqual(commit_message(["-F", "-"]), ([], True, False))
|
|
self.assertEqual(commit_message(["--file=x"]), ([], True, False))
|
|
|
|
def test_other_git_commands_are_no_commit(self):
|
|
self.assertEqual(committed("git log --grep commit; echo git commit"), [])
|
|
|
|
def test_a_line_break_or_a_comment_separates_commands(self):
|
|
self.assertEqual(committed("cd a # go there\nGIT_EDITOR=true git commit -m x"), [CommitCall("/r/a", ["x"], False, False)])
|
|
|
|
def test_a_commit_after_a_heredoc_or_a_blank_line_is_seen(self):
|
|
self.assertEqual(len(committed("cat <<EOF > notes\nhello\nEOF\ngit commit -m a")), 1)
|
|
self.assertEqual(len(committed("git status &&\n\ngit commit -m a")), 1)
|
|
|
|
def test_an_arithmetic_shift_is_no_heredoc(self):
|
|
self.assertEqual(len(committed("echo $((1<<2))\ngit commit -m a\n2")), 1)
|
|
|
|
def test_a_line_that_does_not_parse_hides_only_itself(self):
|
|
self.assertEqual(len(committed("echo 'open\ngit commit -m a")), 1)
|
|
|
|
def test_a_redirect_keeps_its_command_s_words_together(self):
|
|
self.assertEqual(segments("echo hi > f arg"), [["echo", "hi", "arg"]])
|
|
for redirect in [">", ">>", "<", "<<<", "<>", ">&", "<&", ">|", "&>", "&>>"]:
|
|
with self.subTest(redirect):
|
|
self.assertEqual(committed(f"git {redirect} /tmp/x commit -m hi"), [CommitCall("/r", ["hi"], False, False)])
|
|
self.assertEqual(committed("git commit >&2 -m hi 2>&1"), [CommitCall("/r", ["hi"], False, False)])
|
|
self.assertEqual(committed("> out git commit -m hi"), [CommitCall("/r", ["hi"], False, False)])
|
|
|
|
def test_a_separator_glued_to_a_redirect_still_ends_the_command(self):
|
|
self.assertEqual(segments("echo a > f; echo b | cat &> g\nls"), [["echo", "a"], ["echo", "b"], ["cat"], ["ls"]])
|
|
for command in ["echo a &</dev/null git commit -m hi", "echo a;>f git commit -m hi"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(committed(command), [CommitCall("/r", ["hi"], False, False)])
|
|
|
|
|
|
class CommandBreaches(unittest.TestCase):
|
|
def test_one_line_passes(self):
|
|
self.assertEqual(checked("git add . && git commit -m 'ABC-1 - Fix #2'"), [])
|
|
|
|
def test_a_heredoc_a_second_message_or_a_file_breaks_commit_one_line(self):
|
|
for command in ["git commit -m \"$(cat <<'EOF'\nIt's one line\nEOF\n)\"", "git commit -m a -m b", "git commit -F - <<'EOF'\nTitle\n\nIt's the body\nEOF\necho done", "git commit --author '' -m a -m b"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["commit-one-line"])
|
|
|
|
def test_a_round_title_without_its_counts_breaks_round_title_counts(self):
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: fix the parser'"), ["round-title-counts"])
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: 1 🔴, 2 🟡; fix the parser'"), ["round-title-counts"])
|
|
self.assertEqual(checked("git commit --allow-empty -m 'Answer prose round 2: 0 🔴, 0 🟡 (0 misread)'"), [])
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: 1 🔴, 2 🟡 (1 misread); fix the parser'"), [])
|
|
self.assertEqual(checked("git commit -m 'Answer the question in the README'"), [])
|
|
|
|
def test_a_docker_build_or_a_mutation_run_outside_run_heavy_breaks_run_heavy(self):
|
|
for command in ["docker build -t x .", "cd /w && docker buildx build .", "docker compose build", "docker compose -f x.yml --verbose build app", "uvx --with pytest 'mutmut>=3,<4' run", "cargo mutants --in-diff d", "npx stryker run", "vendor/bin/infection --git-diff-lines", "docker compose run --rm app mutmut run", "docker run --rm -m 4g --memory-swap 4g -v /w:/w rust:1.90.0 cargo mutants", "docker compose exec app npx stryker run", "docker run -h box -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "nice -n 19 docker build .", "env DOCKER_BUILDKIT=1 docker build .", "time -p docker compose build", "docker --tlscacert ca.pem build .", "docker --tlscert c.pem --tlskey k.pem build .", "docker buildx --builder b build .", "docker container run -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "uvx mutmut --verbose run"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["run-heavy"])
|
|
|
|
def test_a_heavy_command_under_run_heavy_or_a_light_docker_or_mutmut_call_passes(self):
|
|
for command in ["~/.claude/bin/run-heavy docker build -t x .", "/home/me/.claude/bin/run-heavy uvx mutmut run", "nice -n 19 ~/.claude/bin/run-heavy docker build .", "env X=1 run-heavy docker compose run app mutmut run", "docker images", "docker compose run --rm app go vet", "docker compose run app make build", "uvx 'mutmut>=3,<4' results", "echo docker build", "npx prettier --check infection.json5", "echo mutmut run", "env", "docker build --help", "cargo mutants --list", "infection --version", "uvx mutmut a b run"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), [])
|
|
|
|
def test_a_docker_run_or_create_without_a_memory_limit_breaks_docker_run_memory(self):
|
|
for command in ["docker run --rm node:24.22.1-alpine3.24 node x.js", "cd /w && docker run -v /w:/w alpine:3.22.2 true", "docker create alpine:3.22.2", "docker container run alpine:3.22.2 true", "docker container create alpine:3.22.2", "nice -n 19 docker run alpine:3.22.2 true", "docker -H unix:///d.sock run alpine:3.22.2 true", "docker run --memory-swap 4g alpine:3.22.2 true", "docker run -m 4g alpine:3.22.2 true", "docker run --memory=4g alpine:3.22.2 sh -c 'x --memory-swap 4g'", "docker run -m 4g alpine:3.22.2 tool --memory-swap", "docker run -m 4g --memory-swap -1 alpine:3.22.2 true", "docker run -m 4g --memory-swap=0 alpine:3.22.2 true", "docker run --rm python:3.13.1 python -m pytest", "docker run alpine:3.22.2 git commit -m x", "timeout 60 docker run alpine:3.22.2 true", "docker run node:24.22.1-alpine3.24 node --help", "docker run alpine:3.22.2 --help", "docker run -m 4g alpine:3.22.2 sleep 60", "~/.claude/bin/run-heavy /usr/bin/docker run alpine:3.22.2 true", "docker run --memory-swap 4g alpine:3.22.2 grep -m 1 x", "run-heavy env PATH=/usr/bin docker run alpine:3.22.2 true", "run-heavy sudo docker run alpine:3.22.2 true", "docker run alpine:3.22.2 git tag -m 1.2.3", "run-heavy env -i docker run alpine:3.22.2 true", "run-heavy env -u PATH docker run alpine:3.22.2 true", "run-heavy env --unset=PATH docker run alpine:3.22.2 true", "run-heavy env --ignore-environment docker run alpine:3.22.2 true", "run-heavy doas docker run alpine:3.22.2 true"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["docker-run-memory"])
|
|
|
|
def test_a_docker_run_with_a_memory_limit_or_another_docker_call_passes_docker_run_memory(self):
|
|
for command in ["docker run -m 4g --memory-swap 4g alpine:3.22.2 true", "docker run -m4g --memory-swap=4g alpine:3.22.2 true", "docker run --memory 512M --memory-swap 1g alpine:3.22.2 true", "docker run -m 1073741824 --memory-swap 2g alpine:3.22.2 true", "timeout 60 docker run -m 4g --memory-swap 4g alpine:3.22.2 true", "docker run --rm --memory=1.5GiB --memory-swap=1.5GiB alpine:3.22.2 true", "docker container create -m 1g --memory-swap 1g alpine:3.22.2", "docker run --help", "docker container run --help", "docker --context run run --help", "docker --help", "docker ps", "docker images", "docker container ls", "docker compose run --rm app go vet", "docker exec box true", "echo docker run alpine:3.22.2", "~/.claude/bin/run-heavy docker run alpine:3.22.2 true"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), [])
|
|
|
|
def test_an_attribution_breaks_only_in_a_git_gh_or_tea_command(self):
|
|
self.assertEqual(checked(f"tea pr create --description '{TRAILER}'"), ["no-ai-attribution"])
|
|
self.assertEqual(checked(f"grep -rn '{TRAILER}' ."), [])
|
|
|
|
def test_a_commit_breach_names_the_commit_s_checkout(self):
|
|
self.assertEqual(command_breaches("cd w && git commit -m a -m b", simple_commands("cd w && git commit -m a -m b", "/r", "/h"), "/h")[0].directory, "/r/w")
|
|
|
|
|
|
class FileBreaches(unittest.TestCase):
|
|
def test_an_image_needs_a_patch_version_a_date_or_a_digest(self):
|
|
added = "FROM node:24 AS build\nFROM node:24.18.1-bookworm-slim\nFROM alpine@sha256:ab\nFROM debian:bookworm-20240926-slim\nFROM build\nFROM scratch\nFROM ${BASE}\nFROM python\nCOPY --from=build /a /a\nCOPY --from=0 /b /b\nCOPY --chown=1 --from=alpine:3 /c /c\n"
|
|
self.assertEqual([breach.detail for breach in file_breaches("/r/Dockerfile", "", added, added)], ["/r/Dockerfile: node:24", "/r/Dockerfile: python", "/r/Dockerfile: alpine:3"])
|
|
|
|
def test_a_stage_named_outside_the_edit_is_no_image(self):
|
|
self.assertEqual(file_breaches("app.dockerfile", "FROM node:24.1.0 AS base\n", "FROM base\n", "FROM node:24.1.0 AS base\nFROM base\n"), [])
|
|
|
|
def test_a_workflow_checks_its_images_and_actions(self):
|
|
added = " container: node:24\n image: mysql:8.4\n - image: 'mysql:8.4.7'\n - uses: actions/checkout@v4\n - uses: a/b@0123456789abcdef0123456789abcdef01234567\n - uses: ./local\n"
|
|
self.assertEqual(rules(file_breaches("/r/.gitea/workflows/ci.yml", "", added, added)), ["image-pin", "image-pin", "action-pin"])
|
|
|
|
def test_a_compose_file_checks_its_images(self):
|
|
self.assertEqual(rules(file_breaches("compose.yaml", "", " image: redis\n", " image: redis\n")), ["image-pin"])
|
|
|
|
def test_other_yaml_is_not_checked(self):
|
|
added = "image: /assets/og.png\nimage: { type: string }\nuses: a/b@main\n"
|
|
self.assertEqual(file_breaches("_config.yml", "", added, added), [])
|
|
|
|
def test_an_ignore_file_sorts_each_block_split_by_blank_lines_and_comments(self):
|
|
self.assertEqual(blocks("b\na\n\n# c\nd\n# e\nf\n"), [["b", "a"], ["d"], ["f"]])
|
|
self.assertEqual(rules(file_breaches(".dockerignore", "", "x", "b\na\n")), ["ignorefile-sorted"])
|
|
self.assertEqual(file_breaches(".gitignore", "", "x", "*\n\n!a\n!b\n"), [])
|
|
|
|
def test_a_block_unsorted_before_the_edit_is_left_alone(self):
|
|
self.assertEqual(file_breaches(".gitignore", "node_modules\n.env\n\ndist\n", "out", "node_modules\n.env\n\ndist\nout\n"), [])
|
|
|
|
def test_an_attribution_in_any_file_breaks(self):
|
|
self.assertEqual(rules(file_breaches("notes.md", "", TRAILER, TRAILER)), ["no-ai-attribution"])
|
|
|
|
|
|
class Overrides(unittest.TestCase):
|
|
def test_a_marker_names_a_known_rule_and_a_reason(self):
|
|
self.assertEqual(overrides("x # rule-override image-pin: vendor ships floating tags only\n# rule-override nope: x\n# rule-override action-pin:"), ({"image-pin": "vendor ships floating tags only"}, ["rule-override nope: x", "rule-override action-pin: "]))
|
|
|
|
def test_a_marker_for_a_rule_that_takes_no_override_is_neither_granted_nor_malformed(self):
|
|
self.assertEqual(overrides("# rule-override reviewer-rung: trust me"), ({}, []))
|
|
self.assertEqual(overrides("# rule-override reviewer-rung: trust me\n# rule-override image-pin: a # rule-override nope: b"), ({"image-pin": "a"}, ["rule-override nope: b"]))
|
|
|
|
def test_two_markers_on_a_line_are_read_apart(self):
|
|
self.assertEqual(overrides("git x # rule-override image-pin: a # rule-override nope: b")[1], ["rule-override nope: b"])
|
|
|
|
def test_markdown_takes_an_html_comment(self):
|
|
self.assertEqual(overrides("<!-- rule-override no-ai-attribution: quotes the banned line -->")[0], {"no-ai-attribution": "quotes the banned line"})
|
|
|
|
def test_a_blocked_call_overrides_nothing_yet(self):
|
|
breaches = [Breach("image-pin", "x", "/r"), Breach("action-pin", "y", "/r")]
|
|
blocked_call = check(breaches, "# rule-override image-pin: a", False, False)
|
|
self.assertTrue(blocked_call.blocked)
|
|
passed = check(breaches[:1], "# rule-override image-pin: a", False, False)
|
|
self.assertEqual((passed.blocked, passed.overridden), (False, [(breaches[0], "a")]))
|
|
|
|
def test_a_markdown_file_is_told_the_html_comment(self):
|
|
self.assertIn("`<!-- rule-override no-ai-attribution: <reason> -->`", check([Breach("no-ai-attribution", "x", "/r")], "", True, False).messages[0])
|
|
|
|
|
|
class OverrideLog(unittest.TestCase):
|
|
def test_an_entry_reads_back_as_written(self):
|
|
entry = LogLine("2026-10-13T00:00:00Z", "chunk", "c1", "image-pin", "the vendor\ttags no patch", "Dockerfile")
|
|
self.assertEqual(parsed_entry(entry.line()), LogLine("2026-10-13T00:00:00Z", "chunk", "c1", "image-pin", "the vendor tags no patch", "Dockerfile"))
|
|
self.assertIsNone(parsed_entry("a\tb"))
|
|
|
|
def test_an_override_is_the_chunk_s_by_its_reflog_and_name(self):
|
|
on_default = {"base"}.__contains__
|
|
cases = [("chunk", "c1", True), ("other", "c1", True), ("other", "c9", False), ("chunk", "base", True), ("other", "base", False), ("other", "", True)]
|
|
for branch, commit, expected in cases:
|
|
with self.subTest(branch=branch, commit=commit):
|
|
self.assertEqual(this_chunks(LogLine("t", branch, commit, "image-pin", "r", "w"), {"base", "c1"}, "chunk", on_default), expected)
|
|
|
|
|
|
class ReviewerRung(unittest.TestCase):
|
|
# A worker routed to Sonnet 5.5 at high, on a project whose rung is Opus 5.5 at medium, with its reviewers on the project's rung.
|
|
ENVIRON = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-sonnet-5-5 at high effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"}
|
|
# The same project, with a worker, and so its reviewers, routed above it to Fable 5.1 at max.
|
|
ABOVE = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-1 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"}
|
|
|
|
def refused(self, given: dict, environ: dict[str, str] = ENVIRON) -> list[str]:
|
|
return rules(reviewer_breaches(Call("Agent", given, "/r"), environ))
|
|
|
|
def test_a_branch_changing_only_markdown_reviews_on_the_projects_rung_whatever_the_e_plus_i(self):
|
|
repo = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", repo])
|
|
git(repo, "init", "-q", "-b", "main")
|
|
git(repo, "commit", "-q", "--allow-empty", "-m", "base")
|
|
git(repo, "checkout", "-q", "-b", "chunk")
|
|
pathlib.Path(repo, "docs").mkdir()
|
|
pathlib.Path(repo, "README.md").write_text("Read me.\n")
|
|
pathlib.Path(repo, "docs", "decisions.md").write_text("Decided.\n")
|
|
git(repo, "add", ".")
|
|
git(repo, "commit", "-q", "-m", "prose")
|
|
high = self.ENVIRON | {"LOOP_CLAUDE_ESCAPE_AND_IMPACT": "7"}
|
|
on_project = {"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), [])
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", {**on_project, "model": "sonnet", "effort": "high"}, repo), high)), ["reviewer-rung"])
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), self.ABOVE)), [], "a worker that climbed doesn't lift a Markdown-only review either")
|
|
for instructions in ("CLAUDE.md", "AGENTS.md", "skills/review/SKILL.md", "agents/reviewer.md", "rules/todo.md", "principles/tests.md", "workflow-bench/run-prompt.md"):
|
|
with self.subTest(instructions=instructions):
|
|
path = pathlib.Path(repo, instructions)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("Do this.\n")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"], "Markdown an agent follows is config")
|
|
path.unlink()
|
|
pathlib.Path(repo, "gate.yml").write_text("on: push\n")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"], "an uncommitted change to another file counts")
|
|
git(repo, "add", ".")
|
|
git(repo, "commit", "-q", "-m", "config")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"])
|
|
|
|
def test_a_reviewer_below_the_project_rung_is_refused_naming_the_arguments_to_pass(self):
|
|
breaches = reviewer_breaches(Call("Agent", {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review."}, "/r"), self.ENVIRON)
|
|
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
|
self.assertIn("stability-reviewer", breaches[0].detail)
|
|
self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail)
|
|
self.assertIn("model opus, effort medium", breaches[0].detail)
|
|
|
|
def test_a_reviewer_on_the_project_rung_or_a_rerun_above_it_passes(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus", "effort": "medium"}), [])
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus", "effort": "high"}), [])
|
|
|
|
def test_an_agent_calls_prompt_is_the_text_scanned_for_a_marker(self):
|
|
call = Call("Agent", {"subagent_type": "prose-reviewer", "prompt": "Review. # rule-override reviewer-rung: trust me"}, "/r")
|
|
self.assertEqual(breaches_of(call, "/h", {})[0], "Review. # rule-override reviewer-rung: trust me")
|
|
|
|
def test_a_call_naming_no_model_runs_on_the_workers_rung(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "prompt": "Review."}), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "effort": "medium"}), ["reviewer-rung"])
|
|
|
|
def test_a_model_named_without_an_effort_runs_on_its_cheapest_rung(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus"}), [])
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "sonnet"}), ["reviewer-rung"])
|
|
|
|
def test_an_effort_off_the_ladder_for_its_model_is_refused(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus", "effort": "low"}), ["reviewer-rung"])
|
|
|
|
def test_a_reviewer_is_held_to_the_rung_its_e_plus_i_picks_where_that_is_higher(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, self.ABOVE), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "fable", "effort": "max"}, self.ABOVE), [])
|
|
lifted = self.ENVIRON | {"LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, lifted), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "opus", "effort": "medium"}, lifted), [])
|
|
|
|
def test_holds_a_reviewer_to_the_higher_of_the_workers_and_the_projects_rung_where_loop_claude_handed_no_e_plus_i(self):
|
|
without = {name: rung for name, rung in self.ABOVE.items() if name != "LOOP_CLAUDE_ESCAPE_AND_IMPACT"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, without), ["reviewer-rung"])
|
|
below = {name: rung for name, rung in self.ENVIRON.items() if name != "LOOP_CLAUDE_ESCAPE_AND_IMPACT"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high"}, below), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "sonnet", "effort": "high"}, without), ["reviewer-rung"])
|
|
|
|
def test_a_seat_or_a_security_launch_is_held_to_the_project_rung_alone(self):
|
|
seat = {"prompt": "You are one seat of a comprehension panel. Tools: Read, Grep, Glob.", "model": "opus", "effort": "medium"}
|
|
self.assertEqual(self.refused(seat, self.ABOVE), [])
|
|
self.assertEqual(self.refused({"prompt": "You are the security pass for this chunk. Invoke /security-review over the branch diff.", "model": "sonnet", "effort": "high"}), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are the security pass for this chunk. Run /claude-security scan this branch's changes at high.", "model": "opus", "effort": "medium"}), [])
|
|
|
|
def test_a_seat_named_below_the_project_rung_is_refused(self):
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "sonnet", "effort": "high"}), ["reviewer-rung"])
|
|
|
|
def test_any_other_subagent_keeps_its_own_routing(self):
|
|
self.assertEqual(self.refused({"subagent_type": "Explore", "model": "haiku", "effort": "low", "prompt": "Find the callers of run_worker."}), [])
|
|
self.assertEqual(self.refused({"prompt": "Note that security-review runs in CI.", "model": "haiku", "effort": "low"}), [])
|
|
self.assertEqual(self.refused({"prompt": "Find where /security-review is named.", "model": "haiku", "effort": "low"}), [], "a mention without the opening sentence is no launch")
|
|
self.assertEqual(self.refused({"prompt": "Quote the rule: 'You are the security pass for this chunk.' Then stop.", "model": "haiku", "effort": "low"}), [], "a quote midway is no launch")
|
|
|
|
def test_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "haiku", "effort": "low"}, {}), [])
|
|
|
|
def skill_refused(self, given: dict, environ: dict[str, str] = ENVIRON, subagent_prompt: str | None = None) -> list[Breach]:
|
|
transcript = ""
|
|
if subagent_prompt is not None:
|
|
transcript = os.path.join(tempfile.mkdtemp(), "agent-a1.jsonl")
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", os.path.dirname(transcript)])
|
|
pathlib.Path(transcript).write_text(json.dumps({"type": "user", "message": {"role": "user", "content": subagent_prompt}}) + "\n")
|
|
return skill_breaches(Call("Skill", given, "/r", "general-purpose" if subagent_prompt is not None else "", transcript, subagent_prompt is not None), environ)
|
|
|
|
def test_a_security_skill_from_the_sessions_own_agent_is_refused_below_the_projects_rung(self):
|
|
for skill in ("/security-review", "claude-security", "claude-security:claude-security", "security-review"):
|
|
with self.subTest(skill=skill):
|
|
breaches = self.skill_refused({"skill": skill})
|
|
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
|
self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail)
|
|
self.assertIn("model opus, effort medium", breaches[0].detail)
|
|
self.assertIn("You are the security pass for this chunk.", breaches[0].detail)
|
|
self.assertEqual(rules(self.skill_refused({"skill": skill}, self.ABOVE)), [])
|
|
|
|
def test_a_security_skill_runs_on_its_callers_rung_whatever_model_its_input_names(self):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review", "model": "opus", "effort": "medium"})), ["reviewer-rung"])
|
|
|
|
def test_a_security_skill_passes_only_in_a_subagent_launched_as_the_security_pass(self):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ENVIRON, "You are the security pass for this chunk. Invoke /security-review.")), [])
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ABOVE, "Invoke /security-review.")), ["reviewer-rung"], "a subagent's rung is unknown, whatever the worker's")
|
|
unreadable = skill_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose", "/nonexistent/agent-a1.jsonl", True), self.ENVIRON)
|
|
self.assertEqual(rules(unreadable), ["reviewer-rung"])
|
|
|
|
def test_any_other_skill_or_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "simplify"})), [])
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, {})), [])
|
|
|
|
def test_a_session_started_with_an_agent_is_no_subagent(self):
|
|
hook_input = {"tool_name": "Skill", "tool_input": {"skill": "security-review"}, "cwd": "/r", "agent_type": "claude-security:claude-security", "transcript_path": "/t/s.jsonl"}
|
|
self.assertFalse(parsed_call(hook_input).subagent)
|
|
self.assertTrue(parsed_call({**hook_input, "agent_id": "a1"}).subagent)
|
|
|
|
def test_a_skill_calls_args_are_the_text_scanned_for_a_marker(self):
|
|
found = breaches_of(Call("Skill", {"skill": "security-review", "args": "the branch # rule-override reviewer-rung: trust me"}, "/r"), "/h", {})
|
|
self.assertEqual(found[:2], ("the branch # rule-override reviewer-rung: trust me", "security-review"))
|
|
|
|
def test_settings_run_the_hook_before_each_skill_call(self):
|
|
settings = json.loads(pathlib.Path(__file__).resolve().parents[2].joinpath("settings.json").read_text())
|
|
matchers = [entry["matcher"].split("|") for entry in settings["hooks"]["PreToolUse"]]
|
|
self.assertTrue(any("Skill" in tools for tools in matchers))
|
|
|
|
|
|
class ReviewerReport(unittest.TestCase):
|
|
def breached(self, tool: str, path: str, agent: str = "prose-reviewer", cwd: str = "/r") -> list[str]:
|
|
given = {"file_path": path, "content": "BLOCK\n"} if tool == "Write" else {"file_path": path, "old_string": "a", "new_string": "b"}
|
|
found = breaches_of(Call(tool, given, cwd, agent), "/h", {})
|
|
return rules(found[2]) if found else []
|
|
|
|
def test_a_reviewer_writes_its_report_into_the_reports_folder(self):
|
|
self.assertEqual(self.breached("Write", "/h/.local/state/larv-review/.claude-item-73/prose-round-1.md"), [])
|
|
|
|
def test_a_relative_path_resolves_against_the_calls_cwd(self):
|
|
self.assertEqual(self.breached("Write", "prose-round-1.md", cwd="/h/.local/state/larv-review/x"), [])
|
|
|
|
def test_a_symlink_in_the_reports_folder_is_followed_to_where_it_points(self):
|
|
home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
(home / ".local" / "state" / "larv-review").mkdir(parents=True)
|
|
(home / ".local" / "state" / "larv-review" / "out").symlink_to(home)
|
|
found = breaches_of(Call("Write", {"file_path": f"{home}/.local/state/larv-review/out/a.md", "content": "x"}, "/r", "prose-reviewer"), str(home), {})
|
|
self.assertEqual(rules(found[2]), ["reviewer-report"])
|
|
|
|
def test_a_reviewer_writing_anywhere_else_is_refused(self):
|
|
for path in ["/r/README.md", "/h/.local/state/larv-review", "/h/.local/state/larv-review-x/a.md", "/h/.local/state/larv-review/../a.md", "/tmp/larv-review/a.md"]:
|
|
with self.subTest(path):
|
|
self.assertEqual(self.breached("Write", path), ["reviewer-report"])
|
|
|
|
def test_a_reviewer_edits_no_file_even_in_the_reports_folder(self):
|
|
self.assertEqual(self.breached("Edit", "/h/.local/state/larv-review/x/prose-round-1.md"), ["reviewer-report"])
|
|
|
|
def test_a_reviewers_multiedit_is_refused(self):
|
|
self.assertEqual(self.breached("MultiEdit", "/r/a.py", "stability-reviewer"), ["reviewer-report"])
|
|
|
|
def test_the_worker_and_other_subagents_write_anywhere(self):
|
|
self.assertEqual(self.breached("Write", "/r/README.md", ""), [])
|
|
self.assertEqual(self.breached("Write", "/r/README.md", "general-purpose"), [])
|
|
|
|
|
|
class RepeatRead(unittest.TestCase):
|
|
LINES = [f"line {n}" for n in range(1, 31)]
|
|
# The file is written at WRITTEN and each entry stamped READ_AT, unless a test changes the file at CHANGED and stamps a later entry AFTER_THE_CHANGE.
|
|
WRITTEN = datetime.datetime(2026, 5, 1, tzinfo=datetime.UTC).timestamp()
|
|
READ_AT = "2026-06-01T00:00:00.000Z"
|
|
CHANGED = datetime.datetime(2026, 7, 1, tzinfo=datetime.UTC).timestamp()
|
|
AFTER_THE_CHANGE = "2026-07-01T00:00:01.000Z"
|
|
|
|
def setUp(self):
|
|
self.folder = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.folder])
|
|
self.file = f"{self.folder}/f.txt"
|
|
self.write(self.LINES, self.WRITTEN)
|
|
self.transcript = f"{self.folder}/s.jsonl"
|
|
self.entries: list[dict] = []
|
|
|
|
def write(self, lines: list[str], at: float) -> None:
|
|
pathlib.Path(self.file).write_text("\n".join(lines) + "\n")
|
|
os.utime(self.file, (at, at))
|
|
|
|
def change(self, number: int) -> None:
|
|
self.write([("changed" if n == number else line) for n, line in enumerate(self.LINES, 1)], self.CHANGED)
|
|
|
|
def add(self, entry: dict, parent: str | None = "last") -> str:
|
|
uuid = f"u{len(self.entries)}"
|
|
parent = (self.entries[-1]["uuid"] if self.entries else None) if parent == "last" else parent
|
|
self.entries.append({"uuid": uuid, "parentUuid": parent, "cwd": self.folder, "timestamp": self.READ_AT} | entry)
|
|
pathlib.Path(self.transcript).write_text("\n".join(json.dumps(entry) for entry in self.entries) + "\n")
|
|
return uuid
|
|
|
|
def used(self, tool: str, given: dict, result: str, error: bool = False, at: str = READ_AT) -> None:
|
|
tool_id = f"toolu_{len(self.entries)}"
|
|
self.add({"type": "assistant", "timestamp": at, "message": {"content": [{"type": "tool_use", "id": tool_id, "name": tool, "input": given}]}})
|
|
self.add({"type": "user", "timestamp": at, "message": {"content": [{"type": "tool_result", "tool_use_id": tool_id, "content": result, "is_error": error}]}})
|
|
|
|
def numbered(self, start: int, end: int) -> str:
|
|
return "\n".join(f"{n}\t{self.LINES[n - 1]}" for n in range(start, end + 1))
|
|
|
|
def read(self, start: int, end: int) -> None:
|
|
self.used("Read", {"file_path": self.file, "offset": start, "limit": end - start + 1}, self.numbered(start, end))
|
|
|
|
def shown(self, start: int, end: int) -> str:
|
|
return "\n".join(self.LINES[start - 1:end])
|
|
|
|
def found(self, tool: str, given: dict, agent: str = "") -> list[Breach]:
|
|
return read_breaches(Call(tool, given, self.folder, agent, self.transcript), "/h")
|
|
|
|
def test_a_read_of_lines_already_read_is_blocked_with_a_note(self):
|
|
self.read(1, 10)
|
|
found = self.found("Read", {"file_path": self.file, "offset": 2, "limit": 5})
|
|
self.assertEqual(rules(found), ["repeat-read"])
|
|
self.assertIn(f"lines 2-6 of {self.file} are in your context", found[0].detail)
|
|
|
|
def test_the_note_names_a_single_line_as_one(self):
|
|
self.read(1, 10)
|
|
self.assertIn(f"line 7 of {self.file} is in your context", self.found("Bash", {"command": "sed -n 7p f.txt"})[0].detail)
|
|
|
|
def test_lines_read_in_two_reads_are_in_context(self):
|
|
self.read(1, 10)
|
|
self.read(11, 20)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 5, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_read_past_what_was_read_goes_through(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 5, "limit": 10}), [])
|
|
|
|
def test_a_read_without_a_limit_asks_for_the_whole_file(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file}), [])
|
|
|
|
def test_a_read_result_in_the_older_arrow_format_counts(self):
|
|
self.used("Read", {"file_path": self.file, "offset": 1, "limit": 10}, "\n".join(f"{n:6}→{self.LINES[n - 1]}" for n in range(1, 11)))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_compaction_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.add({"type": "system", "subtype": "compact_boundary"}, parent=None)
|
|
self.add({"type": "user", "message": {"content": "This session is being continued"}})
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_changed_line_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.change(1)
|
|
self.assertEqual(self.found("Bash", {"command": f"head {self.file}"}), [])
|
|
|
|
def test_a_change_in_the_last_asked_line_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.change(10)
|
|
self.assertEqual(self.found("Bash", {"command": "sed -n 1,10p f.txt"}), [])
|
|
|
|
def test_lines_that_did_not_change_stay_in_context(self):
|
|
self.read(1, 10)
|
|
self.change(1)
|
|
self.assertEqual(rules(self.found("Bash", {"command": "sed -n 2,10p f.txt"})), ["repeat-read"])
|
|
|
|
def test_a_read_without_an_offset_starts_at_line_1(self):
|
|
self.read(2, 11)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "limit": 10}), [])
|
|
|
|
def test_calls_that_neither_read_nor_write_the_file_are_passed_over(self):
|
|
self.used("Grep", {"pattern": "line", "path": self.folder}, "f.txt")
|
|
self.used("Bash", {"command": "ls"}, "f.txt")
|
|
self.read(1, 10)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
# Claude Code's Edit refuses a file changed since the agent last read it with Read, or wrote it with Edit or Write, until it reads it again.
|
|
def test_a_change_in_the_millisecond_of_the_last_read_counts_as_before_it(self):
|
|
self.read(1, 10)
|
|
at = datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC).timestamp()
|
|
os.utime(self.file, (at, at))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_file_s_change_is_stamped_as_the_transcript_stamps_its_entries(self):
|
|
at = datetime.datetime(2026, 6, 1, 12, 30, 5, 250000, tzinfo=datetime.UTC).timestamp()
|
|
os.utime(self.file, (at, at))
|
|
self.assertEqual(modified(self.file), "2026-06-01T12:30:05.250Z")
|
|
|
|
def test_a_read_goes_through_once_the_file_changed_after_the_last_read(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_refused_edit_does_not_count_as_reading_the_file(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.used("Edit", {"file_path": self.file, "old_string": "line 5", "new_string": "five"}, "<tool_use_error>File has been modified since read, either by the user or by a linter. Read it again before attempting to write it.</tool_use_error>", error=True, at=self.AFTER_THE_CHANGE)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_an_edit_keeps_the_lines_it_left_unchanged_in_context(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.used("Edit", {"file_path": self.file, "old_string": "line 30", "new_string": "changed"}, f"The file {self.file} has been updated successfully.", at=self.AFTER_THE_CHANGE)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_failed_read_holds_no_lines(self):
|
|
self.used("Read", {"file_path": self.file, "offset": 1, "limit": 10}, self.numbered(1, 10), error=True)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_read_on_a_branch_the_session_rewound_holds_no_lines(self):
|
|
start = self.add({"type": "user", "message": {"content": "go"}})
|
|
self.read(1, 10)
|
|
self.add({"type": "user", "message": {"content": "go again"}}, parent=start)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_head_and_sed_of_lines_read_are_blocked(self):
|
|
self.read(1, 20)
|
|
for command in [f"head -n 5 {self.file}", f"head -n5 {self.file}", f"head -5 {self.file}", f"head {self.file}", "sed -n '3,8p' f.txt", "sed -n 7p f.txt"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(rules(self.found("Bash", {"command": command})), ["repeat-read"])
|
|
|
|
def test_a_head_starts_at_line_1(self):
|
|
self.read(2, 11)
|
|
self.assertEqual(self.found("Bash", {"command": f"head {self.file}"}), [])
|
|
|
|
def test_a_cat_asks_for_the_whole_file(self):
|
|
self.read(1, 20)
|
|
self.assertEqual(self.found("Bash", {"command": f"cat {self.file}"}), [])
|
|
|
|
# Claude Code's Edit and Write take only a Read as having read the file.
|
|
def test_a_read_goes_through_until_the_agent_read_the_file_with_read(self):
|
|
self.used("Bash", {"command": f"sed -n '1,20p' {self.file}"}, self.shown(1, 20))
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 4, "limit": 3}), [])
|
|
|
|
def test_a_bash_output_counts_against_a_read_once_the_agent_read_the_file_with_read(self):
|
|
self.read(21, 30)
|
|
self.used("Bash", {"command": f"sed -n '1,20p' {self.file}"}, self.shown(1, 20))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 4, "limit": 3})), ["repeat-read"])
|
|
|
|
def test_a_cat_output_puts_every_line_in_context(self):
|
|
self.used("Bash", {"command": f"cat {self.file}"}, self.shown(1, 30))
|
|
self.assertEqual(rules(self.found("Bash", {"command": "sed -n '25,$p' f.txt"})), ["repeat-read"])
|
|
|
|
def test_a_relative_path_in_the_transcript_is_under_its_entry_s_cwd(self):
|
|
self.used("Bash", {"command": "cat f.txt"}, self.shown(1, 30))
|
|
self.assertEqual(rules(read_breaches(Call("Bash", {"command": f"head -n 5 {self.file}"}, "/", "", self.transcript), "/h")), ["repeat-read"])
|
|
|
|
def test_a_tilde_path_is_under_the_home_folder(self):
|
|
self.used("Read", {"file_path": "~/f.txt", "offset": 1, "limit": 10}, self.numbered(1, 10))
|
|
self.used("Bash", {"command": "cat ~/f.txt"}, self.shown(1, 30))
|
|
for tool, given in [("Read", {"file_path": "~/f.txt", "offset": 1, "limit": 10}), ("Bash", {"command": "sed -n 1,30p ~/f.txt"})]:
|
|
with self.subTest(tool):
|
|
self.assertEqual(rules(read_breaches(Call(tool, given, "/", "", self.transcript), self.folder)), ["repeat-read"])
|
|
|
|
def test_other_commands_on_the_file_are_not_checked(self):
|
|
self.used("Bash", {"command": f"cat {self.file}"}, self.shown(1, 30))
|
|
for command in [f"cat {self.file} > copy", f"cat {self.file} | wc -l", f"cat {self.file} {self.file}", f"head -c 5 {self.file}", f"sed 's/a/b/' {self.file}"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(self.found("Bash", {"command": command}), [])
|
|
|
|
# As Claude Code 2.1.312 writes a Bash output it cut short.
|
|
def test_an_output_cut_short_shows_no_lines(self):
|
|
for cut in [
|
|
f"<persisted-output>\nOutput too large (37.9KB). Full output saved to: {self.folder}/tool-results/b2hymdt7w.txt\n\nPreview (first 2KB):\n{self.shown(1, 30)}\n...\n</persisted-output>",
|
|
f"<truncated-output>\nOutput too large (37.9KB). It could not be saved, so only the first 2KB are shown; the rest was dropped. If the tool can page or filter its results, call it again for the part you need.\n\n{self.shown(1, 30)}\n</truncated-output>",
|
|
self.shown(1, 5) + "\n\n... [25 lines truncated] ...",
|
|
]:
|
|
with self.subTest(cut[:20]):
|
|
self.assertEqual(result_lines("Bash", Asked(self.file, 1, None), cut), {})
|
|
|
|
def test_a_fifo_is_never_opened(self):
|
|
fifo = f"{self.folder}/fifo"
|
|
os.mkfifo(fifo)
|
|
self.used("Bash", {"command": f"cat {fifo}"}, "line 1")
|
|
# Opened, the FIFO would block the hook, since it has no writer.
|
|
ran = hook({"cwd": self.folder, "tool_name": "Bash", "tool_input": {"command": f"head -n 1 {fifo}"}, "transcript_path": self.transcript}, self.folder)
|
|
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
|
|
|
|
def called(self, agent_id: str = "") -> Call:
|
|
given = {"cwd": self.folder, "tool_name": "Read", "tool_input": {"file_path": self.file, "offset": 1, "limit": 10}, "transcript_path": self.transcript}
|
|
return parsed_call(given | ({"agent_id": agent_id, "agent_type": "Explore"} if agent_id else {}))
|
|
|
|
def test_a_subagent_s_transcript_is_beside_the_session_s(self):
|
|
self.assertEqual(self.called("a1").transcript, f"{self.folder}/s/subagents/agent-a1.jsonl")
|
|
|
|
def test_the_session_s_reads_are_not_in_a_subagent_s_context(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(read_breaches(self.called("a1"), "/h"), [])
|
|
|
|
def test_the_session_is_judged_on_its_own_transcript(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(rules(read_breaches(self.called(), "/h")), ["repeat-read"])
|
|
|
|
def test_a_missing_transcript_lets_the_read_through(self):
|
|
self.assertEqual(read_breaches(Call("Read", {"file_path": self.file}, self.folder, "", f"{self.folder}/none.jsonl"), "/h"), [])
|
|
|
|
def test_a_read_of_a_missing_path_or_a_directory_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
for path in (f"{self.folder}/none.txt", self.folder):
|
|
with self.subTest(path=path):
|
|
self.assertEqual(self.found("Read", {"file_path": path}), [])
|
|
|
|
def test_the_hook_blocks_a_repeated_read_without_offering_an_override(self):
|
|
self.read(1, 10)
|
|
self.used("Bash", {"command": f"head -n 5 {self.file}"}, self.shown(1, 5))
|
|
for tool, given in [("Read", {"file_path": self.file, "offset": 1, "limit": 10}), ("Bash", {"command": f"head -n 5 {self.file}"})]:
|
|
with self.subTest(tool):
|
|
ran = hook({"cwd": self.folder, "tool_name": tool, "tool_input": given, "transcript_path": self.transcript}, self.folder)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule repeat-read: ", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
|
|
class Hook(unittest.TestCase):
|
|
def setUp(self):
|
|
self.repo = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.repo])
|
|
git(self.repo, "init", "-q", "-b", "main")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "base")
|
|
git(self.repo, "checkout", "-q", "-b", "chunk")
|
|
|
|
def test_runs_git_guarded(self):
|
|
git(self.repo, "config", "core.fsmonitor", "/bin/false")
|
|
self.assertEqual(guarded_git(self.repo, "config", "core.fsmonitor"), "false")
|
|
|
|
def listed(self) -> str:
|
|
return subprocess.run([str(HOOK), "overrides"], capture_output=True, check=True, cwd=self.repo, env=isolated_env(), text=True).stdout
|
|
|
|
def test_a_breach_exits_2_naming_the_rule_and_the_override(self):
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": "FROM node:24\n"}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule image-pin: ", ran.stderr)
|
|
self.assertIn("`# rule-override image-pin: <reason>`", ran.stderr)
|
|
|
|
def test_an_edit_of_a_fifo_or_a_device_finishes(self):
|
|
fifo = f"{self.repo}/pipe"
|
|
os.mkfifo(fifo)
|
|
# A cap on the hook's memory, so a read of /dev/zero fails fast instead of filling the machine.
|
|
capped = lambda: resource.setrlimit(resource.RLIMIT_AS, (1 << 30, 1 << 30))
|
|
for path, (tool, given) in itertools.product((fifo, "/dev/zero"), (("Write", {"content": "x\n"}), ("Edit", {"old_string": "a", "new_string": "b"}), ("MultiEdit", {"edits": [{"old_string": "a", "new_string": "b"}]}))):
|
|
with self.subTest(path=path, tool=tool):
|
|
call = {"cwd": self.repo, "tool_input": {"file_path": path, **given}, "tool_name": tool}
|
|
ran = subprocess.run([str(HOOK)], capture_output=True, cwd=self.repo, env=isolated_env(), input=json.dumps(call), preexec_fn=capped, text=True, timeout=10)
|
|
self.assertEqual(ran.returncode, 0)
|
|
|
|
def test_an_override_passes_and_is_listed_for_the_chunk(self):
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/sub/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
|
|
self.assertRegex(self.listed(), r"^\S+Z\tchunk\t[0-9a-f]{40}\timage-pin\tthe vendor tags no patch versions\t\S+/sub/Dockerfile\n$")
|
|
git(self.repo, "checkout", "-q", "main")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "elsewhere")
|
|
self.assertEqual(self.listed(), "", "the default branch holds no override of its own")
|
|
|
|
def test_an_override_outlives_an_amend(self):
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "work")
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
git(self.repo, "commit", "-q", "--amend", "--allow-empty", "-m", "reworded")
|
|
git(self.repo, "branch", "-m", "renamed")
|
|
self.assertIn("\timage-pin\t", self.listed(), "the reflog keeps the amended commit")
|
|
|
|
def test_an_unreadable_reflog_fails_the_listing(self):
|
|
unborn = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", unborn])
|
|
git(unborn, "init", "-q", "-b", "chunk")
|
|
ran = subprocess.run([str(HOOK), "overrides"], capture_output=True, cwd=unborn, env=isolated_env(), text=True)
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("cannot read the branch's reflog", ran.stderr)
|
|
git(self.repo, "reflog", "expire", "--expire=now", "--all")
|
|
self.assertEqual(subprocess.run([str(HOOK), "overrides"], capture_output=True, cwd=self.repo, env=isolated_env(), text=True).returncode, 1, "an expired reflog holds no HEAD")
|
|
|
|
def test_a_sibling_branch_s_override_is_not_listed(self):
|
|
git(self.repo, "checkout", "-q", "-b", "sibling")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "sibling work")
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
git(self.repo, "checkout", "-q", "chunk")
|
|
self.assertEqual(self.listed(), "")
|
|
git(self.repo, "merge", "-q", "--squash", "sibling")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "squashed")
|
|
self.assertEqual(self.listed(), "", "a squash merge brings in none of the sibling's commits")
|
|
|
|
def test_a_blocked_call_logs_no_override(self):
|
|
command = f"git commit -m a -m '{TRAILER}' # rule-override no-ai-attribution: quotes it"
|
|
self.assertEqual(hook({"cwd": self.repo, "tool_input": {"command": command}, "tool_name": "Bash"}, self.repo).returncode, 2)
|
|
self.assertEqual(self.listed(), "")
|
|
|
|
def test_an_agent_call_below_its_rung_is_refused_and_offers_no_override(self):
|
|
call = {"cwd": self.repo, "tool_input": {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review. # rule-override reviewer-rung: trust me"}, "tool_name": "Agent"}
|
|
ran = hook(call, self.repo, ReviewerRung.ENVIRON)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule reviewer-rung", ran.stderr)
|
|
self.assertIn("stability-reviewer", ran.stderr)
|
|
self.assertIn("claude-sonnet-5-5 at high effort", ran.stderr)
|
|
self.assertIn("model opus, effort medium", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_an_agent_call_at_its_rung_passes(self):
|
|
call = {"cwd": self.repo, "tool_input": {"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium", "prompt": "Review."}, "tool_name": "Agent"}
|
|
self.assertEqual(hook(call, self.repo, ReviewerRung.ENVIRON).returncode, 0)
|
|
|
|
def test_a_reviewers_write_outside_the_reports_folder_is_refused_and_offers_no_override(self):
|
|
call = {"agent_type": "prose-reviewer", "cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/a.md", "content": "<!-- rule-override reviewer-report: trust me -->\n"}, "tool_name": "Write"}
|
|
ran = hook(call, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule reviewer-report: prose-reviewer would write ", ran.stderr)
|
|
self.assertIn("/.local/state/larv-review/", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_a_commit_override_is_logged_in_the_commit_s_checkout(self):
|
|
outside = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", outside])
|
|
ran = hook({"cwd": outside, "tool_input": {"command": f"git -C {self.repo} commit -m a -m b # rule-override commit-one-line: a body is needed"}, "tool_name": "Bash"}, outside)
|
|
self.assertEqual(ran.returncode, 0)
|
|
self.assertIn("\tcommit-one-line\ta body is needed\t", self.listed())
|
|
|
|
def test_an_edit_is_judged_on_the_file_it_leaves(self):
|
|
pathlib.Path(self.repo, ".gitignore").write_text("a\nc\n")
|
|
call = {"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/.gitignore", "new_string": "d", "old_string": "a"}, "tool_name": "Edit"}
|
|
self.assertEqual(hook(call, self.repo).returncode, 2)
|
|
|
|
def round_commit(self, title: str, given: str = "") -> subprocess.CompletedProcess:
|
|
return hook({"cwd": self.repo, "tool_input": {"command": f"git commit {given}-m '{title}'"}, "tool_name": "Bash"}, self.repo)
|
|
|
|
def test_a_round_commit_past_round_2_is_refused_and_offers_no_override(self):
|
|
ran = self.round_commit("Answer stability round 3: 0 🔴, 1 🟡 (0 misread); fix # rule-override round-cap: the human said yes")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule round-cap: this would be stability's round 3, past its 2", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); fix").returncode, 0)
|
|
|
|
def test_a_round_commit_naming_no_pass_is_refused_and_offers_no_override(self):
|
|
ran = self.round_commit("Answer architectur round 1: 0 🔴, 1 🟡 (0 misread); fix # rule-override round-cap: a typo")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule round-cap: `architectur` names no larv-review pass, so its rounds would never reach a cap; name one of: architecture, principles, full scan,", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_a_third_round_commit_of_a_pass_on_the_branch_is_refused_whatever_its_number(self):
|
|
for title in ["Answer stability round 1: 1 🔴, 0 🟡 (0 misread); a", "Answer prose round 1: 0 🔴, 1 🟡 (0 misread); b", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
ran = self.round_commit("Answer stability round 2: 0 🔴, 0 🟡 (0 misread); d")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("stability's round commits on this branch: `Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c`, `Answer stability round 1: 1 🔴, 0 🟡 (0 misread); a`", ran.stderr)
|
|
self.assertIn("`git commit --amend` where that is the newest commit", ran.stderr)
|
|
self.assertEqual(self.round_commit("Answer prose round 2: 0 🔴, 0 🟡 (0 misread); d").returncode, 0)
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c, reworded", "--amend --allow-empty ").returncode, 0, "an amend replaces the commit it counts")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "fix a typo")
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); e", "--amend --allow-empty ").returncode, 2, "an amend replaces the newest commit, not the oldest")
|
|
|
|
def test_a_stalled_origin_holds_up_neither_a_round_commit_nor_the_overrides_listing(self):
|
|
git(self.repo, "config", "protocol.ext.allow", "always")
|
|
git(self.repo, "remote", "add", "origin", "ext::sleep 30")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a").returncode, 0)
|
|
hook({"cwd": self.repo, "tool_input": {"command": "git commit -m a -m b # rule-override commit-one-line: a body is needed"}, "tool_name": "Bash"}, self.repo)
|
|
listed = subprocess.run([str(HOOK), "overrides"], capture_output=True, check=True, cwd=self.repo, env=isolated_env(), text=True, timeout=10).stdout
|
|
self.assertIn("\tcommit-one-line\ta body is needed\t", listed)
|
|
|
|
def test_round_commits_merged_in_from_the_default_branch_do_not_count(self):
|
|
git(self.repo, "checkout", "-q", "main")
|
|
for title in ["Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); b"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
git(self.repo, "checkout", "-q", "chunk")
|
|
git(self.repo, "merge", "-q", "--no-ff", "-m", "Merge main", "main")
|
|
git(self.repo, "branch", "-f", "main", "main~2")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); c").returncode, 0, "a stale default branch still leaves another chunk's rounds out")
|
|
|
|
def test_a_stale_local_main_does_not_stand_in_for_a_default_branch_origin_left_unrecorded(self):
|
|
git(self.repo, "checkout", "-q", "main")
|
|
for title in ["Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); b"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
git(self.repo, "checkout", "-q", "-B", "chunk", "main")
|
|
git(self.repo, "branch", "-f", "main", "main~2")
|
|
git(self.repo, "remote", "add", "origin", f"{self.repo}/gone.git")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); c").returncode, 0, "another chunk's rounds, fast-forwarded into the default branch, are not this chunk's")
|
|
|
|
def test_a_commit_while_an_unranked_todo_has_changes_is_blocked(self):
|
|
pathlib.Path(self.repo, "docs").mkdir()
|
|
for name, content in [("README.md", README), ("docs/decisions.md", DECISIONS), ("todo.md", TODO)]:
|
|
pathlib.Path(self.repo, name).write_text(content)
|
|
git(self.repo, "add", "README.md", "docs", "todo.md")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
call = {"cwd": self.repo, "tool_input": {"command": "git add -A && git commit -m x"}, "tool_name": "Bash"}
|
|
self.assertEqual(hook(call, self.repo).returncode, 0, "todo.md is unchanged")
|
|
pathlib.Path(self.repo, "todo.md").write_text(TODO + "\n")
|
|
ran = hook(call, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("todo.md: not ranked", ran.stderr)
|
|
|
|
def test_a_staged_todo_change_is_checked_and_no_clean_filter_runs(self):
|
|
pathlib.Path(self.repo, "docs").mkdir()
|
|
for name, content in [("README.md", README), ("docs/decisions.md", DECISIONS), ("todo.md", TODO), (".gitattributes", "todo.md filter=planted\n")]:
|
|
pathlib.Path(self.repo, name).write_text(content)
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "config", "filter.planted.clean", f"touch {self.repo}/ran; cat")
|
|
pathlib.Path(self.repo, "ran").unlink(missing_ok=True)
|
|
# A stat-dirty index entry, as extracting an archive leaves, is what makes `git status` re-run the filter.
|
|
os.utime(pathlib.Path(self.repo, "todo.md"), (1, 1))
|
|
ran = hook({"cwd": self.repo, "tool_input": {"command": "git commit -m x"}, "tool_name": "Bash"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2, ran.stderr)
|
|
self.assertFalse(pathlib.Path(self.repo, "ran").exists())
|
|
|
|
def test_a_symlinked_log_is_refused(self):
|
|
target = pathlib.Path(self.repo, "elsewhere")
|
|
pathlib.Path(self.repo, ".git", "rule-overrides.tsv").symlink_to(target)
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertFalse(target.exists())
|
|
|
|
def test_a_malformed_override_is_named(self):
|
|
ran = hook({"cwd": self.repo, "tool_input": {"command": f"git commit -m a -m b -m '{TRAILER}' # rule-override no-ai-attribution: quotes it # rule-override commit-oneline: x"}, "tool_name": "Bash"}, self.repo)
|
|
self.assertIn("`rule-override commit-oneline: x` names no rule below", ran.stderr)
|
|
|
|
def test_a_missing_field_breaks_nothing(self):
|
|
for given in [{"command": None}, {"content": None, "file_path": f"{self.repo}/a.md"}, {"edits": [None], "file_path": f"{self.repo}/a.md"}]:
|
|
with self.subTest(given):
|
|
tool = "Bash" if "command" in given else "Write" if "content" in given else "MultiEdit"
|
|
self.assertEqual(hook({"cwd": self.repo, "tool_input": given, "tool_name": tool}, self.repo).returncode, 0)
|
|
|
|
def test_unreadable_input_does_not_block(self):
|
|
for given in ["{", json.dumps({"cwd": ".", "tool_input": None, "tool_name": "Bash"})]:
|
|
with self.subTest(given):
|
|
self.assertEqual(subprocess.run([str(HOOK)], capture_output=True, env=isolated_env(), input=given, text=True).returncode, 1)
|
|
|
|
def test_help_lists_every_rule(self):
|
|
helped = subprocess.run([str(HOOK), "--help"], capture_output=True, check=True, env=isolated_env(), text=True).stdout
|
|
for rule in ["action-pin", "commit-one-line", "docker-run-memory", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check"]:
|
|
with self.subTest(rule):
|
|
self.assertIn(f" {rule}\n The rule: ", helped)
|
|
self.assertIn("reviewer-report\n The rule: a reviewer writes nothing but its report", helped)
|
|
self.assertRegex(helped, r" reviewer-report\n( .*\n){2} Takes no override\.\n")
|
|
self.assertRegex(helped, r" image-pin\n( .*\n){2} \S", "a rule a marker waives says nothing of overrides")
|