1473 lines
98 KiB
Python
1473 lines
98 KiB
Python
import datetime
|
|
import itertools
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import resource
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import typing
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from rule_hook.repeat_read import Asked, modified, read_breaches, result_lines
|
|
from project_config.templates import BARE, Titles
|
|
from rule_hook.rules import blocks, Breach, check, command_breaches, file_breaches, overrides, RuleName, RULES, title_breaches
|
|
from rule_hook.main import breaches_of
|
|
from rule_hook.call import Call, parsed_call
|
|
from git_cli.git_command import commit_message, CommitCall, commits
|
|
from project_config.test_project_config import configured, write_config
|
|
from plan.test_ranking import DECISIONS, README, TODO
|
|
from rule_hook.git import guarded_git
|
|
from test_env import isolated_env
|
|
from rule_hook.override_log import LogLine, parsed_entry, this_chunks
|
|
from rule_hook.reviewer_rung import reviewer_breaches, skill_breaches
|
|
from command_lines.shell_command import segments, simple_commands
|
|
from rule_hook.todo_check import todo_breaches
|
|
|
|
HOOK = pathlib.Path(__file__).resolve().parents[1] / "rule-hook"
|
|
# Built from parts, so this file carries no attribution of its own.
|
|
TRAILER = "Co-Authored-" + "By: Claude <noreply@anthropic.com>"
|
|
|
|
|
|
def rules(breaches: list[Breach]) -> list[str]:
|
|
return [breach.rule for breach in breaches]
|
|
|
|
|
|
def committed(command: str, cwd: str = "/r") -> list[CommitCall]:
|
|
return commits(simple_commands(command, cwd, "/h"), "/h")
|
|
|
|
|
|
def breaches(command: str, titles: Titles = BARE, held: bool = False) -> list[Breach]:
|
|
"""The command's breaches and those of each commit's title, read through titles; held is whether the title must fit the commit template."""
|
|
simples = simple_commands(command, "/r", "/h")
|
|
return command_breaches(command, simples) + [breach for commit in commits(simples, "/h") for breach in title_breaches(commit, titles, "item-{id}", "item-7" if held else "main")]
|
|
|
|
|
|
def checked(command: str) -> list[str]:
|
|
return rules(breaches(command))
|
|
|
|
|
|
def hook(call: dict, cwd: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess:
|
|
return subprocess.run([str(HOOK)], capture_output=True, cwd=cwd, env=isolated_env() | (env or {}), input=json.dumps(call), text=True, timeout=10)
|
|
|
|
|
|
def git(cwd: str, *args: str) -> str:
|
|
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, check=True, cwd=cwd, env=isolated_env(), text=True).stdout
|
|
|
|
|
|
class Commits(unittest.TestCase):
|
|
def test_cd_and_dash_capital_c_move_the_commit(self):
|
|
self.assertEqual(committed("cd a && git -C ~/b commit -am 'x'"), [CommitCall("/h/b", ["x"], False, False)])
|
|
self.assertEqual(committed("cd a && git -c x=y -C b commit -m x")[0].directory, "/r/a/b")
|
|
|
|
def test_each_message_spelling_is_read(self):
|
|
self.assertEqual(commit_message(["--message=a", "-m", "b=c", "--message", "c", "-mz", "-am", "d"]), (["a", "b=c", "c", "z", "d"], False, False))
|
|
|
|
def test_an_amend_is_seen(self):
|
|
self.assertEqual([commit.amends for commit in committed("git commit --amend -m x && git commit -m y")], [True, False])
|
|
self.assertEqual(commit_message(["--", "--amend"]), ([], False, False), "after --, --amend names a path")
|
|
|
|
def test_a_file_message_is_seen(self):
|
|
self.assertEqual(commit_message(["-F", "-"]), ([], True, False))
|
|
self.assertEqual(commit_message(["--file=x"]), ([], True, False))
|
|
|
|
def test_other_git_commands_are_no_commit(self):
|
|
self.assertEqual(committed("git log --grep commit; echo git commit"), [])
|
|
|
|
def test_a_line_break_or_a_comment_separates_commands(self):
|
|
self.assertEqual(committed("cd a # go there\nGIT_EDITOR=true git commit -m x"), [CommitCall("/r/a", ["x"], False, False)])
|
|
|
|
def test_a_commit_after_a_heredoc_or_a_blank_line_is_seen(self):
|
|
self.assertEqual(committed("cat <<EOF > notes\nhello\nEOF\ngit commit -m a"), [CommitCall("/r", ["a"], False, False)])
|
|
self.assertEqual(committed("git status &&\n\ngit commit -m a"), [CommitCall("/r", ["a"], False, False)])
|
|
|
|
def test_an_arithmetic_shift_is_no_heredoc(self):
|
|
self.assertEqual(committed("echo $((1<<2))\ngit commit -m a\n2"), [CommitCall("/r", ["a"], False, False)])
|
|
|
|
def test_a_line_that_does_not_parse_hides_only_itself(self):
|
|
self.assertEqual(committed("echo 'open\ngit commit -m a"), [CommitCall("/r", ["a"], False, False)])
|
|
|
|
def test_a_redirect_keeps_its_command_s_words_together(self):
|
|
self.assertEqual(segments("echo hi > f arg"), [["echo", "hi", "arg"]])
|
|
for redirect in [">", ">>", "<", "<<<", "<>", ">&", "<&", ">|", "&>", "&>>"]:
|
|
with self.subTest(redirect):
|
|
self.assertEqual(committed(f"git {redirect} /tmp/x commit -m hi"), [CommitCall("/r", ["hi"], False, False)])
|
|
self.assertEqual(committed("git commit >&2 -m hi 2>&1"), [CommitCall("/r", ["hi"], False, False)])
|
|
self.assertEqual(committed("> out git commit -m hi"), [CommitCall("/r", ["hi"], False, False)])
|
|
|
|
def test_a_separator_glued_to_a_redirect_still_ends_the_command(self):
|
|
self.assertEqual(segments("echo a > f; echo b | cat &> g\nls"), [["echo", "a"], ["echo", "b"], ["cat"], ["ls"]])
|
|
for command in ["echo a &</dev/null git commit -m hi", "echo a;>f git commit -m hi"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(committed(command), [CommitCall("/r", ["hi"], False, False)])
|
|
|
|
|
|
class CommandBreaches(unittest.TestCase):
|
|
def test_one_line_passes(self):
|
|
self.assertEqual(checked("git add . && git commit -m 'ABC-1 - Fix #2'"), [])
|
|
|
|
def test_a_heredoc_a_second_message_or_a_file_breaks_commit_one_line(self):
|
|
for command in ["git commit -m \"$(cat <<'EOF'\nIt's one line\nEOF\n)\"", "git commit -m a -m b", "git commit -F - <<'EOF'\nTitle\n\nIt's the body\nEOF\necho done", "git commit --author '' -m a -m b"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["commit-one-line"])
|
|
|
|
def test_a_round_title_without_its_counts_breaks_round_title_counts(self):
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: fix the parser'"), ["round-title-counts"])
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: 1 🔴, 2 🟡; fix the parser'"), ["round-title-counts"])
|
|
command = "git commit -m 'Answer stability round 1: fix the parser'"
|
|
self.assertEqual(breaches(command), [Breach("round-title-counts", "open the summary after the round's colon with `<k> 🔴, <m> 🟡 (<j> misread);`, where <j> is at most <k> + <m>", "/r")])
|
|
self.assertEqual(checked("git commit --allow-empty -m 'Answer prose round 2: 0 🔴, 0 🟡 (0 misread)'"), [])
|
|
self.assertEqual(checked("git commit -m 'Answer stability round 1: 1 🔴, 2 🟡 (1 misread); fix the parser'"), [])
|
|
self.assertEqual(checked("git commit -m 'Answer the question in the README'"), [])
|
|
|
|
def test_a_docker_build_or_a_mutation_run_outside_run_heavy_breaks_run_heavy(self):
|
|
for command in ["docker build -t x .", "sudo docker build -t x .", "doas docker build .", "uv run mutmut run", "uv run --with pytest mutmut run", "poetry run mutmut run", "cd /w && docker buildx build .", "docker compose build", "docker compose -f x.yml --verbose build app", "uvx --with pytest 'mutmut>=3,<4' run", "cargo mutants --in-diff d", "npx stryker run", "vendor/bin/infection --git-diff-lines", "docker compose run --rm app mutmut run", "docker run --rm -m 4g --memory-swap 4g -v /w:/w rust:1.90.0 cargo mutants", "docker compose exec app npx stryker run", "docker run -h box -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "nice -n 19 docker build .", "env DOCKER_BUILDKIT=1 docker build .", "time -p docker compose build", "docker --tlscacert ca.pem build .", "docker --tlscert c.pem --tlskey k.pem build .", "docker buildx --builder b build .", "docker container run -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "uvx mutmut --verbose run", "~/.claude/bin/mutate-changes", "cd /w && mutate-changes --base main"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["run-heavy"])
|
|
|
|
def test_a_heavy_command_under_run_heavy_or_a_light_docker_or_mutmut_call_passes(self):
|
|
for command in ["~/.claude/bin/run-heavy docker build -t x .", "/home/me/.claude/bin/run-heavy uvx mutmut run", "nice -n 19 ~/.claude/bin/run-heavy docker build .", "env X=1 run-heavy docker compose run app mutmut run", "docker images", "docker compose run --rm app go vet", "docker compose run app make build", "uvx 'mutmut>=3,<4' results", "echo docker build", "npx prettier --check infection.json5", "echo mutmut run", "env", "docker build --help", "cargo mutants --list", "infection --version", "uvx mutmut a b run", "~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --jobs 2", "~/.claude/bin/mutate-changes --help"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), [])
|
|
|
|
def test_a_docker_run_or_create_without_a_memory_limit_breaks_docker_run_memory(self):
|
|
for command in ["docker run --rm node:24.22.1-alpine3.24 node x.js", "cd /w && docker run -v /w:/w alpine:3.22.2 true", "docker create alpine:3.22.2", "docker container run alpine:3.22.2 true", "docker container create alpine:3.22.2", "nice -n 19 docker run alpine:3.22.2 true", "docker -H unix:///d.sock run alpine:3.22.2 true", "docker run --memory-swap 4g alpine:3.22.2 true", "docker run -m 4g alpine:3.22.2 true", "docker run --memory=4g alpine:3.22.2 sh -c 'x --memory-swap 4g'", "docker run -m 4g alpine:3.22.2 tool --memory-swap", "docker run -m 4g --memory-swap -1 alpine:3.22.2 true", "docker run -m 4g --memory-swap=0 alpine:3.22.2 true", "docker run --rm python:3.13.1 python -m pytest", "docker run alpine:3.22.2 git commit -m x", "timeout 60 docker run alpine:3.22.2 true", "docker run node:24.22.1-alpine3.24 node --help", "docker run alpine:3.22.2 --help", "docker run -m 4g alpine:3.22.2 sleep 60", "~/.claude/bin/run-heavy /usr/bin/docker run alpine:3.22.2 true", "docker run --memory-swap 4g alpine:3.22.2 grep -m 1 x", "run-heavy env PATH=/usr/bin docker run alpine:3.22.2 true", "run-heavy sudo docker run alpine:3.22.2 true", "docker run alpine:3.22.2 git tag -m 1.2.3", "run-heavy env -i docker run alpine:3.22.2 true", "run-heavy env -u PATH docker run alpine:3.22.2 true", "run-heavy env --unset=PATH docker run alpine:3.22.2 true", "run-heavy env --ignore-environment docker run alpine:3.22.2 true", "run-heavy doas docker run alpine:3.22.2 true"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), ["docker-run-memory"])
|
|
|
|
def test_a_docker_run_with_a_memory_limit_or_another_docker_call_passes_docker_run_memory(self):
|
|
for command in ["docker run -m 4g --memory-swap 4g alpine:3.22.2 true", "docker run -m4g --memory-swap=4g alpine:3.22.2 true", "docker run --memory 512M --memory-swap 1g alpine:3.22.2 true", "docker run -m 1073741824 --memory-swap 2g alpine:3.22.2 true", "timeout 60 docker run -m 4g --memory-swap 4g alpine:3.22.2 true", "docker run --rm --memory=1.5GiB --memory-swap=1.5GiB alpine:3.22.2 true", "docker container create -m 1g --memory-swap 1g alpine:3.22.2", "docker run --help", "docker container run --help", "docker --context run run --help", "docker --help", "docker ps", "docker images", "docker container ls", "docker compose run --rm app go vet", "docker exec box true", "echo docker run alpine:3.22.2", "~/.claude/bin/run-heavy docker run alpine:3.22.2 true"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(checked(command), [])
|
|
|
|
def test_an_attribution_breaks_only_in_a_git_gh_tea_or_chunk_pr_command(self):
|
|
self.assertEqual(checked(f"tea pr create --description '{TRAILER}'"), ["no-ai-attribution"])
|
|
self.assertEqual(checked(f"~/.claude/bin/chunk-pr open 'Do it' '{TRAILER}'"), ["no-ai-attribution"])
|
|
self.assertEqual(checked(f"grep -rn '{TRAILER}' ."), [])
|
|
|
|
def test_a_commit_breach_names_the_commit_s_checkout(self):
|
|
self.assertEqual(breaches("cd w && git commit -m a -m b")[0].directory, "/r/w")
|
|
|
|
def test_a_title_must_fit_the_commit_template_only_where_the_branch_holds_it_to_it(self):
|
|
titles = Titles("PAY-{id} - {summary}", "{summary}")
|
|
self.assertEqual(breaches("git commit -m 'Release review of 0.3.0'", titles, held=True), [Breach("commit-title", "title the commit as the project's config asks, `PAY-{id} - {summary}`, where {id} is the ID of the chunk's first item; `Release review of 0.3.0` doesn't fit", "/r")])
|
|
self.assertEqual(breaches("git commit -m 'Release review of 0.3.0'", titles), [])
|
|
self.assertEqual(breaches("git commit -m 'PAY-7 - Tidy'", titles, held=True), [])
|
|
|
|
def test_a_round_title_is_read_through_either_template_for_its_counts(self):
|
|
titles = Titles("PAY-{id} - {summary}", "fix: {summary}")
|
|
for title in ("PAY-7 - Answer prose round 1: fix", "fix: Answer prose round 1: fix", "Answer prose round 1: fix"):
|
|
with self.subTest(title):
|
|
self.assertEqual(rules(breaches(f"git commit -m '{title}'", titles)), ["round-title-counts"])
|
|
self.assertEqual(breaches("git commit -m 'PAY-7 - Answer prose round 1: 0 🔴, 0 🟡 (0 misread)'", titles, held=True), [])
|
|
|
|
|
|
class FileBreaches(unittest.TestCase):
|
|
def test_an_image_needs_a_patch_version_a_date_or_a_digest(self):
|
|
added = "FROM node:24 AS build\nFROM node:24.18.1-bookworm-slim\nFROM alpine@sha256:ab\nFROM debian:bookworm-20240926-slim\nFROM build\nFROM scratch\nFROM ${BASE}\nFROM python\nCOPY --from=build /a /a\nCOPY --from=0 /b /b\nCOPY --chown=1 --from=alpine:3 /c /c\n"
|
|
self.assertEqual([breach.detail for breach in file_breaches("/r/Dockerfile", "", added, added)], ["/r/Dockerfile: node:24", "/r/Dockerfile: python", "/r/Dockerfile: alpine:3"])
|
|
|
|
def test_a_stage_named_outside_the_edit_is_no_image(self):
|
|
self.assertEqual(file_breaches("app.dockerfile", "FROM node:24.1.0 AS base\n", "FROM base\n", "FROM node:24.1.0 AS base\nFROM base\n"), [])
|
|
|
|
def test_a_workflow_checks_its_images_and_actions(self):
|
|
added = " container: node:24\n image: mysql:8.4\n - image: 'mysql:8.4.7'\n - uses: actions/checkout@v4\n - uses: a/b@0123456789abcdef0123456789abcdef01234567\n - uses: ./local\n"
|
|
self.assertEqual(rules(file_breaches("/r/.gitea/workflows/ci.yml", "", added, added)), ["image-pin", "image-pin", "action-pin"])
|
|
|
|
def test_a_compose_file_checks_its_images(self):
|
|
self.assertEqual(rules(file_breaches("compose.yaml", "", " image: redis\n", " image: redis\n")), ["image-pin"])
|
|
|
|
def test_other_yaml_is_not_checked(self):
|
|
added = "image: /assets/og.png\nimage: { type: string }\nuses: a/b@main\n"
|
|
self.assertEqual(file_breaches("_config.yml", "", added, added), [])
|
|
|
|
def test_an_ignore_file_sorts_each_block_split_by_blank_lines_and_comments(self):
|
|
self.assertEqual(blocks("b\na\n\n# c\nd\n# e\nf\n"), [["b", "a"], ["d"], ["f"]])
|
|
self.assertEqual(rules(file_breaches(".dockerignore", "", "x", "b\na\n")), ["ignorefile-sorted"])
|
|
self.assertEqual(file_breaches(".gitignore", "", "x", "*\n\n!a\n!b\n"), [])
|
|
|
|
def test_a_block_unsorted_before_the_edit_is_left_alone(self):
|
|
self.assertEqual(file_breaches(".gitignore", "node_modules\n.env\n\ndist\n", "out", "node_modules\n.env\n\ndist\nout\n"), [])
|
|
|
|
def test_an_attribution_in_any_file_breaks(self):
|
|
self.assertEqual(rules(file_breaches("notes.md", "", TRAILER, TRAILER)), ["no-ai-attribution"])
|
|
|
|
|
|
class Overrides(unittest.TestCase):
|
|
def test_a_marker_names_a_known_rule_and_a_reason(self):
|
|
self.assertEqual(overrides("x # rule-override image-pin: vendor ships floating tags only\n# rule-override nope: x\n# rule-override action-pin:"), ({"image-pin": "vendor ships floating tags only"}, ["rule-override nope: x", "rule-override action-pin: "]))
|
|
|
|
def test_a_marker_for_a_rule_that_takes_no_override_is_neither_granted_nor_malformed(self):
|
|
self.assertEqual(overrides("# rule-override reviewer-rung: trust me"), ({}, []))
|
|
self.assertEqual(overrides("# rule-override reviewer-rung: trust me\n# rule-override image-pin: a # rule-override nope: b"), ({"image-pin": "a"}, ["rule-override nope: b"]))
|
|
|
|
def test_two_markers_on_a_line_are_read_apart(self):
|
|
self.assertEqual(overrides("git x # rule-override image-pin: a # rule-override nope: b")[1], ["rule-override nope: b"])
|
|
|
|
def test_markdown_takes_an_html_comment(self):
|
|
self.assertEqual(overrides("<!-- rule-override no-ai-attribution: quotes the banned line -->")[0], {"no-ai-attribution": "quotes the banned line"})
|
|
|
|
def test_a_blocked_call_overrides_nothing_yet(self):
|
|
breaches = [Breach("image-pin", "x", "/r"), Breach("action-pin", "y", "/r")]
|
|
blocked_call = check(breaches, "# rule-override image-pin: a", False, False)
|
|
self.assertTrue(blocked_call.blocked)
|
|
passed = check(breaches[:1], "# rule-override image-pin: a", False, False)
|
|
self.assertEqual((passed.blocked, passed.overridden), (False, [(breaches[0], "a")]))
|
|
|
|
def test_a_markdown_file_is_told_the_html_comment(self):
|
|
self.assertIn("`<!-- rule-override no-ai-attribution: <reason> -->`", check([Breach("no-ai-attribution", "x", "/r")], "", True, False).messages[0])
|
|
|
|
|
|
class OverrideLog(unittest.TestCase):
|
|
def test_an_entry_reads_back_as_written(self):
|
|
entry = LogLine("2026-10-13T00:00:00Z", "chunk", "c1", "image-pin", "the vendor\ttags no patch", "Dockerfile")
|
|
self.assertEqual(parsed_entry(entry.line()), LogLine("2026-10-13T00:00:00Z", "chunk", "c1", "image-pin", "the vendor tags no patch", "Dockerfile"))
|
|
self.assertIsNone(parsed_entry("a\tb"))
|
|
|
|
def test_an_override_is_the_chunk_s_by_its_reflog_and_name(self):
|
|
on_default = {"base"}.__contains__
|
|
cases = [("chunk", "c1", True), ("other", "c1", True), ("other", "c9", False), ("chunk", "base", True), ("other", "base", False), ("other", "", True)]
|
|
for branch, commit, expected in cases:
|
|
with self.subTest(branch=branch, commit=commit):
|
|
self.assertEqual(this_chunks(LogLine("t", branch, commit, "image-pin", "r", "w"), {"base", "c1"}, "chunk", on_default), expected)
|
|
|
|
|
|
# A home that holds no project config, so a review launched there is checked on the ladder this setup ships.
|
|
HOME = "/h"
|
|
|
|
|
|
class ReviewerRung(unittest.TestCase):
|
|
# A worker routed to Sonnet 5.5 at high, on a project whose rung is Opus 5.5 at medium, with its reviewers on the project's rung.
|
|
ENVIRON = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-sonnet-5-5 at high effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"}
|
|
# The same project, with a worker, and so its reviewers, routed above it to Fable 5.2 at max.
|
|
ABOVE = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"}
|
|
|
|
def refused(self, given: dict, environ: dict[str, str] = ENVIRON) -> list[str]:
|
|
return rules(reviewer_breaches(Call("Agent", given, "/r"), environ, HOME))
|
|
|
|
def test_a_branch_changing_only_markdown_reviews_on_the_projects_rung_whatever_the_e_plus_i(self):
|
|
repo = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", repo])
|
|
git(repo, "init", "-q", "-b", "main")
|
|
git(repo, "commit", "-q", "--allow-empty", "-m", "base")
|
|
git(repo, "checkout", "-q", "-b", "chunk")
|
|
pathlib.Path(repo, "docs").mkdir()
|
|
pathlib.Path(repo, "README.md").write_text("Read me.\n")
|
|
pathlib.Path(repo, "docs", "decisions.md").write_text("Decided.\n")
|
|
git(repo, "add", ".")
|
|
git(repo, "commit", "-q", "-m", "prose")
|
|
high = self.ENVIRON | {"LOOP_CLAUDE_ESCAPE_AND_IMPACT": "7"}
|
|
on_project = {"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), [])
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", {**on_project, "model": "sonnet", "effort": "high"}, repo), high, HOME)), ["reviewer-rung"])
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), self.ABOVE, HOME)), [], "a worker that climbed doesn't lift a Markdown-only review either")
|
|
for instructions in ("CLAUDE.md", "AGENTS.md", "skills/review/SKILL.md", "agents/reviewer.md", "rules/todo.md", "principles/tests.md", "workflow-bench/run-prompt.md"):
|
|
with self.subTest(instructions=instructions):
|
|
path = pathlib.Path(repo, instructions)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("Do this.\n")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"], "Markdown an agent follows is config")
|
|
path.unlink()
|
|
pathlib.Path(repo, "gate.yml").write_text("on: push\n")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"], "an uncommitted change to another file counts")
|
|
git(repo, "add", ".")
|
|
git(repo, "commit", "-q", "-m", "config")
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"])
|
|
|
|
def test_a_reviewer_below_the_project_rung_is_refused_naming_the_arguments_to_pass(self):
|
|
breaches = reviewer_breaches(Call("Agent", {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review."}, "/r"), self.ENVIRON, HOME)
|
|
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
|
self.assertIn("stability-reviewer", breaches[0].detail)
|
|
self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail)
|
|
self.assertIn("model opus, effort medium", breaches[0].detail)
|
|
|
|
def test_a_reviewer_on_the_project_rung_or_a_rerun_above_it_passes(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus", "effort": "medium"}), [])
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "fable", "effort": "low"}), [])
|
|
|
|
def test_an_agent_calls_prompt_is_the_text_scanned_for_a_marker(self):
|
|
call = Call("Agent", {"subagent_type": "prose-reviewer", "prompt": "Review. # rule-override reviewer-rung: trust me"}, "/r")
|
|
self.assertEqual(breaches_of(call, "/h", {})[0], "Review. # rule-override reviewer-rung: trust me")
|
|
|
|
def test_a_call_naming_no_model_runs_on_the_workers_rung(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "prompt": "Review."}), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "effort": "medium"}), ["reviewer-rung"])
|
|
|
|
def test_a_model_named_without_an_effort_runs_on_its_cheapest_rung(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus"}), [])
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "sonnet"}), ["reviewer-rung"])
|
|
|
|
def test_an_effort_off_the_ladder_for_its_model_is_refused(self):
|
|
self.assertEqual(self.refused({"subagent_type": "prose-reviewer", "model": "opus", "effort": "low"}), ["reviewer-rung"])
|
|
|
|
def test_a_reviewer_is_held_to_the_rung_its_e_plus_i_picks_where_that_is_higher(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, self.ABOVE), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "fable", "effort": "max"}, self.ABOVE), [])
|
|
lifted = self.ENVIRON | {"LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, lifted), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "opus", "effort": "medium"}, lifted), [])
|
|
|
|
def test_holds_a_reviewer_to_the_higher_of_the_workers_and_the_projects_rung_where_loop_claude_handed_no_e_plus_i(self):
|
|
without = {name: rung for name, rung in self.ABOVE.items() if name != "LOOP_CLAUDE_ESCAPE_AND_IMPACT"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"}, without), ["reviewer-rung"])
|
|
below = {name: rung for name, rung in self.ENVIRON.items() if name != "LOOP_CLAUDE_ESCAPE_AND_IMPACT"}
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high"}, below), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "sonnet", "effort": "high"}, without), ["reviewer-rung"])
|
|
|
|
def test_a_seat_or_a_security_launch_is_held_to_the_project_rung_alone(self):
|
|
seat = {"prompt": "You are one seat of a comprehension panel. Tools: Read, Grep, Glob.", "model": "opus", "effort": "medium"}
|
|
self.assertEqual(self.refused(seat, self.ABOVE), [])
|
|
self.assertEqual(self.refused({"prompt": "You are the security pass for this chunk. Invoke /security-review over the branch diff.", "model": "sonnet", "effort": "high"}), ["reviewer-rung"])
|
|
self.assertEqual(self.refused({"prompt": "You are the security pass for this chunk. Run /claude-security scan this branch's changes at high.", "model": "opus", "effort": "medium"}), [])
|
|
|
|
def test_a_seat_named_below_the_project_rung_is_refused(self):
|
|
self.assertEqual(self.refused({"prompt": "You are one seat of a comprehension panel.", "model": "sonnet", "effort": "high"}), ["reviewer-rung"])
|
|
|
|
def test_any_other_subagent_keeps_its_own_routing(self):
|
|
self.assertEqual(self.refused({"subagent_type": "Explore", "model": "haiku", "effort": "low", "prompt": "Find the callers of run_worker."}), [])
|
|
self.assertEqual(self.refused({"prompt": "Note that security-review runs in CI.", "model": "haiku", "effort": "low"}), [])
|
|
self.assertEqual(self.refused({"prompt": "Find where /security-review is named.", "model": "haiku", "effort": "low"}), [], "a mention without the opening sentence is no launch")
|
|
self.assertEqual(self.refused({"prompt": "Quote the rule: 'You are the security pass for this chunk.' Then stop.", "model": "haiku", "effort": "low"}), [], "a quote midway is no launch")
|
|
|
|
def test_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
|
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "haiku", "effort": "low"}, {}), [])
|
|
|
|
def project(self, ladder: str) -> tuple[str, str]:
|
|
"""A checkout whose config's folder holds ladder as its routing ladder, and its home."""
|
|
home = pathlib.Path(tempfile.mkdtemp()).resolve()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", str(home)])
|
|
checkout = home / "code" / "acme"
|
|
checkout.mkdir(parents=True)
|
|
(configured(self, checkout, home=home) / "routing-ladder.json").write_text(ladder)
|
|
return str(checkout), str(home)
|
|
|
|
# A project that runs on Fable 5.2 at low and at high alone, with a worker on its rung and an E + I of 5, which picks the rung one up.
|
|
TWO_RUNGS = '[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "high"}]'
|
|
ON_TWO_RUNGS = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-fable-5-2 at low effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at low effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5"}
|
|
|
|
def test_a_review_is_held_to_the_rung_picked_on_the_projects_own_ladder(self):
|
|
checkout, home = self.project(self.TWO_RUNGS)
|
|
reviewer = {"subagent_type": "stability-reviewer", "model": "fable"}
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", {**reviewer, "effort": "high"}, checkout), self.ON_TWO_RUNGS, home)), [])
|
|
# One up from Fable 5.2 at low is its medium effort on the shipped ladder, and its high effort on the project's, which holds no medium.
|
|
breaches = reviewer_breaches(Call("Agent", {**reviewer, "effort": "medium"}, checkout), self.ON_TWO_RUNGS, home)
|
|
self.assertEqual([breach.detail for breach in breaches], ["stability-reviewer would run on claude-fable-5-2 at medium effort, below claude-fable-5-2 at high effort: launch it with model fable, effort high"])
|
|
self.assertEqual(rules(reviewer_breaches(Call("Agent", {**reviewer, "effort": "medium"}, "/r"), self.ON_TWO_RUNGS, HOME)), [])
|
|
self.assertEqual(rules(breaches_of(Call("Agent", {**reviewer, "effort": "medium"}, checkout), home, self.ON_TWO_RUNGS)[2]), ["reviewer-rung"], "the hook reads the ladder under the home it is given")
|
|
|
|
def test_a_rung_loop_claude_handed_that_the_projects_ladder_does_not_hold_is_not_checked(self):
|
|
# As when the ladder's file changed after loop-claude started the worker: the next cycle routes on the new ladder.
|
|
checkout, home = self.project(self.TWO_RUNGS)
|
|
reviewer = Call("Agent", {"subagent_type": "stability-reviewer", "model": "haiku", "effort": "low"}, checkout)
|
|
for name in ("LOOP_CLAUDE_PROJECT_RUNG", "LOOP_CLAUDE_WORKER_RUNG"):
|
|
with self.subTest(name=name):
|
|
environ = self.ON_TWO_RUNGS | {name: "claude-opus-5-5 at medium effort"}
|
|
self.assertEqual(reviewer_breaches(reviewer, environ, home), [])
|
|
self.assertEqual(skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), environ, home), [])
|
|
|
|
def test_a_security_skill_is_held_to_the_projects_rung_on_the_projects_own_ladder(self):
|
|
checkout, home = self.project('[{"model": "claude-fable-5-2", "effort": "high"}, {"model": "claude-fable-5-2", "effort": "low"}]')
|
|
# On this ladder the worker's low effort ranks above the project's high one; on the shipped ladder it ranks below.
|
|
environ = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-fable-5-2 at high effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at low effort"}
|
|
self.assertEqual(skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), environ, home), [])
|
|
self.assertEqual(rules(skill_breaches(Call("Skill", {"skill": "security-review"}, "/r"), environ, HOME)), ["reviewer-rung"])
|
|
self.assertEqual(rules(breaches_of(Call("Skill", {"skill": "security-review"}, "/r"), home, environ)[2]), ["reviewer-rung"])
|
|
self.assertEqual(breaches_of(Call("Skill", {"skill": "security-review"}, checkout), home, environ)[2], [])
|
|
on_the_projects = environ | {"LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at high effort"}
|
|
self.assertEqual(skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), on_the_projects, home), [], "a worker on the project's rung runs the skill itself")
|
|
|
|
def test_a_review_is_refused_where_the_projects_ladder_cannot_be_read_and_loop_claude_handed_rungs(self):
|
|
checkout, home = self.project("[]")
|
|
own = pathlib.Path(home, ".claude-projects", "acme", "routing-ladder.json")
|
|
reviewer = Call("Agent", {"subagent_type": "stability-reviewer", "model": "fable", "effort": "max"}, checkout)
|
|
for breaches in (reviewer_breaches(reviewer, self.ON_TWO_RUNGS, home), skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), self.ON_TWO_RUNGS, home)):
|
|
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
|
self.assertTrue(breaches[0].detail.startswith(f"the rung this review needs cannot be worked out: {own}: it holds no list of rungs. "), breaches[0].detail)
|
|
self.assertEqual(breaches[0].directory, checkout)
|
|
self.assertEqual(reviewer_breaches(reviewer, {}, home), [], "a claude loop-claude did not start is not checked")
|
|
self.assertEqual(reviewer_breaches(Call("Agent", {"subagent_type": "Explore"}, checkout), self.ON_TWO_RUNGS, home), [], "nor is a subagent that reviews nothing")
|
|
self.assertEqual(skill_breaches(Call("Skill", {"skill": "simplify"}, checkout), self.ON_TWO_RUNGS, home), [])
|
|
|
|
def skill_refused(self, given: dict, environ: dict[str, str] = ENVIRON, subagent_prompt: str | list | None = None) -> list[Breach]:
|
|
transcript = ""
|
|
if subagent_prompt is not None:
|
|
transcript = os.path.join(tempfile.mkdtemp(), "agent-a1.jsonl")
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", os.path.dirname(transcript)])
|
|
pathlib.Path(transcript).write_text(json.dumps({"type": "user", "message": {"role": "user", "content": subagent_prompt}}) + "\n")
|
|
return skill_breaches(Call("Skill", given, "/r", "general-purpose" if subagent_prompt is not None else "", transcript, subagent_prompt is not None), environ, HOME)
|
|
|
|
def test_a_security_skill_from_the_sessions_own_agent_is_refused_below_the_projects_rung(self):
|
|
for skill in ("/security-review", "claude-security", "claude-security:claude-security", "security-review"):
|
|
with self.subTest(skill=skill):
|
|
breaches = self.skill_refused({"skill": skill})
|
|
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
|
self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail)
|
|
self.assertIn("model opus, effort medium", breaches[0].detail)
|
|
self.assertIn("You are the security pass for this chunk.", breaches[0].detail)
|
|
self.assertEqual(rules(self.skill_refused({"skill": skill}, self.ABOVE)), [])
|
|
|
|
def test_a_security_skill_runs_on_its_callers_rung_whatever_model_its_input_names(self):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review", "model": "opus", "effort": "medium"})), ["reviewer-rung"])
|
|
|
|
def test_a_security_skill_passes_only_in_a_subagent_launched_as_the_security_pass(self):
|
|
for prompt in ("You are the security pass for this chunk. Invoke /security-review.", "\n You are the security pass for this chunk.", [{"type": "text", "text": "You are the security pass for this chunk."}, {"type": "image", "source": {}}]):
|
|
with self.subTest(prompt=prompt):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ENVIRON, prompt)), [])
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ABOVE, "Invoke /security-review.")), ["reviewer-rung"], "a subagent's rung is unknown, whatever the worker's")
|
|
unreadable = skill_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose", "/nonexistent/agent-a1.jsonl", True), self.ENVIRON, HOME)
|
|
self.assertEqual(rules(unreadable), ["reviewer-rung"])
|
|
|
|
def test_any_other_skill_or_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
|
self.assertEqual(rules(self.skill_refused({"skill": "simplify"})), [])
|
|
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, {})), [])
|
|
|
|
def test_a_session_started_with_an_agent_is_no_subagent(self):
|
|
hook_input = {"tool_name": "Skill", "tool_input": {"skill": "security-review"}, "cwd": "/r", "agent_type": "claude-security:claude-security", "transcript_path": "/t/s.jsonl"}
|
|
self.assertFalse(parsed_call(hook_input).subagent)
|
|
self.assertTrue(parsed_call({**hook_input, "agent_id": "a1"}).subagent)
|
|
|
|
def test_a_skill_calls_args_are_the_text_scanned_for_a_marker(self):
|
|
found = breaches_of(Call("Skill", {"skill": "security-review", "args": "the branch # rule-override reviewer-rung: trust me"}, "/r"), "/h", {})
|
|
self.assertEqual(found[:2], ("the branch # rule-override reviewer-rung: trust me", "security-review"))
|
|
|
|
def test_settings_run_the_hook_before_each_skill_call(self):
|
|
settings = json.loads(pathlib.Path(__file__).resolve().parents[2].joinpath("settings.json").read_text())
|
|
matchers = [entry["matcher"].split("|") for entry in settings["hooks"]["PreToolUse"]]
|
|
self.assertTrue(any("Skill" in tools for tools in matchers))
|
|
|
|
|
|
class EnvFileRead(unittest.TestCase):
|
|
"""A home with ~/.claude/.env and a config whose env-file lies outside any checkout."""
|
|
def setUp(self) -> None:
|
|
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
(self.home / ".claude").mkdir()
|
|
(self.home / ".claude" / ".env").write_text("TOKEN=x\n")
|
|
write_config(self.home, self.home / "code" / "acme")
|
|
config = self.home / ".claude-projects" / "acme" / "config.toml"
|
|
config.write_text(config.read_text() + 'env-file = "~/secrets/acme.env"\n')
|
|
(self.home / "secrets").mkdir()
|
|
(self.home / "secrets" / "acme.env").write_text("TOKEN=y\n")
|
|
|
|
def breached(self, tool: str, given: dict[str, typing.Any], cwd: str = "/r") -> list[str]:
|
|
found = breaches_of(Call(tool, given, cwd), str(self.home), {})
|
|
return sorted({rule for rule in rules(found[2]) if rule == "env-file-read"}) if found else []
|
|
|
|
def test_a_read_of_claude_s_env_or_a_config_s_env_file_is_refused(self):
|
|
self.assertEqual(self.breached("Read", {"file_path": f"{self.home}/.claude/.env"}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Read", {"file_path": "acme.env"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
|
|
def test_a_read_spelt_from_home_is_refused_naming_the_file(self):
|
|
found = breaches_of(Call("Read", {"file_path": "~/.claude/.env"}, "/r"), str(self.home), {})
|
|
self.assertEqual([breach.detail for breach in found[2]], [f"the call reads {self.home}/.claude/.env, whose credentials would land in the transcript"])
|
|
|
|
def test_the_log_names_a_read_by_its_file_and_a_command_by_its_first_line(self):
|
|
self.assertEqual(breaches_of(Call("Read", {"file_path": "a.txt"}, "/r"), str(self.home), {})[1], "a.txt")
|
|
self.assertEqual(breaches_of(Call("Bash", {"command": "ls -l\nls -a\nls"}, "/r"), str(self.home), {})[1], "ls -l")
|
|
|
|
def test_a_read_through_a_symlink_is_refused(self):
|
|
(self.home / "link").symlink_to(self.home / "secrets" / "acme.env")
|
|
self.assertEqual(self.breached("Read", {"file_path": f"{self.home}/link"}), ["env-file-read"])
|
|
|
|
def test_a_read_of_another_file_passes(self):
|
|
self.assertEqual(self.breached("Read", {"file_path": f"{self.home}/.claude/settings.json"}), [])
|
|
|
|
def test_a_grep_of_the_file_or_of_a_folder_holding_it_is_refused(self):
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": f"{self.home}/secrets/acme.env"}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": str(self.home)}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": "~/secrets"}), ["env-file-read"])
|
|
|
|
def test_a_grep_of_a_folder_whose_checkout_ignores_the_file_or_of_another_folder_passes(self):
|
|
subprocess.run(["git", "init", "--quiet", str(self.home / ".claude")], check=True, env=isolated_env())
|
|
(self.home / ".claude" / ".gitignore").write_text(".env\n")
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": f"{self.home}/.claude"}), [])
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": f"{self.home}/code"}), [])
|
|
|
|
def test_a_command_naming_the_file_is_refused(self):
|
|
for command in ("cat ~/.claude/.env", "sh -c 'cat ~/.claude/.env'", f"sh -c 'cat {self.home}/secrets/acme.env'", f"grep TOKEN {self.home}/secrets/acme.env", "source $HOME/secrets/acme.env", "wc < ${HOME}/.claude/.env", "cd ~/secrets && cat acme.env"):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), ["env-file-read"])
|
|
|
|
def test_a_command_naming_another_file_passes(self):
|
|
self.assertEqual(self.breached("Bash", {"command": f"cat ~/.claude/.envrc /backup{self.home}/.claude/.env .env"}, "/r"), [])
|
|
self.assertEqual(self.breached("Bash", {"command": "cat 'a\0b'"}, f"{self.home}/.claude"), [])
|
|
|
|
def test_a_search_s_refusal_says_it_searches_a_folder_holding_the_file_and_what_to_do(self):
|
|
found = breaches_of(Call("Bash", {"command": "rg TOKEN ~/secrets"}, "/r"), str(self.home), {})
|
|
details = [breach.detail for breach in found[2] if breach.rule == "env-file-read"]
|
|
self.assertEqual(details, [f"the command searches a folder holding {self.home}/secrets/acme.env, a file of credentials that must stay out of the transcript; search a folder without it"])
|
|
|
|
def test_a_command_s_refusal_says_it_names_the_file_not_that_it_reads_it(self):
|
|
found = breaches_of(Call("Bash", {"command": "ls -l ~/.claude/.env"}, "/r"), str(self.home), {})
|
|
details = [breach.detail for breach in found[2] if breach.rule == "env-file-read"]
|
|
self.assertEqual(len(details), 1)
|
|
self.assertTrue(details[0].startswith(f"the command names {self.home}/.claude/.env"), details[0])
|
|
self.assertNotIn("reads", details[0])
|
|
|
|
def test_the_rule_names_the_way_on_for_a_command_that_only_mentions_the_path(self):
|
|
holds = RULES["env-file-read"].holds
|
|
self.assertIn('"the .env in ~/.claude"', holds)
|
|
self.assertIn("`~/.claude/bin/project-config env-file`", holds)
|
|
self.assertNotIn(". ", holds, "one sentence")
|
|
|
|
def test_a_command_redirecting_from_the_file_is_refused(self):
|
|
self.assertEqual(self.breached("Bash", {"command": "cd ~/secrets && wc < acme.env"}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "wc < acme.env"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "while read l; do echo $l; done < .env"}, f"{self.home}/.claude"), ["env-file-read"])
|
|
|
|
def test_a_command_naming_the_file_through_a_quoted_home_is_refused(self):
|
|
for command in ('cat "$HOME"/.claude/.env', 'cat "${HOME}"/.claude/.env'):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), ["env-file-read"])
|
|
|
|
def test_a_grep_s_glob_is_refused_only_where_it_can_match_the_ignored_file(self):
|
|
subprocess.run(["git", "init", "--quiet", str(self.home / ".claude")], check=True, env=isolated_env())
|
|
(self.home / ".claude" / ".gitignore").write_text(".env\n")
|
|
for glob, expected in (("**/*.md", []), ("rules/*.md", []), ("*.py", []), ("!.env", []), ("*", ["env-file-read"]), (".env*", ["env-file-read"]), ("?env", ["env-file-read"]), ("?nv", []), ("{*.md,.env}", ["env-file-read"])):
|
|
with self.subTest(glob):
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": f"{self.home}/.claude", "glob": glob}), expected)
|
|
|
|
def test_a_grep_s_glob_with_a_slash_is_matched_against_the_path_from_the_searched_folder(self):
|
|
subprocess.run(["git", "init", "--quiet", str(self.home / ".claude")], check=True, env=isolated_env())
|
|
(self.home / ".claude" / ".gitignore").write_text(".env\n")
|
|
# ~/secrets/acme.env is in no checkout, so only the ignored file's breach tells the globs apart.
|
|
for glob, refused in ((".claude/*", True), ("**/.env", True), ("/.claude/.env", True), (".env", True), (".claude/**", True), ("secrets/*", False), ("*/*.md", False), (".env/*", False)):
|
|
with self.subTest(glob):
|
|
found = breaches_of(Call("Grep", {"pattern": "TOKEN", "path": str(self.home), "glob": glob}, "/r"), str(self.home), {})
|
|
self.assertEqual(any(breach.detail.startswith(f"the call reads {self.home}/.claude/.env,") for breach in found[2]), refused)
|
|
|
|
def ignoring(self, folder: str, patterns: str) -> None:
|
|
"""Make folder a checkout whose .gitignore holds patterns."""
|
|
subprocess.run(["git", "init", "--quiet", str(self.home / folder)], check=True, env=isolated_env())
|
|
(self.home / folder / ".gitignore").write_text(patterns)
|
|
|
|
def test_a_recursive_search_of_a_folder_holding_the_file_is_refused_for_each_tool(self):
|
|
commands = (
|
|
"grep -r TOKEN ~/secrets", "grep -R TOKEN ~/secrets", "grep -rn TOKEN $HOME/secrets", "egrep -ri TOKEN ~/secrets", "fgrep --recursive TOKEN ~/secrets",
|
|
"rg TOKEN ~/secrets", "rg -n TOKEN ~/secrets", "ag TOKEN ~/secrets", "nice rg TOKEN ~/secrets",
|
|
"rg -e TOKEN ~/secrets", "rg -A 3 TOKEN ~/secrets", "cat x | rg -g '*.env' TOKEN ~/secrets",
|
|
"grep -d recurse TOKEN ~/secrets", "grep -nd recurse TOKEN ~/secrets", "grep --directories recurse TOKEN ~/secrets", "grep -drecurse TOKEN ~/secrets", "rg -eTOKEN ~/secrets", "rg --regexp=TOKEN ~/secrets", "grep -rf patterns ~/secrets", "rg TOKEN -- ~/secrets", "rg -- TOKEN ~/secrets", "ag -g '*.md' TOKEN ~/secrets",
|
|
"grep TOKEN x; rg TOKEN ~/secrets", "cd ~/code && rg TOKEN ~/secrets", "rg -Tgo TOKEN ~/secrets", "rg -rgone TOKEN ~/secrets", "grep -rm1 TOKEN ~/secrets", "grep -rC2 TOKEN ~/secrets", "rg -e1 -nA3 TOKEN ~/secrets", "echo hi && ag TOKEN ~/secrets",
|
|
)
|
|
for command in commands:
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), ["env-file-read"])
|
|
|
|
def test_a_recursive_search_with_no_folder_searches_the_directory_cd_left_it_in(self):
|
|
self.assertEqual(self.breached("Bash", {"command": "grep -r TOKEN"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "cd ~/secrets && rg TOKEN"}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "cd ~/secrets && grep -rn -A 3 TOKEN"}), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "ag TOKEN"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "grep -rnA3 TOKEN"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
# patterns is a file the cwd holds, given as the value of -f.
|
|
(self.home / "secrets" / "patterns").write_text("TOKEN\n")
|
|
self.assertEqual(self.breached("Bash", {"command": "rg -f patterns"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "grep -rf patterns"}, f"{self.home}/secrets"), ["env-file-read"])
|
|
|
|
def test_a_search_of_another_folder_or_a_grep_that_is_not_recursive_passes(self):
|
|
(self.home / "code").mkdir()
|
|
for command in ("grep -r TOKEN ~/code", "rg TOKEN ~/code", "ag TOKEN ~/code", "grep TOKEN ~/secrets", "grep -n TOKEN ~/secrets/*", "echo rg TOKEN ~/secrets", "rg --version", "grep -n recurse notes.md"):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), [])
|
|
self.assertEqual(self.breached("Bash", {"command": "rg TOKEN"}, f"{self.home}/code"), [])
|
|
# The word recurse is only a pattern or a file name here, and -d read is no recursion.
|
|
for command in ("grep -n recurse notes.md", "grep -d read TOKEN ~/secrets", "grep --directories read TOKEN ~/secrets"):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}, str(self.home)), [])
|
|
# secrets is the pattern, not the folder ~/secrets that the home directory holds.
|
|
self.assertEqual(self.breached("Bash", {"command": "rg secrets ~/code"}, str(self.home)), [])
|
|
|
|
def test_a_folder_that_starts_with_a_dash_is_searched_after_double_dash(self):
|
|
write_config(self.home, self.home / "code" / "other")
|
|
config = self.home / ".claude-projects" / "other" / "config.toml"
|
|
config.write_text(config.read_text() + 'env-file = "~/real/x.env"\n')
|
|
(self.home / "real").mkdir()
|
|
(self.home / "real" / "x.env").write_text("TOKEN=z\n")
|
|
(self.home / "sub").mkdir()
|
|
# The link keeps ~/sub clear of the file, so only the operand -d reaches it.
|
|
(self.home / "sub" / "-d").symlink_to(self.home / "real")
|
|
self.assertEqual(self.breached("Bash", {"command": "rg TOKEN -- -d"}, f"{self.home}/sub"), ["env-file-read"])
|
|
self.assertEqual(self.breached("Bash", {"command": "rg TOKEN"}, f"{self.home}/sub"), [])
|
|
|
|
def test_rg_and_ag_skip_hidden_files_but_grep_does_not(self):
|
|
for command, expected in (("rg TOKEN ~/.claude", []), ("ag TOKEN ~/.claude", []), ("grep -r TOKEN ~/.claude", ["env-file-read"]), ("rg --hidden TOKEN ~/.claude", ["env-file-read"]), ("rg -. TOKEN ~/.claude", ["env-file-read"]), ("rg -uu TOKEN ~/.claude", ["env-file-read"]), ("rg -u -u TOKEN ~/.claude", ["env-file-read"]), ("rg --unrestricted TOKEN ~/.claude", []), ("rg --unrestricted --unrestricted TOKEN ~/.claude", ["env-file-read"]), ("ag --hidden TOKEN ~/.claude", ["env-file-read"]), ("ag -u TOKEN ~/.claude", ["env-file-read"])):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), expected)
|
|
|
|
def test_a_search_that_starts_in_a_hidden_folder_reaches_a_visible_file_below_it(self):
|
|
write_config(self.home, self.home / "code" / "other")
|
|
config = self.home / ".claude-projects" / "other" / "config.toml"
|
|
config.write_text(config.read_text() + 'env-file = "~/.claude-projects/other.env"\n')
|
|
(self.home / ".claude-projects" / "other.env").write_text("TOKEN=z\n")
|
|
for command, expected in (("rg TOKEN ~/.claude-projects", ["env-file-read"]), ("ag TOKEN ~/.claude-projects", ["env-file-read"]), ("grep -r TOKEN ~/.claude-projects", ["env-file-read"])):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), expected)
|
|
(self.home / "code").mkdir()
|
|
self.assertEqual(self.breached("Bash", {"command": "rg TOKEN ~/code"}), [])
|
|
|
|
def test_rg_and_ag_skip_a_file_below_any_hidden_folder_of_the_search(self):
|
|
write_config(self.home, self.home / "code" / "other")
|
|
config = self.home / ".claude-projects" / "other" / "config.toml"
|
|
config.write_text(config.read_text() + 'env-file = "~/outer/deep/.h/b.env"\n')
|
|
(self.home / "outer" / "deep" / ".h").mkdir(parents=True)
|
|
(self.home / "outer" / "deep" / ".h" / "b.env").write_text("TOKEN=z\n")
|
|
for command, expected in (("rg TOKEN ~/outer", []), ("ag TOKEN ~/outer", []), ("rg --hidden TOKEN ~/outer", ["env-file-read"]), ("grep -r TOKEN ~/outer", ["env-file-read"]), ("rg TOKEN ~/outer/deep/.h", ["env-file-read"])):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), expected)
|
|
|
|
def test_rg_and_ag_skip_an_ignored_file_unless_an_option_or_a_glob_lifts_it(self):
|
|
self.ignoring("secrets", "*.env\n")
|
|
for command, expected in (
|
|
("rg TOKEN ~/secrets", []), ("ag TOKEN ~/secrets", []), ("grep -r TOKEN ~/secrets", ["env-file-read"]),
|
|
("rg --no-ignore TOKEN ~/secrets", ["env-file-read"]), ("rg -u TOKEN ~/secrets", ["env-file-read"]), ("rg --no-ignore-vcs TOKEN ~/secrets", ["env-file-read"]),
|
|
("ag -U TOKEN ~/secrets", ["env-file-read"]), ("ag --skip-vcs-ignores TOKEN ~/secrets", ["env-file-read"]), ("ag -u TOKEN ~/secrets", ["env-file-read"]),
|
|
("rg -g '*.env' TOKEN ~/secrets", ["env-file-read"]), ("rg --glob=acme.env TOKEN ~/secrets", ["env-file-read"]), ("rg --iglob '*.env' TOKEN ~/secrets", ["env-file-read"]), ("ag -a TOKEN ~/secrets", ["env-file-read"]), ("ag -t TOKEN ~/secrets", ["env-file-read"]), ("ag --all-text TOKEN ~/secrets", ["env-file-read"]), ("ag --all-types TOKEN ~/secrets", ["env-file-read"]), ("rg -uu TOKEN ~/secrets", ["env-file-read"]),
|
|
("rg -g '*.md' TOKEN ~/secrets", []), ("rg -g '!*.env' TOKEN ~/secrets", []),
|
|
):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), expected)
|
|
|
|
def test_rg_searches_only_the_files_a_positive_glob_matches(self):
|
|
for command, expected in (("rg -g '*.md' TOKEN ~/secrets", []), ("rg -g '*.md' -g '*.env' TOKEN ~/secrets", ["env-file-read"]), ("rg -g '!*.md' TOKEN ~/secrets", ["env-file-read"]), ("rg --glob '*.md' TOKEN ~/secrets", []), ("rg --iglob '*.ENV' TOKEN ~/secrets", ["env-file-read"]), ("rg --glob-case-insensitive -g '*.ENV' TOKEN ~/secrets", ["env-file-read"]), ("rg TOKEN ~/secrets -g", ["env-file-read"]), ("rg TOKEN ~/secrets -g '*.env'", ["env-file-read"]), ("rg TOKEN ~/secrets -g '*.md'", [])):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}), expected)
|
|
|
|
def test_a_path_glob_of_rg_lifts_the_skip_since_rg_anchors_it_at_the_directory_it_runs_in(self):
|
|
self.ignoring("secrets", "*.env\n")
|
|
for command in ("rg -g 'secrets/*.env' TOKEN secrets", "rg -g 'secrets/*.env' TOKEN ~/secrets", "rg -g '/secrets/*.env' TOKEN secrets"):
|
|
with self.subTest(command):
|
|
self.assertEqual(self.breached("Bash", {"command": command}, str(self.home)), ["env-file-read"])
|
|
|
|
def test_a_search_s_breach_is_logged_in_the_directory_the_command_runs_in(self):
|
|
found = breaches_of(Call("Bash", {"command": "grep -r TOKEN ~/secrets"}, "/r"), str(self.home), {})
|
|
self.assertEqual([breach.directory for breach in found[2] if breach.rule == "env-file-read"], ["/r"])
|
|
|
|
def test_a_file_several_configs_name_is_reported_once(self):
|
|
write_config(self.home, self.home / "code" / "other")
|
|
config = self.home / ".claude-projects" / "other" / "config.toml"
|
|
config.write_text(config.read_text() + 'env-file = "~/secrets/../secrets/acme.env"\n')
|
|
found = breaches_of(Call("Read", {"file_path": "~/secrets/acme.env"}, "/r"), str(self.home), {})
|
|
self.assertEqual(len([breach for breach in found[2] if breach.rule == "env-file-read"]), 1)
|
|
|
|
def test_a_grep_of_a_path_holding_a_nul_byte_passes(self):
|
|
self.assertEqual(self.breached("Grep", {"pattern": "TOKEN", "path": "a\0b"}), [])
|
|
|
|
|
|
class ReviewerReport(unittest.TestCase):
|
|
def breached(self, tool: str, path: str, agent: str = "prose-reviewer", cwd: str = "/r") -> list[str]:
|
|
given = {"file_path": path, "content": "BLOCK\n"} if tool == "Write" else {"file_path": path, "old_string": "a", "new_string": "b"}
|
|
found = breaches_of(Call(tool, given, cwd, agent), "/h", {})
|
|
return rules(found[2]) if found else []
|
|
|
|
def test_a_reviewer_writes_its_report_into_the_reports_folder(self):
|
|
self.assertEqual(self.breached("Write", "/h/.local/state/larv-review/.claude-item-73/prose-round-1.md"), [])
|
|
|
|
def test_a_relative_path_resolves_against_the_calls_cwd(self):
|
|
self.assertEqual(self.breached("Write", "prose-round-1.md", cwd="/h/.local/state/larv-review/x"), [])
|
|
|
|
def test_a_symlink_in_the_reports_folder_is_followed_to_where_it_points(self):
|
|
home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
(home / ".local" / "state" / "larv-review").mkdir(parents=True)
|
|
(home / ".local" / "state" / "larv-review" / "out").symlink_to(home)
|
|
found = breaches_of(Call("Write", {"file_path": f"{home}/.local/state/larv-review/out/a.md", "content": "x"}, "/r", "prose-reviewer"), str(home), {})
|
|
self.assertEqual(rules(found[2]), ["reviewer-report"])
|
|
|
|
def test_a_reviewer_writing_anywhere_else_is_refused(self):
|
|
for path in ["/r/README.md", "/h/.local/state/larv-review", "/h/.local/state/larv-review-x/a.md", "/h/.local/state/larv-review/../a.md", "/tmp/larv-review/a.md"]:
|
|
with self.subTest(path):
|
|
self.assertEqual(self.breached("Write", path), ["reviewer-report"])
|
|
|
|
def test_a_reviewer_edits_no_file_even_in_the_reports_folder(self):
|
|
self.assertEqual(self.breached("Edit", "/h/.local/state/larv-review/x/prose-round-1.md"), ["reviewer-report"])
|
|
|
|
def test_a_reviewers_multiedit_is_refused(self):
|
|
self.assertEqual(self.breached("MultiEdit", "/r/a.py", "stability-reviewer"), ["reviewer-report"])
|
|
|
|
def test_the_worker_and_other_subagents_write_anywhere(self):
|
|
self.assertEqual(self.breached("Write", "/r/README.md", ""), [])
|
|
self.assertEqual(self.breached("Write", "/r/README.md", "general-purpose"), [])
|
|
|
|
|
|
class RepeatRead(unittest.TestCase):
|
|
LINES = [f"line {n}" for n in range(1, 31)]
|
|
# The file is written at WRITTEN and each entry stamped READ_AT, unless a test changes the file at CHANGED and stamps a later entry AFTER_THE_CHANGE.
|
|
WRITTEN = datetime.datetime(2026, 5, 1, tzinfo=datetime.UTC).timestamp()
|
|
READ_AT = "2026-06-01T00:00:00.000Z"
|
|
CHANGED = datetime.datetime(2026, 7, 1, tzinfo=datetime.UTC).timestamp()
|
|
AFTER_THE_CHANGE = "2026-07-01T00:00:01.000Z"
|
|
|
|
def setUp(self):
|
|
self.folder = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.folder])
|
|
self.file = f"{self.folder}/f.txt"
|
|
self.write(self.LINES, self.WRITTEN)
|
|
self.transcript = f"{self.folder}/s.jsonl"
|
|
self.entries: list[dict] = []
|
|
|
|
def write(self, lines: list[str], at: float) -> None:
|
|
pathlib.Path(self.file).write_text("\n".join(lines) + "\n")
|
|
os.utime(self.file, (at, at))
|
|
|
|
def change(self, number: int) -> None:
|
|
self.write([("changed" if n == number else line) for n, line in enumerate(self.LINES, 1)], self.CHANGED)
|
|
|
|
def add(self, entry: dict, parent: str | None = "last") -> str:
|
|
uuid = f"u{len(self.entries)}"
|
|
parent = (self.entries[-1]["uuid"] if self.entries else None) if parent == "last" else parent
|
|
self.entries.append({"uuid": uuid, "parentUuid": parent, "cwd": self.folder, "timestamp": self.READ_AT} | entry)
|
|
pathlib.Path(self.transcript).write_text("\n".join(json.dumps(entry) for entry in self.entries) + "\n")
|
|
return uuid
|
|
|
|
def used(self, tool: str, given: dict, result: str, error: bool = False, at: str = READ_AT) -> None:
|
|
tool_id = f"toolu_{len(self.entries)}"
|
|
self.add({"type": "assistant", "timestamp": at, "message": {"content": [{"type": "tool_use", "id": tool_id, "name": tool, "input": given}]}})
|
|
self.add({"type": "user", "timestamp": at, "message": {"content": [{"type": "tool_result", "tool_use_id": tool_id, "content": result, "is_error": error}]}})
|
|
|
|
def numbered(self, start: int, end: int) -> str:
|
|
return "\n".join(f"{n}\t{self.LINES[n - 1]}" for n in range(start, end + 1))
|
|
|
|
def read(self, start: int, end: int) -> None:
|
|
self.used("Read", {"file_path": self.file, "offset": start, "limit": end - start + 1}, self.numbered(start, end))
|
|
|
|
def shown(self, start: int, end: int) -> str:
|
|
return "\n".join(self.LINES[start - 1:end])
|
|
|
|
def found(self, tool: str, given: dict, agent: str = "") -> list[Breach]:
|
|
return read_breaches(Call(tool, given, self.folder, agent, self.transcript), "/h")
|
|
|
|
def test_a_read_of_lines_already_read_is_blocked_with_a_note(self):
|
|
self.read(1, 10)
|
|
found = self.found("Read", {"file_path": self.file, "offset": 2, "limit": 5})
|
|
self.assertEqual(rules(found), ["repeat-read"])
|
|
self.assertIn(f"lines 2-6 of {self.file} are in your context", found[0].detail)
|
|
|
|
def test_the_note_names_a_single_line_as_one(self):
|
|
self.read(1, 10)
|
|
self.assertIn(f"line 7 of {self.file} is in your context", self.found("Bash", {"command": "sed -n 7p f.txt"})[0].detail)
|
|
|
|
def test_lines_read_in_two_reads_are_in_context(self):
|
|
self.read(1, 10)
|
|
self.read(11, 20)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 5, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_read_past_what_was_read_goes_through(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 5, "limit": 10}), [])
|
|
|
|
def test_a_read_without_a_limit_asks_for_the_whole_file(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file}), [])
|
|
|
|
def test_a_read_result_in_the_older_arrow_format_counts(self):
|
|
self.used("Read", {"file_path": self.file, "offset": 1, "limit": 10}, "\n".join(f"{n:6}→{self.LINES[n - 1]}" for n in range(1, 11)))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_compaction_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.add({"type": "system", "subtype": "compact_boundary"}, parent=None)
|
|
self.add({"type": "user", "message": {"content": "This session is being continued"}})
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_changed_line_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.change(1)
|
|
self.assertEqual(self.found("Bash", {"command": f"head {self.file}"}), [])
|
|
|
|
def test_a_change_in_the_last_asked_line_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
self.change(10)
|
|
self.assertEqual(self.found("Bash", {"command": "sed -n 1,10p f.txt"}), [])
|
|
|
|
def test_lines_that_did_not_change_stay_in_context(self):
|
|
self.read(1, 10)
|
|
self.change(1)
|
|
self.assertEqual(rules(self.found("Bash", {"command": "sed -n 2,10p f.txt"})), ["repeat-read"])
|
|
|
|
def test_a_read_without_an_offset_starts_at_line_1(self):
|
|
self.read(2, 11)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "limit": 10}), [])
|
|
|
|
def test_calls_that_neither_read_nor_write_the_file_are_passed_over(self):
|
|
self.used("Grep", {"pattern": "line", "path": self.folder}, "f.txt")
|
|
self.used("Bash", {"command": "ls"}, "f.txt")
|
|
self.read(1, 10)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
# Claude Code's Edit refuses a file changed since the agent last read it with Read, or wrote it with Edit or Write, until it reads it again.
|
|
def test_a_change_in_the_millisecond_of_the_last_read_counts_as_before_it(self):
|
|
self.read(1, 10)
|
|
at = datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC).timestamp()
|
|
os.utime(self.file, (at, at))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_file_s_change_is_stamped_as_the_transcript_stamps_its_entries(self):
|
|
at = datetime.datetime(2026, 6, 1, 12, 30, 5, 250000, tzinfo=datetime.UTC).timestamp()
|
|
os.utime(self.file, (at, at))
|
|
self.assertEqual(modified(self.file), "2026-06-01T12:30:05.250Z")
|
|
|
|
def test_a_read_goes_through_once_the_file_changed_after_the_last_read(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_refused_edit_does_not_count_as_reading_the_file(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.used("Edit", {"file_path": self.file, "old_string": "line 5", "new_string": "five"}, "<tool_use_error>File has been modified since read, either by the user or by a linter. Read it again before attempting to write it.</tool_use_error>", error=True, at=self.AFTER_THE_CHANGE)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_an_edit_keeps_the_lines_it_left_unchanged_in_context(self):
|
|
self.read(1, 10)
|
|
self.change(30)
|
|
self.used("Edit", {"file_path": self.file, "old_string": "line 30", "new_string": "changed"}, f"The file {self.file} has been updated successfully.", at=self.AFTER_THE_CHANGE)
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10})), ["repeat-read"])
|
|
|
|
def test_a_failed_read_holds_no_lines(self):
|
|
self.used("Read", {"file_path": self.file, "offset": 1, "limit": 10}, self.numbered(1, 10), error=True)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_a_read_on_a_branch_the_session_rewound_holds_no_lines(self):
|
|
start = self.add({"type": "user", "message": {"content": "go"}})
|
|
self.read(1, 10)
|
|
self.add({"type": "user", "message": {"content": "go again"}}, parent=start)
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 1, "limit": 10}), [])
|
|
|
|
def test_head_and_sed_of_lines_read_are_blocked(self):
|
|
self.read(1, 20)
|
|
for command in [f"head -n 5 {self.file}", f"head -n5 {self.file}", f"head -5 {self.file}", f"head {self.file}", "sed -n '3,8p' f.txt", "sed -n 7p f.txt"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(rules(self.found("Bash", {"command": command})), ["repeat-read"])
|
|
|
|
def test_a_head_starts_at_line_1(self):
|
|
self.read(2, 11)
|
|
self.assertEqual(self.found("Bash", {"command": f"head {self.file}"}), [])
|
|
|
|
def test_a_cat_asks_for_the_whole_file(self):
|
|
self.read(1, 20)
|
|
self.assertEqual(self.found("Bash", {"command": f"cat {self.file}"}), [])
|
|
|
|
# Claude Code's Edit and Write take only a Read as having read the file.
|
|
def test_a_read_goes_through_until_the_agent_read_the_file_with_read(self):
|
|
self.used("Bash", {"command": f"sed -n '1,20p' {self.file}"}, self.shown(1, 20))
|
|
self.assertEqual(self.found("Read", {"file_path": self.file, "offset": 4, "limit": 3}), [])
|
|
|
|
def test_a_bash_output_counts_against_a_read_once_the_agent_read_the_file_with_read(self):
|
|
self.read(21, 30)
|
|
self.used("Bash", {"command": f"sed -n '1,20p' {self.file}"}, self.shown(1, 20))
|
|
self.assertEqual(rules(self.found("Read", {"file_path": self.file, "offset": 4, "limit": 3})), ["repeat-read"])
|
|
|
|
def test_a_cat_output_puts_every_line_in_context(self):
|
|
self.used("Bash", {"command": f"cat {self.file}"}, self.shown(1, 30))
|
|
self.assertEqual(rules(self.found("Bash", {"command": "sed -n '25,$p' f.txt"})), ["repeat-read"])
|
|
|
|
def test_a_relative_path_in_the_transcript_is_under_its_entry_s_cwd(self):
|
|
self.used("Bash", {"command": "cat f.txt"}, self.shown(1, 30))
|
|
self.assertEqual(rules(read_breaches(Call("Bash", {"command": f"head -n 5 {self.file}"}, "/", "", self.transcript), "/h")), ["repeat-read"])
|
|
|
|
def test_a_tilde_path_is_under_the_home_folder(self):
|
|
self.used("Read", {"file_path": "~/f.txt", "offset": 1, "limit": 10}, self.numbered(1, 10))
|
|
self.used("Bash", {"command": "cat ~/f.txt"}, self.shown(1, 30))
|
|
for tool, given in [("Read", {"file_path": "~/f.txt", "offset": 1, "limit": 10}), ("Bash", {"command": "sed -n 1,30p ~/f.txt"})]:
|
|
with self.subTest(tool):
|
|
self.assertEqual(rules(read_breaches(Call(tool, given, "/", "", self.transcript), self.folder)), ["repeat-read"])
|
|
|
|
def test_other_commands_on_the_file_are_not_checked(self):
|
|
self.used("Bash", {"command": f"cat {self.file}"}, self.shown(1, 30))
|
|
for command in [f"cat {self.file} > copy", f"cat {self.file} | wc -l", f"cat {self.file} {self.file}", f"head -c 5 {self.file}", f"sed 's/a/b/' {self.file}"]:
|
|
with self.subTest(command):
|
|
self.assertEqual(self.found("Bash", {"command": command}), [])
|
|
|
|
# As Claude Code 2.1.312 writes a Bash output it cut short.
|
|
def test_an_output_cut_short_shows_no_lines(self):
|
|
for cut in [
|
|
f"<persisted-output>\nOutput too large (37.9KB). Full output saved to: {self.folder}/tool-results/b2hymdt7w.txt\n\nPreview (first 2KB):\n{self.shown(1, 30)}\n...\n</persisted-output>",
|
|
f"<truncated-output>\nOutput too large (37.9KB). It could not be saved, so only the first 2KB are shown; the rest was dropped. If the tool can page or filter its results, call it again for the part you need.\n\n{self.shown(1, 30)}\n</truncated-output>",
|
|
self.shown(1, 5) + "\n\n... [25 lines truncated] ...",
|
|
]:
|
|
with self.subTest(cut[:20]):
|
|
self.assertEqual(result_lines("Bash", Asked(self.file, 1, None), cut), {})
|
|
|
|
def test_a_fifo_is_never_opened(self):
|
|
fifo = f"{self.folder}/fifo"
|
|
os.mkfifo(fifo)
|
|
self.used("Bash", {"command": f"cat {fifo}"}, "line 1")
|
|
# Opened, the FIFO would block the hook, since it has no writer.
|
|
ran = hook({"cwd": self.folder, "tool_name": "Bash", "tool_input": {"command": f"head -n 1 {fifo}"}, "transcript_path": self.transcript}, self.folder)
|
|
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
|
|
|
|
def called(self, agent_id: str = "") -> Call:
|
|
given = {"cwd": self.folder, "tool_name": "Read", "tool_input": {"file_path": self.file, "offset": 1, "limit": 10}, "transcript_path": self.transcript}
|
|
return parsed_call(given | ({"agent_id": agent_id, "agent_type": "Explore"} if agent_id else {}))
|
|
|
|
def test_a_subagent_s_transcript_is_beside_the_session_s(self):
|
|
self.assertEqual(self.called("a1").transcript, f"{self.folder}/s/subagents/agent-a1.jsonl")
|
|
|
|
def test_the_session_s_reads_are_not_in_a_subagent_s_context(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(read_breaches(self.called("a1"), "/h"), [])
|
|
|
|
def test_the_session_is_judged_on_its_own_transcript(self):
|
|
self.read(1, 10)
|
|
self.assertEqual(rules(read_breaches(self.called(), "/h")), ["repeat-read"])
|
|
|
|
def test_a_missing_transcript_lets_the_read_through(self):
|
|
self.assertEqual(read_breaches(Call("Read", {"file_path": self.file}, self.folder, "", f"{self.folder}/none.jsonl"), "/h"), [])
|
|
|
|
def test_a_read_of_a_missing_path_or_a_directory_lets_the_read_through(self):
|
|
self.read(1, 10)
|
|
for path in (f"{self.folder}/none.txt", self.folder):
|
|
with self.subTest(path=path):
|
|
self.assertEqual(self.found("Read", {"file_path": path}), [])
|
|
|
|
def test_the_hook_blocks_a_repeated_read_without_offering_an_override(self):
|
|
self.read(1, 10)
|
|
self.used("Bash", {"command": f"head -n 5 {self.file}"}, self.shown(1, 5))
|
|
for tool, given in [("Read", {"file_path": self.file, "offset": 1, "limit": 10}), ("Bash", {"command": f"head -n 5 {self.file}"})]:
|
|
with self.subTest(tool):
|
|
ran = hook({"cwd": self.folder, "tool_name": tool, "tool_input": given, "transcript_path": self.transcript}, self.folder)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule repeat-read: ", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
|
|
class Hook(unittest.TestCase):
|
|
def setUp(self):
|
|
self.repo = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.repo])
|
|
git(self.repo, "init", "-q", "-b", "main")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "base")
|
|
git(self.repo, "checkout", "-q", "-b", "chunk")
|
|
|
|
def test_runs_git_guarded(self):
|
|
git(self.repo, "config", "core.fsmonitor", "/bin/false")
|
|
self.assertEqual(guarded_git(self.repo, "config", "core.fsmonitor"), "false")
|
|
|
|
def listed(self) -> str:
|
|
return subprocess.run([str(HOOK), "overrides"], capture_output=True, check=True, cwd=self.repo, env=isolated_env(), text=True).stdout
|
|
|
|
def test_a_breach_exits_2_naming_the_rule_and_the_override(self):
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": "FROM node:24\n"}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule image-pin: ", ran.stderr)
|
|
self.assertIn("`# rule-override image-pin: <reason>`", ran.stderr)
|
|
|
|
def test_an_edit_of_a_fifo_or_a_device_finishes(self):
|
|
fifo = f"{self.repo}/pipe"
|
|
os.mkfifo(fifo)
|
|
# A cap on the hook's memory, so a read of /dev/zero fails fast instead of filling the machine.
|
|
capped = lambda: resource.setrlimit(resource.RLIMIT_AS, (1 << 30, 1 << 30))
|
|
for path, (tool, given) in itertools.product((fifo, "/dev/zero"), (("Write", {"content": "x\n"}), ("Edit", {"old_string": "a", "new_string": "b"}), ("MultiEdit", {"edits": [{"old_string": "a", "new_string": "b"}]}))):
|
|
with self.subTest(path=path, tool=tool):
|
|
call = {"cwd": self.repo, "tool_input": {"file_path": path, **given}, "tool_name": tool}
|
|
ran = subprocess.run([str(HOOK)], capture_output=True, cwd=self.repo, env=isolated_env(), input=json.dumps(call), preexec_fn=capped, text=True, timeout=10)
|
|
self.assertEqual(ran.returncode, 0)
|
|
|
|
def test_an_override_passes_and_is_listed_for_the_chunk(self):
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/sub/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual((ran.returncode, ran.stderr), (0, ""))
|
|
self.assertRegex(self.listed(), r"^\S+Z\tchunk\t[0-9a-f]{40}\timage-pin\tthe vendor tags no patch versions\t\S+/sub/Dockerfile\n$")
|
|
git(self.repo, "checkout", "-q", "main")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "elsewhere")
|
|
self.assertEqual(self.listed(), "", "the default branch holds no override of its own")
|
|
|
|
def test_an_override_outlives_an_amend(self):
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "work")
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
git(self.repo, "commit", "-q", "--amend", "--allow-empty", "-m", "reworded")
|
|
git(self.repo, "branch", "-m", "renamed")
|
|
self.assertIn("\timage-pin\t", self.listed(), "the reflog keeps the amended commit")
|
|
|
|
def test_a_folder_named_like_the_branch_doesn_t_hide_the_overrides(self):
|
|
os.mkdir(f"{self.repo}/chunk")
|
|
pathlib.Path(self.repo, "chunk", "notes.md").write_text("x\n", encoding="utf-8")
|
|
git(self.repo, "add", "chunk")
|
|
git(self.repo, "commit", "-q", "-m", "a folder named like the branch")
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
self.assertIn("\timage-pin\t", self.listed())
|
|
|
|
def test_an_unreadable_reflog_fails_the_listing(self):
|
|
unborn = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", unborn])
|
|
git(unborn, "init", "-q", "-b", "chunk")
|
|
ran = subprocess.run([str(HOOK), "overrides"], capture_output=True, cwd=unborn, env=isolated_env(), text=True)
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("cannot read the branch's reflog", ran.stderr)
|
|
git(self.repo, "reflog", "expire", "--expire=now", "--all")
|
|
self.assertEqual(subprocess.run([str(HOOK), "overrides"], capture_output=True, cwd=self.repo, env=isolated_env(), text=True).returncode, 1, "an expired reflog holds no HEAD")
|
|
|
|
def test_a_sibling_branch_s_override_is_not_listed(self):
|
|
git(self.repo, "checkout", "-q", "-b", "sibling")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "sibling work")
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
git(self.repo, "checkout", "-q", "chunk")
|
|
self.assertEqual(self.listed(), "")
|
|
git(self.repo, "merge", "-q", "--squash", "sibling")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "squashed")
|
|
self.assertEqual(self.listed(), "", "a squash merge brings in none of the sibling's commits")
|
|
|
|
def test_a_blocked_call_logs_no_override(self):
|
|
command = f"git commit -m a -m '{TRAILER}' # rule-override no-ai-attribution: quotes it"
|
|
self.assertEqual(hook({"cwd": self.repo, "tool_input": {"command": command}, "tool_name": "Bash"}, self.repo).returncode, 2)
|
|
self.assertEqual(self.listed(), "")
|
|
|
|
def test_an_agent_call_below_its_rung_is_refused_and_offers_no_override(self):
|
|
call = {"cwd": self.repo, "tool_input": {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review. # rule-override reviewer-rung: trust me"}, "tool_name": "Agent"}
|
|
ran = hook(call, self.repo, ReviewerRung.ENVIRON)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule reviewer-rung", ran.stderr)
|
|
self.assertIn("stability-reviewer", ran.stderr)
|
|
self.assertIn("claude-sonnet-5-5 at high effort", ran.stderr)
|
|
self.assertIn("model opus, effort medium", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_an_agent_call_at_its_rung_passes(self):
|
|
call = {"cwd": self.repo, "tool_input": {"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium", "prompt": "Review."}, "tool_name": "Agent"}
|
|
self.assertEqual(hook(call, self.repo, ReviewerRung.ENVIRON).returncode, 0)
|
|
|
|
def test_a_reviewers_write_outside_the_reports_folder_is_refused_and_offers_no_override(self):
|
|
call = {"agent_type": "prose-reviewer", "cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/a.md", "content": "<!-- rule-override reviewer-report: trust me -->\n"}, "tool_name": "Write"}
|
|
ran = hook(call, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule reviewer-report: prose-reviewer would write ", ran.stderr)
|
|
self.assertIn("/.local/state/larv-review/", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_a_commit_override_is_logged_in_the_commit_s_checkout(self):
|
|
outside = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", outside])
|
|
ran = hook({"cwd": outside, "tool_input": {"command": f"git -C {self.repo} commit -m a -m b # rule-override commit-one-line: a body is needed"}, "tool_name": "Bash"}, outside)
|
|
self.assertEqual(ran.returncode, 0)
|
|
self.assertIn("\tcommit-one-line\ta body is needed\t", self.listed())
|
|
|
|
def test_an_edit_is_judged_on_the_file_it_leaves(self):
|
|
pathlib.Path(self.repo, ".gitignore").write_text("a\nc\n")
|
|
call = {"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/.gitignore", "new_string": "d", "old_string": "a"}, "tool_name": "Edit"}
|
|
self.assertEqual(hook(call, self.repo).returncode, 2)
|
|
|
|
def round_commit(self, title: str, given: str = "") -> subprocess.CompletedProcess:
|
|
return hook({"cwd": self.repo, "tool_input": {"command": f"git commit {given}-m '{title}'"}, "tool_name": "Bash"}, self.repo)
|
|
|
|
def test_a_round_commit_past_round_2_is_refused_and_offers_no_override(self):
|
|
ran = self.round_commit("Answer stability round 3: 0 🔴, 1 🟡 (0 misread); fix # rule-override round-cap: the human said yes")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule round-cap: this would be stability's round 3, past its 2", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); fix").returncode, 0)
|
|
|
|
def test_a_round_commit_naming_no_pass_is_refused_and_offers_no_override(self):
|
|
ran = self.round_commit("Answer architectur round 1: 0 🔴, 1 🟡 (0 misread); fix # rule-override round-cap: a typo")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("Blocked by rule round-cap: `architectur` names no larv-review pass, so its rounds would never reach a cap; name one of: architecture, principles, full scan,", ran.stderr)
|
|
self.assertNotIn("rule-override", ran.stderr)
|
|
|
|
def test_a_third_round_commit_of_a_pass_on_the_branch_is_refused_whatever_its_number(self):
|
|
for title in ["Answer stability round 1: 1 🔴, 0 🟡 (0 misread); a", "Answer prose round 1: 0 🔴, 1 🟡 (0 misread); b", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
ran = self.round_commit("Answer stability round 2: 0 🔴, 0 🟡 (0 misread); d")
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("stability's round commits on this branch: `Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c`, `Answer stability round 1: 1 🔴, 0 🟡 (0 misread); a`", ran.stderr)
|
|
self.assertIn("`git commit --amend` where that is the newest commit", ran.stderr)
|
|
self.assertEqual(self.round_commit("Answer prose round 2: 0 🔴, 0 🟡 (0 misread); d").returncode, 0)
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c, reworded", "--amend --allow-empty ").returncode, 0, "an amend replaces the commit it counts")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "fix a typo")
|
|
self.assertEqual(self.round_commit("Answer stability round 2: 0 🔴, 1 🟡 (0 misread); e", "--amend --allow-empty ").returncode, 2, "an amend replaces the newest commit, not the oldest")
|
|
|
|
def test_a_stalled_origin_holds_up_neither_a_round_commit_nor_the_overrides_listing(self):
|
|
git(self.repo, "config", "protocol.ext.allow", "always")
|
|
git(self.repo, "remote", "add", "origin", "ext::sleep 30")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a").returncode, 0)
|
|
hook({"cwd": self.repo, "tool_input": {"command": "git commit -m a -m b # rule-override commit-one-line: a body is needed"}, "tool_name": "Bash"}, self.repo)
|
|
listed = subprocess.run([str(HOOK), "overrides"], capture_output=True, check=True, cwd=self.repo, env=isolated_env(), text=True, timeout=10).stdout
|
|
self.assertIn("\tcommit-one-line\ta body is needed\t", listed)
|
|
|
|
def test_round_commits_merged_in_from_the_default_branch_do_not_count(self):
|
|
git(self.repo, "checkout", "-q", "main")
|
|
for title in ["Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); b"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
git(self.repo, "checkout", "-q", "chunk")
|
|
git(self.repo, "merge", "-q", "--no-ff", "-m", "Merge main", "main")
|
|
git(self.repo, "branch", "-f", "main", "main~2")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); c").returncode, 0, "a stale default branch still leaves another chunk's rounds out")
|
|
|
|
def test_a_stale_local_main_does_not_stand_in_for_a_default_branch_origin_left_unrecorded(self):
|
|
git(self.repo, "checkout", "-q", "main")
|
|
for title in ["Answer stability round 1: 0 🔴, 1 🟡 (0 misread); a", "Answer stability round 2: 0 🔴, 1 🟡 (0 misread); b"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
git(self.repo, "checkout", "-q", "-B", "chunk", "main")
|
|
git(self.repo, "branch", "-f", "main", "main~2")
|
|
git(self.repo, "remote", "add", "origin", f"{self.repo}/gone.git")
|
|
self.assertEqual(self.round_commit("Answer stability round 1: 0 🔴, 1 🟡 (0 misread); c").returncode, 0, "another chunk's rounds, fast-forwarded into the default branch, are not this chunk's")
|
|
|
|
def test_a_commit_is_titled_by_the_configs_template_and_its_round_title_read_through_it(self):
|
|
config = configured(self, pathlib.Path(self.repo)) / "config.toml"
|
|
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\n')
|
|
git(self.repo, "checkout", "-q", "-b", "item-7")
|
|
for title in ["PAY-7 - Answer stability round 1: 1 🔴, 0 🟡 (0 misread); a", "PAY-7 - Answer stability round 2: 0 🔴, 1 🟡 (0 misread); c"]:
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", title)
|
|
past = self.round_commit("PAY-7 - Answer stability round 2: 0 🔴, 0 🟡 (0 misread); d")
|
|
self.assertEqual(past.returncode, 2)
|
|
self.assertIn("Blocked by rule round-cap: this would be stability's round 3, past its 2", past.stderr)
|
|
self.assertIn("Blocked by rule round-title-counts", self.round_commit("PAY-7 - Answer prose round 1: fix").stderr)
|
|
bare = self.round_commit("Answer prose round 1: 0 🔴, 0 🟡 (0 misread); x")
|
|
self.assertEqual(bare.returncode, 2)
|
|
self.assertIn("Blocked by rule commit-title: title the commit as the project's config asks, `PAY-{id} - {summary}`", bare.stderr)
|
|
self.assertEqual(self.round_commit("PAY-7 - Answer prose round 1: 0 🔴, 0 🟡 (0 misread); x").returncode, 0)
|
|
self.assertEqual(self.round_commit("Tidy # rule-override commit-title: the client asked for this title").returncode, 0)
|
|
|
|
def test_a_commit_on_a_branch_that_doesnt_fit_the_configs_branch_template_takes_a_bare_title(self):
|
|
config = configured(self, pathlib.Path(self.repo)) / "config.toml"
|
|
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\nbranch = "feature/PAY-{id}"\n')
|
|
git(self.repo, "checkout", "-q", "-b", "release-review-0.3.0")
|
|
self.assertEqual(self.round_commit("Release review of 0.3.0", "--allow-empty ").returncode, 0)
|
|
self.assertIn("Blocked by rule round-title-counts", self.round_commit("PAY-7 - Answer prose round 1: fix").stderr)
|
|
git(self.repo, "checkout", "-q", "-b", "feature/PAY-7")
|
|
held = self.round_commit("Release review of 0.3.0", "--allow-empty ")
|
|
self.assertEqual(held.returncode, 2)
|
|
self.assertIn("Blocked by rule commit-title: title the commit as the project's config asks, `PAY-{id} - {summary}`", held.stderr)
|
|
self.assertEqual(self.round_commit("PAY-7 - Tidy").returncode, 0)
|
|
|
|
def test_a_commit_is_refused_naming_the_fault_while_a_config_cannot_be_read(self):
|
|
config = configured(self, pathlib.Path(self.repo)) / "config.toml"
|
|
config.write_text(config.read_text() + 'commit-title = "PAY-{id}"\n')
|
|
refused = self.round_commit("PAY-7 - Tidy")
|
|
self.assertEqual(refused.returncode, 2)
|
|
self.assertIn("Blocked by rule commit-title: the project's config cannot be read, so the title is not checked against its template: ", refused.stderr)
|
|
self.assertIn("`commit-title` is 'PAY-{id}'", refused.stderr)
|
|
config.write_text(config.read_text().replace('commit-title = "PAY-{id}"\n', ""))
|
|
other = config.parents[1] / "other" / "config.toml"
|
|
other.parent.mkdir()
|
|
other.write_text("checkout = [")
|
|
refused = self.round_commit("Tidy")
|
|
self.assertEqual(refused.returncode, 2)
|
|
self.assertIn(f"cannot read {other}", refused.stderr)
|
|
other.unlink()
|
|
self.assertEqual(self.round_commit("Tidy").returncode, 0)
|
|
|
|
def test_a_commit_while_an_unranked_todo_has_changes_is_blocked(self):
|
|
configured(self, pathlib.Path(self.repo))
|
|
pathlib.Path(self.repo, "docs").mkdir()
|
|
for name, content in [("README.md", README), ("docs/decisions.md", DECISIONS), ("todo.md", TODO)]:
|
|
pathlib.Path(self.repo, name).write_text(content)
|
|
git(self.repo, "add", "README.md", "docs", "todo.md")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
call = {"cwd": self.repo, "tool_input": {"command": "git add -A && git commit -m x"}, "tool_name": "Bash"}
|
|
self.assertEqual(hook(call, self.repo).returncode, 0, "todo.md is unchanged")
|
|
pathlib.Path(self.repo, "todo.md").write_text(TODO + "\n")
|
|
ran = hook(call, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertIn("todo.md: not ranked", ran.stderr)
|
|
|
|
def test_a_staged_todo_change_is_checked_and_no_clean_filter_runs(self):
|
|
configured(self, pathlib.Path(self.repo))
|
|
pathlib.Path(self.repo, "docs").mkdir()
|
|
for name, content in [("README.md", README), ("docs/decisions.md", DECISIONS), ("todo.md", TODO), (".gitattributes", "todo.md filter=planted\n")]:
|
|
pathlib.Path(self.repo, name).write_text(content)
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "config", "filter.planted.clean", f"touch {self.repo}/ran; cat")
|
|
pathlib.Path(self.repo, "ran").unlink(missing_ok=True)
|
|
# A stat-dirty index entry, as extracting an archive leaves, is what makes `git status` re-run the filter.
|
|
os.utime(pathlib.Path(self.repo, "todo.md"), (1, 1))
|
|
ran = hook({"cwd": self.repo, "tool_input": {"command": "git commit -m x"}, "tool_name": "Bash"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2, ran.stderr)
|
|
self.assertFalse(pathlib.Path(self.repo, "ran").exists())
|
|
|
|
def test_a_symlinked_log_is_refused(self):
|
|
target = pathlib.Path(self.repo, "elsewhere")
|
|
pathlib.Path(self.repo, ".git", "rule-overrides.tsv").symlink_to(target)
|
|
content = "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n"
|
|
ran = hook({"cwd": self.repo, "tool_input": {"file_path": f"{self.repo}/Dockerfile", "content": content}, "tool_name": "Write"}, self.repo)
|
|
self.assertEqual(ran.returncode, 2)
|
|
self.assertFalse(target.exists())
|
|
|
|
def test_a_malformed_override_is_named(self):
|
|
ran = hook({"cwd": self.repo, "tool_input": {"command": f"git commit -m a -m b -m '{TRAILER}' # rule-override no-ai-attribution: quotes it # rule-override commit-oneline: x"}, "tool_name": "Bash"}, self.repo)
|
|
self.assertIn("`rule-override commit-oneline: x` names no rule below", ran.stderr)
|
|
|
|
def test_a_missing_field_breaks_nothing(self):
|
|
for given in [{"command": None}, {"content": None, "file_path": f"{self.repo}/a.md"}, {"edits": [None], "file_path": f"{self.repo}/a.md"}]:
|
|
with self.subTest(given):
|
|
tool = "Bash" if "command" in given else "Write" if "content" in given else "MultiEdit"
|
|
self.assertEqual(hook({"cwd": self.repo, "tool_input": given, "tool_name": tool}, self.repo).returncode, 0)
|
|
|
|
def test_unreadable_input_does_not_block(self):
|
|
for given in ["{", json.dumps({"cwd": ".", "tool_input": None, "tool_name": "Bash"})]:
|
|
with self.subTest(given):
|
|
self.assertEqual(subprocess.run([str(HOOK)], capture_output=True, env=isolated_env(), input=given, text=True).returncode, 1)
|
|
|
|
def test_help_lists_every_rule(self):
|
|
helped = subprocess.run([str(HOOK), "--help"], capture_output=True, check=True, env=isolated_env(), text=True).stdout
|
|
for rule in ["action-pin", "commit-one-line", "commit-title", "docker-run-memory", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check"]:
|
|
with self.subTest(rule):
|
|
self.assertIn(f" {rule}\n The rule: ", helped)
|
|
self.assertIn("reviewer-report\n The rule: a reviewer writes only its report", helped)
|
|
self.assertRegex(helped, r" reviewer-report\n( .*\n){2} Takes no override\.\n")
|
|
self.assertRegex(helped, r" image-pin\n( .*\n){2} \S", "a rule a marker waives doesn't say anything of overrides")
|
|
|
|
|
|
class TodoRankCheck(unittest.TestCase):
|
|
"""A checkout of the repo the hook runs from checks todo.md with its own bin/rank_todo.py; any other checkout with the hook's."""
|
|
STUB = "import sys\nprint('own script checked ' + sys.argv[-1], file=sys.stderr)\nsys.exit(int(sys.argv[1] == '--check'))\n"
|
|
|
|
def setUp(self):
|
|
self.folder = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.folder])
|
|
self.own = f"{self.folder}/own"
|
|
pathlib.Path(self.own).mkdir()
|
|
git(self.own, "init", "-q", "-b", "main")
|
|
git(self.own, "commit", "-q", "--allow-empty", "-m", "root")
|
|
git(self.folder, "clone", "-q", self.own, "clone")
|
|
self.other = f"{self.folder}/other"
|
|
pathlib.Path(self.other).mkdir()
|
|
git(self.other, "init", "-q", "-b", "main")
|
|
git(self.other, "commit", "-q", "--allow-empty", "-m", "another root")
|
|
|
|
def changed_plan(self, checkout: str, stub: str) -> str:
|
|
pathlib.Path(checkout, "bin").mkdir()
|
|
pathlib.Path(checkout, "bin", "rank_todo.py").write_text(stub)
|
|
pathlib.Path(checkout, "todo.md").write_text("# todo\n")
|
|
self.config = configured(self, pathlib.Path(checkout)) / "config.toml"
|
|
return checkout
|
|
|
|
def test_a_changed_plan_that_cannot_be_read_is_a_breach(self):
|
|
checkout = self.changed_plan(self.other, "")
|
|
pathlib.Path(checkout, "todo.md").chmod(0)
|
|
self.addCleanup(pathlib.Path(checkout, "todo.md").chmod, 0o600)
|
|
self.assertEqual(["Permission denied" in detail for detail in self.details(checkout)], [True])
|
|
|
|
def details(self, checkout: str) -> list[str]:
|
|
return [breach.detail for breach in todo_breaches(checkout, pathlib.Path.home(), pathlib.Path(self.own))]
|
|
|
|
def test_a_checkout_of_the_hooks_repo_runs_its_own_rank_todo(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
self.assertEqual(self.details(checkout), [f"own script checked {checkout}/todo.md"])
|
|
top = str(pathlib.Path(checkout).resolve())
|
|
self.assertEqual([(breach.rule, breach.directory) for breach in todo_breaches(checkout, pathlib.Path.home(), pathlib.Path(self.own))], [("todo-rank-check", top)])
|
|
self.config.unlink()
|
|
self.assertEqual([(breach.rule, breach.directory) for breach in todo_breaches(checkout, pathlib.Path.home(), pathlib.Path(self.own))], [("todo-rank-check", top)], "a todo.md with no config")
|
|
|
|
def test_its_own_rank_todo_passing_passes(self):
|
|
self.assertEqual(self.details(self.changed_plan(f"{self.folder}/clone", "")), [])
|
|
|
|
def test_its_own_rank_todo_failing_silently_names_the_script_and_exit(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", "import sys\nsys.exit(3)\n")
|
|
self.assertEqual(self.details(checkout), [f"{checkout}/bin/rank_todo.py --check failed with exit 3 and printed no output"])
|
|
|
|
def test_its_own_rank_todo_running_past_the_limit_is_named(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", "import signal\nsignal.pause()\n")
|
|
with unittest.mock.patch("rule_hook.todo_check.OWN_CHECK_SECONDS", 0.2):
|
|
self.assertEqual(self.details(checkout), [f"{checkout}/bin/rank_todo.py --check did not finish in 0.2 seconds"])
|
|
|
|
def test_another_repos_rank_todo_never_runs(self):
|
|
reasons = self.details(self.changed_plan(self.other, self.STUB))
|
|
self.assertTrue(reasons)
|
|
self.assertNotIn("own script", " ".join(reasons))
|
|
|
|
def test_a_plan_the_config_puts_outside_the_checkout_is_checked_in_place_of_the_checkouts_todo_md(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
self.config.write_text(self.config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
|
|
self.assertEqual(self.details(checkout), [], "the project has no plan yet, and the checkout's todo.md is its owners'")
|
|
(self.config.parent / "todo.md").write_text("# todo\n")
|
|
self.assertEqual(self.details(checkout), [f"own script checked {self.config.parent}/todo.md"])
|
|
|
|
def test_a_plan_in_jira_is_not_checked_in_the_checkout_or_the_configs_folder(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
self.config.write_text(self.config.read_text().replace('"checkout:todo.md"', '"jira:https://acme.atlassian.net/browse/ABC-12"') + 'env-file = "~/acme.env"\njira-score-formula = "V"\njira-score-bar = 1\n')
|
|
folder = str(self.config.parent)
|
|
git(folder, "init", "-q")
|
|
(self.config.parent / "todo.md").write_text("# todo\n")
|
|
self.assertEqual((self.details(checkout), self.details(folder)), ([], []))
|
|
|
|
def test_a_commit_in_the_configs_folder_checks_the_plan_it_keeps_there_once_it_changes(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
folder = str(self.config.parent)
|
|
git(folder, "init", "-q", "-b", "main")
|
|
self.assertEqual(self.details(folder), [], "the plan is the checkout's todo.md, which a commit here doesn't hold")
|
|
self.config.write_text(self.config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
|
|
(self.config.parent / "todo.md").write_text("# todo\n")
|
|
self.assertEqual(self.details(folder), [f"own script checked {folder}/todo.md"], "checked with the checkout's own rank_todo.py, as a commit there would be")
|
|
self.assertEqual([(breach.rule, breach.directory) for breach in todo_breaches(folder, pathlib.Path.home(), pathlib.Path(self.own))], [("todo-rank-check", str(pathlib.Path(folder).resolve()))])
|
|
git(folder, "add", "-A")
|
|
git(folder, "commit", "-q", "-m", "plan")
|
|
self.assertEqual(self.details(folder), [], "committed, so unchanged")
|
|
|
|
def test_an_unreadable_config_does_not_block_a_commit_in_its_folder(self):
|
|
self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
folder = str(self.config.parent)
|
|
git(folder, "init", "-q", "-b", "main")
|
|
self.config.write_text("checkout = 3\n")
|
|
(self.config.parent / "todo.md").write_text("# changed\n")
|
|
self.assertEqual(self.details(folder), [], "the commit that fixes the config goes through")
|
|
|
|
def test_a_commit_in_the_configs_folder_of_another_repo_checks_its_plan_with_the_hooks_ranking(self):
|
|
self.changed_plan(self.other, self.STUB)
|
|
self.config.write_text(self.config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
|
|
folder = str(self.config.parent)
|
|
git(folder, "init", "-q", "-b", "main")
|
|
(self.config.parent / "todo.md").write_text("# todo\n")
|
|
reasons = self.details(folder)
|
|
self.assertTrue(reasons)
|
|
self.assertNotIn("own script", " ".join(reasons))
|
|
|
|
def test_a_plan_in_a_folder_of_the_checkout_is_checked_once_it_changes(self):
|
|
checkout = self.changed_plan(f"{self.folder}/clone", self.STUB)
|
|
self.config.write_text(self.config.read_text().replace('"checkout:todo.md"', '"checkout:plan/todo.md"'))
|
|
pathlib.Path(checkout, "plan").mkdir()
|
|
pathlib.Path(checkout, "plan", "todo.md").write_text("# todo\n")
|
|
self.assertEqual(self.details(checkout), [f"own script checked {checkout}/plan/todo.md"])
|
|
git(checkout, "add", "plan")
|
|
git(checkout, "commit", "-q", "-m", "plan")
|
|
self.assertEqual(self.details(checkout), [], "committed, so unchanged")
|
|
|
|
def test_a_checkout_with_a_changed_todo_md_and_no_config_is_told_to_write_one(self):
|
|
checkout = self.changed_plan(self.other, self.STUB)
|
|
self.config.unlink()
|
|
self.assertEqual(self.details(checkout), [f"no config names the checkout {checkout}: write ~/.claude-projects/<key>/config.toml, as `project-config --help` shows"])
|
|
git(checkout, "add", "todo.md")
|
|
git(checkout, "commit", "-q", "-m", "plan")
|
|
self.assertEqual(self.details(checkout), [], "a commit that leaves todo.md alone, as in a clone of someone else's project, is not held")
|
|
pathlib.Path(checkout, "todo.md").unlink()
|
|
self.assertEqual(self.details(checkout), [])
|
|
|
|
|
|
class CommittedFiles(unittest.TestCase):
|
|
"""A git commit holds the files it would commit to the file rules, as an Edit or Write of each would be."""
|
|
UNPINNED = "jobs:\n a:\n steps:\n - uses: actions/checkout@main\n"
|
|
|
|
def setUp(self):
|
|
self.repo = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", self.repo])
|
|
git(self.repo, "init", "-q", "-b", "main")
|
|
git(self.repo, "commit", "-q", "--allow-empty", "-m", "base")
|
|
|
|
def write(self, name: str, content: str) -> None:
|
|
path = pathlib.Path(self.repo, name)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content)
|
|
|
|
def commit(self, command: str = "git add -A && git commit -m x") -> subprocess.CompletedProcess:
|
|
return hook({"cwd": self.repo, "tool_input": {"command": command}, "tool_name": "Bash"}, self.repo)
|
|
|
|
def test_a_file_a_command_wrote_is_checked_at_the_commit(self):
|
|
self.write(".github/workflows/ci.yml", self.UNPINNED)
|
|
self.write("Dockerfile", "FROM node:24\n")
|
|
self.write(".gitignore", "b\na\n")
|
|
self.write("NOTES.md", "Co-Authored-" + "By: Claude <noreply@anthropic.com>\n")
|
|
ran = self.commit()
|
|
self.assertEqual(ran.returncode, 2)
|
|
for rule in ["action-pin", "image-pin", "ignorefile-sorted", "no-ai-attribution"]:
|
|
with self.subTest(rule):
|
|
self.assertIn(f"Blocked by rule {rule}: ", ran.stderr)
|
|
|
|
def test_a_staged_file_is_checked_and_a_clean_tree_passes(self):
|
|
self.assertEqual(self.commit().returncode, 0)
|
|
self.write(".github/workflows/ci.yml", self.UNPINNED)
|
|
git(self.repo, "add", "-A")
|
|
self.assertEqual(self.commit("git commit -m x").returncode, 2)
|
|
|
|
def test_a_breach_in_both_copies_of_a_file_is_one_breach_which_the_work_trees_marker_overrides(self):
|
|
self.write("Dockerfile", "FROM node:24\n")
|
|
git(self.repo, "add", "-A")
|
|
self.write("Dockerfile", "FROM node:24\nRUN true\n")
|
|
blocked = self.commit("git commit -m x")
|
|
self.assertEqual((blocked.returncode, blocked.stderr.count("Blocked by rule image-pin: ")), (2, 1))
|
|
self.write("Dockerfile", "FROM node:24\n# rule-override image-pin: the test needs a moving tag\n")
|
|
self.assertEqual(self.commit("git commit -m x").returncode, 0)
|
|
|
|
def test_a_breach_the_head_already_holds_is_left_alone(self):
|
|
self.write("Dockerfile", "FROM node:24\n")
|
|
self.write(".gitignore", "b\na\n")
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
self.write("Dockerfile", "FROM node:24\nRUN true\n")
|
|
self.write(".gitignore", "b\na\n\n# more\nc\n")
|
|
self.assertEqual(self.commit().returncode, 0)
|
|
self.write("Dockerfile", "FROM node:24\nRUN true\nFROM node:25\n")
|
|
self.assertIn("image-pin: ", self.commit().stderr)
|
|
|
|
def test_a_marker_in_the_command_or_the_file_lets_the_commit_through(self):
|
|
self.write("Dockerfile", "FROM vendor/tool:2\n")
|
|
self.assertEqual(self.commit("git add -A && git commit -m x # rule-override image-pin: the vendor tags no patch versions").returncode, 0)
|
|
self.write("Dockerfile", "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n")
|
|
self.assertEqual(self.commit().returncode, 0)
|
|
|
|
def test_a_staged_breach_blocks_the_commit_after_the_work_tree_is_fixed(self):
|
|
self.write("Dockerfile", "FROM node:24\n")
|
|
git(self.repo, "add", "Dockerfile")
|
|
self.write("Dockerfile", "FROM node:24.18.1\n")
|
|
self.assertIn("image-pin: ", self.commit("git commit -m x").stderr)
|
|
|
|
def test_a_stat_dirty_file_runs_no_clean_filter(self):
|
|
self.write("a.txt", "a\n")
|
|
self.write(".gitattributes", "a.txt filter=planted\n")
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
git(self.repo, "config", "filter.planted.clean", f"touch {self.repo}/ran; cat")
|
|
pathlib.Path(self.repo, "ran").unlink(missing_ok=True)
|
|
os.utime(pathlib.Path(self.repo, "a.txt"), (1, 1))
|
|
self.assertEqual(self.commit("git commit -m x").returncode, 0)
|
|
self.assertFalse(pathlib.Path(self.repo, "ran").exists())
|
|
|
|
def test_a_submodule_runs_no_clean_filter(self):
|
|
inner = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", inner])
|
|
git(inner, "init", "-q", "-b", "main")
|
|
pathlib.Path(inner, "b.txt").write_text("b\n")
|
|
pathlib.Path(inner, ".gitattributes").write_text("b.txt filter=planted\n")
|
|
git(inner, "add", "-A")
|
|
git(inner, "commit", "-qm", "x", "--no-verify")
|
|
git(self.repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", inner, "sub")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
git(f"{self.repo}/sub", "config", "filter.planted.clean", f"touch {self.repo}/ran; cat")
|
|
os.utime(pathlib.Path(self.repo, "sub", "b.txt"), (1, 1))
|
|
self.assertEqual(self.commit("git commit -m x").returncode, 0)
|
|
self.assertFalse(pathlib.Path(self.repo, "ran").exists())
|
|
|
|
def test_a_checkout_with_no_commit_checks_its_index_and_untracked_files(self):
|
|
fresh = tempfile.mkdtemp()
|
|
self.addCleanup(subprocess.run, ["rm", "-rf", fresh])
|
|
git(fresh, "init", "-q", "-b", "main")
|
|
pathlib.Path(fresh, "Dockerfile").write_text("FROM node:24\n")
|
|
git(fresh, "add", "Dockerfile")
|
|
pathlib.Path(fresh, "NOTES.md").write_text(TRAILER + "\n")
|
|
ran = hook({"cwd": fresh, "tool_input": {"command": "git commit -m x"}, "tool_name": "Bash"}, fresh)
|
|
for rule in ["image-pin", "no-ai-attribution"]:
|
|
self.assertIn(f"Blocked by rule {rule}: ", ran.stderr)
|
|
|
|
def test_a_change_that_only_reorders_an_ignore_file_is_checked(self):
|
|
self.write(".gitignore", "a\nb\nc\n")
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
self.write(".gitignore", "c\nb\na\n")
|
|
self.assertIn("ignorefile-sorted: ", self.commit().stderr)
|
|
|
|
def test_a_marker_the_head_already_holds_does_not_override_a_new_breach(self):
|
|
self.write("Dockerfile", "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\n")
|
|
git(self.repo, "add", "-A")
|
|
git(self.repo, "commit", "-qm", "x", "--no-verify")
|
|
self.write("Dockerfile", "# rule-override image-pin: the vendor tags no patch versions\nFROM vendor/tool:2\nFROM node:24\n")
|
|
self.assertIn("image-pin: ", self.commit().stderr)
|
|
|
|
def test_a_marker_in_another_file_does_not_override_a_breach(self):
|
|
self.write("Dockerfile", "FROM vendor/tool:2\n")
|
|
self.write("NOTES.md", "# rule-override image-pin: quoted in a note\n")
|
|
self.assertIn("Blocked by rule image-pin: ", self.commit().stderr)
|
|
|
|
def test_an_ignored_or_binary_file_is_left_alone(self):
|
|
self.write(".gitignore", "build/\n")
|
|
self.write("build/Dockerfile", "FROM node:24\n")
|
|
pathlib.Path(self.repo, "image.png").write_bytes(b"\xff\xfe\x00FROM node:24\n")
|
|
self.assertEqual(self.commit().returncode, 0)
|
|
|
|
|
|
class RuleNames(unittest.TestCase):
|
|
def test_every_rule_name_has_a_rule_and_every_rule_a_name(self):
|
|
self.assertEqual(set(RULES), set(typing.get_args(RuleName)))
|