Answer stability round 1: 0 🔴, 4 🟡 (0 misread); refuse a security skill in a subagent not launched as the security pass, split skill_breaches out, tell a subagent by agent_id
This commit is contained in:
+1
-1
@@ -4,7 +4,7 @@
|
||||
|
||||
- A release review now fixes the findings of its architecture and principles passes in the release it reviews. Before, it filed each one in `todo.md`, so the release held the drift the review had found. It files only three kinds of finding, each under the release being reviewed: one whose fix needs a human answer, marked `question`; one whose fix adds a feature; and one whose fix the five rounds cannot finish. Until each is done, the release isn't ready to cut.
|
||||
- In a repo with no tag yet, `chunk-preflight` now calls a release finished once `todo.md` files no item under it: its release review is due, and no later release's item is taken. Before, it lost sight of such a release.
|
||||
- In a run `loop-claude` started, `rule-hook` now refuses a `/security-review` or `/claude-security` Skill call that the worker makes in its own context on a rung below the project's. The refusal says to invoke it from a subagent on the project's rung. Before, only an Agent call that launched the security pass was held to that rung.
|
||||
- In a run `loop-claude` started, `rule-hook` now refuses a `/security-review` or `/claude-security` Skill call that the worker makes in its own context on a rung below the project's. It also refuses one from any subagent whose prompt doesn't open with `You are the security pass for this chunk.`, since that subagent's rung is unchecked. The refusal says to invoke the skill from a subagent on the project's rung, its prompt opening with that sentence. Before, only an Agent call that launched the security pass was held to that rung.
|
||||
|
||||
## 0.4.0
|
||||
|
||||
|
||||
@@ -12,10 +12,12 @@ class Call:
|
||||
tool: str
|
||||
given: dict[str, typing.Any]
|
||||
cwd: str
|
||||
# The subagent the call comes from, such as prose-reviewer; empty for the session's own agent.
|
||||
# The agent that makes the call, such as prose-reviewer, or the one a session was started with by --agent; empty for a session's default agent.
|
||||
agent: str = ""
|
||||
# The transcript of the agent that makes the call: a subagent's own, not its session's.
|
||||
transcript: str = ""
|
||||
# Whether a subagent makes the call.
|
||||
subagent: bool = False
|
||||
|
||||
|
||||
def parsed_call(hook_input: object) -> Call | Failure:
|
||||
@@ -29,7 +31,7 @@ def parsed_call(hook_input: object) -> Call | Failure:
|
||||
if isinstance(agent_id, str) and transcript:
|
||||
# Inside a subagent, transcript_path still names the session's transcript.
|
||||
transcript = subagent_transcript(transcript, agent_id)
|
||||
return Call(tool, given, cwd, agent if isinstance(agent, str) else "", transcript)
|
||||
return Call(tool, given, cwd, agent if isinstance(agent, str) else "", transcript, isinstance(agent_id, str))
|
||||
|
||||
|
||||
def text(given: dict[str, typing.Any], key: str) -> str:
|
||||
|
||||
@@ -43,7 +43,7 @@ from failure import Failure
|
||||
from rule_hook.override_log import Log, log_path, logged, print_overrides, write_override
|
||||
from rule_hook.repeat_read import read_breaches
|
||||
from rule_hook.reviewer_report import report_breaches
|
||||
from rule_hook.reviewer_rung import reviewer_breaches
|
||||
from rule_hook.reviewer_rung import reviewer_breaches, skill_breaches
|
||||
from rule_hook.round_cap import round_cap_breaches
|
||||
from shell_command import simple_commands
|
||||
from rule_hook.todo_check import todo_breaches
|
||||
@@ -77,7 +77,7 @@ def breaches_of(call: Call, home: str, environ: typing.Mapping[str, str]) -> tup
|
||||
if call.tool == "Agent":
|
||||
return text(call.given, "prompt"), text(call.given, "subagent_type"), reviewer_breaches(call, environ)
|
||||
if call.tool == "Skill":
|
||||
return text(call.given, "args"), text(call.given, "skill"), reviewer_breaches(call, environ)
|
||||
return text(call.given, "args"), text(call.given, "skill"), skill_breaches(call, environ)
|
||||
if call.tool == "Bash":
|
||||
command = text(call.given, "command")
|
||||
return command, command.split("\n", 1)[0], bash_breaches(command, call.cwd, home) + read_breaches(call, home)
|
||||
|
||||
@@ -4,11 +4,13 @@ import typing
|
||||
|
||||
from rule_hook.rules import Breach
|
||||
from rule_hook.call import Call, text
|
||||
from claude_code.entries import content
|
||||
from failure import Failure
|
||||
from rule_hook.git import git
|
||||
import git_repo
|
||||
from model_routing.worker_rungs import handed
|
||||
from model_routing.ladder import position, review_rungs, ReviewKind, Rung
|
||||
from claude_code.transcript import read_transcript
|
||||
from review_passes import REVIEWERS
|
||||
|
||||
# The pass each agent under agents/ reviews.
|
||||
@@ -19,31 +21,51 @@ SEAT = "You are one seat of a comprehension panel."
|
||||
SECURITY_OPENER = "You are the security pass for this chunk."
|
||||
SUBJECTS: dict[ReviewKind, str] = {"seat": "a comprehension seat", "security": "the security pass's subagent"}
|
||||
# The skills that run the security pass, by their names with any plugin prefix cut.
|
||||
SECURITY_SKILLS = {"security-review", "claude-security"}
|
||||
SECURITY_SKILLS = {"claude-security", "security-review"}
|
||||
# Markdown an agent follows, which counts as config: Claude Code's memory files, and the folders that hold Claude Code's skills, agents and rules and this repo's principles.
|
||||
INSTRUCTION_FILES = {"CLAUDE.md", "AGENTS.md"}
|
||||
INSTRUCTION_FOLDERS = {"agents", "principles", "rules", "skills"}
|
||||
|
||||
|
||||
def reviewer_breaches(call: Call, environ: typing.Mapping[str, str]) -> list[Breach]:
|
||||
"""The breach of an Agent or Skill call that launches a review below its rung; none where loop-claude handed no rungs."""
|
||||
"""The breach of an Agent call that launches a review below its rung; none where loop-claude handed no rungs."""
|
||||
project, worker, escape_and_impact = handed(environ)
|
||||
kind = security_skill(call) if call.tool == "Skill" else review_kind(call.given)
|
||||
kind = review_kind(call.given)
|
||||
if kind is None or project is None or worker is None:
|
||||
return []
|
||||
required = review_rungs(project, worker, escape_and_impact)[kind]
|
||||
# A branch that changes only Markdown no agent follows holds no code or config for a reviewer on a higher rung to catch a defect in: skills/larv-review/SKILL.md → "Markdown only".
|
||||
if kind in REVIEWERS and position(required) > position(project) and markdown_only(call.cwd):
|
||||
required = project
|
||||
named = worker if call.tool == "Skill" else named_rung(call.given, worker)
|
||||
named = named_rung(call.given, worker)
|
||||
if position(named) >= position(required):
|
||||
return []
|
||||
if call.tool == "Skill":
|
||||
return [Breach("reviewer-rung", f"{text(call.given, 'skill')} would run in this context on {named}, below {required}: invoke it from a subagent launched with {required.agent_args()}, its prompt opening with `{SECURITY_OPENER}`", call.cwd)]
|
||||
subject = SUBJECTS.get(kind) or text(call.given, "subagent_type")
|
||||
return [Breach("reviewer-rung", f"{subject} would run on {named}, below {required}: launch it with {required.agent_args()}", call.cwd)]
|
||||
|
||||
|
||||
def skill_breaches(call: Call, environ: typing.Mapping[str, str]) -> list[Breach]:
|
||||
"""The breach of a Skill call that runs the security pass below the project's rung; none where loop-claude handed no rungs.
|
||||
|
||||
The skill runs on its caller's rung. A subagent launched as the security pass was held to the project's rung by reviewer_breaches. The worker's own rung is known, so its call is refused only below the project's rung. Any other subagent's rung is unknown, so its call is refused."""
|
||||
project, worker, _ = handed(environ)
|
||||
skill = text(call.given, "skill").lstrip("/").rsplit(":", 1)[-1]
|
||||
if skill not in SECURITY_SKILLS or project is None or worker is None or opening_prompt(call.transcript).startswith(SECURITY_OPENER):
|
||||
return []
|
||||
if not call.subagent and position(worker) >= position(project):
|
||||
return []
|
||||
runs_on = "on a rung rule-hook cannot see" if call.subagent else f"in this context on {worker}, below {project}"
|
||||
return [Breach("reviewer-rung", f"{skill} would run {runs_on}: invoke it from a subagent launched with {project.agent_args()}, its prompt opening with `{SECURITY_OPENER}`", call.cwd)]
|
||||
|
||||
|
||||
def opening_prompt(transcript: str) -> str:
|
||||
"""The prompt a transcript opens with, such as a subagent's from its Agent call; empty where it cannot be read."""
|
||||
entries = read_transcript(transcript) if transcript else []
|
||||
first = entries[0] if isinstance(entries, list) and entries else {}
|
||||
prompt = content(first) if first.get("type") == "user" else None
|
||||
return prompt.lstrip() if isinstance(prompt, str) else ""
|
||||
|
||||
|
||||
def review_kind(given: dict[str, typing.Any]) -> ReviewKind | None:
|
||||
"""What a launch reviews, a reviewer's pass, a comprehension seat or the security pass; None for any other subagent."""
|
||||
if pass_reviewed := PASS_OF.get(text(given, "subagent_type")):
|
||||
@@ -56,12 +78,6 @@ def review_kind(given: dict[str, typing.Any]) -> ReviewKind | None:
|
||||
return None
|
||||
|
||||
|
||||
def security_skill(call: Call) -> ReviewKind | None:
|
||||
"""The security pass, for a Skill call from the session's own agent that runs it; None for any other. A subagent's Skill call runs on the rung its Agent call was held to."""
|
||||
skill = text(call.given, "skill").lstrip("/").rsplit(":", 1)[-1]
|
||||
return "security" if skill in SECURITY_SKILLS and not call.agent else None
|
||||
|
||||
|
||||
def named_rung(given: dict[str, typing.Any], worker: Rung) -> Rung:
|
||||
"""The rung an Agent call's model and effort name. A call naming no model runs on the worker's model, at the effort it names or else the worker's; one naming a model but no effort runs on that model's cheapest rung."""
|
||||
model, effort = text(given, "model"), text(given, "effort")
|
||||
|
||||
@@ -60,8 +60,8 @@ RULES: dict[RuleName, Rule] = {
|
||||
),
|
||||
# A rung is not a judgment call, and the refusal names the arguments to pass, so no marker waives it.
|
||||
"reviewer-rung": Rule(
|
||||
"a reviewer, a comprehension seat or the security pass's subagent runs on the rung loop-claude names for it: for a seat or the security pass the project's rung, and for a reviewer the highest of the project's rung, the worker's and the one the chunk's item's E + I picks, or the project's rung where the branch changes only Markdown no agent follows, so a worker below the project's rung cannot hand a review to its own cheaper model",
|
||||
"an Agent call, in a claude loop-claude started, whose subagent type is an agent under agents/, or whose prompt opens with seat-prompts.md's opening sentence or the one larv-review's Security section sets; and a Skill call there of `security-review` or `claude-security` from the session's own agent, which runs on the worker's rung",
|
||||
"a reviewer, a comprehension seat or the security pass runs on the rung loop-claude names for it: for a seat or the security pass the project's rung, and for a reviewer the highest of the project's rung, the worker's and the one the chunk's item's E + I picks, or the project's rung where the branch changes only Markdown no agent follows, so a worker below the project's rung cannot hand a review to its own cheaper model",
|
||||
"an Agent call, in a claude loop-claude started, whose subagent type is an agent under agents/, or whose prompt opens with seat-prompts.md's opening sentence or the one larv-review's Security section sets; and a `security-review` or `claude-security` Skill call in such a claude, which runs on its caller's rung, unless the caller is a subagent whose prompt opens with that sentence",
|
||||
overridable=False,
|
||||
),
|
||||
# The cap holds with no ask, so no marker waives it.
|
||||
|
||||
@@ -19,7 +19,7 @@ from test_rank_todo import DECISIONS, README, TODO
|
||||
from test_env import isolated_env
|
||||
from model_routing import ladder
|
||||
from rule_hook.override_log import LogLine, parsed_entry, this_chunks
|
||||
from rule_hook.reviewer_rung import reviewer_breaches
|
||||
from rule_hook.reviewer_rung import reviewer_breaches, skill_breaches
|
||||
from rule_hook.git import git as rule_hook_git
|
||||
from model_routing.ladder import Rung
|
||||
from shell_command import segments, simple_commands
|
||||
@@ -315,21 +315,41 @@ class ReviewerRung(unittest.TestCase):
|
||||
def test_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
||||
self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "haiku", "effort": "low"}, {}), [])
|
||||
|
||||
def test_a_security_skill_in_the_sessions_own_context_runs_on_the_workers_rung(self):
|
||||
for skill in ("security-review", "claude-security", "claude-security:claude-security"):
|
||||
def skill_refused(self, given: dict, environ: dict[str, str] = ENVIRON, subagent_prompt: str | None = None) -> list[Breach]:
|
||||
transcript = ""
|
||||
if subagent_prompt is not None:
|
||||
transcript = os.path.join(tempfile.mkdtemp(), "agent-a1.jsonl")
|
||||
self.addCleanup(subprocess.run, ["rm", "-rf", os.path.dirname(transcript)])
|
||||
pathlib.Path(transcript).write_text(json.dumps({"type": "user", "message": {"role": "user", "content": subagent_prompt}}) + "\n")
|
||||
return skill_breaches(Call("Skill", given, "/r", "general-purpose" if subagent_prompt is not None else "", transcript, subagent_prompt is not None), environ)
|
||||
|
||||
def test_a_security_skill_from_the_sessions_own_agent_is_refused_below_the_projects_rung(self):
|
||||
for skill in ("/security-review", "claude-security", "claude-security:claude-security", "security-review"):
|
||||
with self.subTest(skill=skill):
|
||||
breaches = reviewer_breaches(Call("Skill", {"skill": skill}, "/r"), self.ENVIRON)
|
||||
breaches = self.skill_refused({"skill": skill})
|
||||
self.assertEqual(rules(breaches), ["reviewer-rung"])
|
||||
self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail)
|
||||
self.assertIn("model opus, effort medium", breaches[0].detail)
|
||||
self.assertIn("You are the security pass for this chunk.", breaches[0].detail)
|
||||
self.assertEqual(rules(reviewer_breaches(Call("Skill", {"skill": skill}, "/r"), self.ABOVE)), [])
|
||||
self.assertEqual(rules(self.skill_refused({"skill": skill}, self.ABOVE)), [])
|
||||
|
||||
def test_a_security_skill_inside_a_subagent_runs_on_the_rung_its_launch_was_held_to(self):
|
||||
self.assertEqual(rules(reviewer_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose"), self.ENVIRON)), [])
|
||||
def test_a_security_skill_runs_on_its_callers_rung_whatever_model_its_input_names(self):
|
||||
self.assertEqual(rules(self.skill_refused({"skill": "security-review", "model": "opus", "effort": "medium"})), ["reviewer-rung"])
|
||||
|
||||
def test_any_other_skill_is_not_checked(self):
|
||||
self.assertEqual(rules(reviewer_breaches(Call("Skill", {"skill": "simplify"}, "/r"), self.ENVIRON)), [])
|
||||
def test_a_security_skill_passes_only_in_a_subagent_launched_as_the_security_pass(self):
|
||||
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ENVIRON, "You are the security pass for this chunk. Invoke /security-review.")), [])
|
||||
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ABOVE, "Invoke /security-review.")), ["reviewer-rung"], "a subagent's rung is unknown, whatever the worker's")
|
||||
unreadable = skill_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose", "/nonexistent/agent-a1.jsonl", True), self.ENVIRON)
|
||||
self.assertEqual(rules(unreadable), ["reviewer-rung"])
|
||||
|
||||
def test_any_other_skill_or_a_claude_loop_claude_did_not_start_is_not_checked(self):
|
||||
self.assertEqual(rules(self.skill_refused({"skill": "simplify"})), [])
|
||||
self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, {})), [])
|
||||
|
||||
def test_a_session_started_with_an_agent_is_no_subagent(self):
|
||||
hook_input = {"tool_name": "Skill", "tool_input": {"skill": "security-review"}, "cwd": "/r", "agent_type": "claude-security:claude-security", "transcript_path": "/t/s.jsonl"}
|
||||
self.assertFalse(parsed_call(hook_input).subagent)
|
||||
self.assertTrue(parsed_call({**hook_input, "agent_id": "a1"}).subagent)
|
||||
|
||||
def test_a_skill_calls_args_are_the_text_scanned_for_a_marker(self):
|
||||
found = breaches_of(Call("Skill", {"skill": "security-review", "args": "the branch # rule-override reviewer-rung: trust me"}, "/r"), "/h", {})
|
||||
|
||||
Reference in New Issue
Block a user