diff --git a/bin/repo_checks/test_headless_mcp.py b/bin/repo_checks/test_headless_mcp.py new file mode 100644 index 00000000..29642f02 --- /dev/null +++ b/bin/repo_checks/test_headless_mcp.py @@ -0,0 +1,35 @@ +import ast +import pathlib +import subprocess +import sys +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from repo_checks import ROOT + + +def denies_tools_without_strict_mcp(source: str) -> bool: + """Whether the source passes "--tools" but never "--strict-mcp-config", so the MCP servers of the user's config would load beside the denied tools.""" + strings = {node.value for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Constant) and isinstance(node.value, str)} + return "--tools" in strings and "--strict-mcp-config" not in strings + + +class HeadlessMcp(unittest.TestCase): + def test_each_file_that_passes_tools_passes_strict_mcp_config_too(self): + # A file not yet staged counts too, so a new call is checked before its first commit. + listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "*.py"], capture_output=True, check=True, cwd=ROOT, text=True).stdout.split() + self.assertIn("reviewer-merge-trials/analyze.py", listed) + self.assertIn("bin/agent_cycle/judge.py", listed) + files = [name for name in listed if not pathlib.PurePath(name).name.startswith("test_")] + lacking = {name for name in files if denies_tools_without_strict_mcp((ROOT / name).read_text(encoding="utf-8"))} + self.assertEqual(lacking, set(), "add --strict-mcp-config beside --tools, so no MCP server loads where the prompt holds text an outsider can write") + + def test_a_file_with_tools_and_no_strict_mcp_config_is_found(self): + self.assertTrue(denies_tools_without_strict_mcp('args = ["--tools", ""]')) + self.assertFalse(denies_tools_without_strict_mcp('args = ["--tools", "", "--strict-mcp-config"]')) + self.assertFalse(denies_tools_without_strict_mcp('args = ["-p"]')) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/reading-cost.md b/docs/reading-cost.md index e24b99c9..9b21d49f 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3219,3 +3219,20 @@ Read most (opens and searches): - `README.md` 4 - `bin/do_epic/backend/contract.py` 4 - `bin/do_epic/backend/fake.py` 4 + +## Reading cost, chunk 381d450e408f, finished 2026-11-20T17:01:28Z + +71227 new input tokens before the first commit. + +Read most (opens and searches): + +- `bin/agent_cycle/test_judge.py` 5 +- `bin/agent_cycle/test_stuck_workers.py` 3 +- `bin/do_epic/test_split.py` 2 +- `bin/review_replay/test_review_replay.py` 2 +- `README.md` 1 +- `bin/agent_cycle/judge.py` 1 +- `bin/do_epic/repo_pick.py` 1 +- `bin/do_epic/test_issue_scores.py` 1 +- `bin/do_epic/test_repo_pick.py` 1 +- `bin/model_routing/ladder.py` 1 diff --git a/reviewer-merge-trials/analyze.py b/reviewer-merge-trials/analyze.py index 3d9a734d..1df343a4 100644 --- a/reviewer-merge-trials/analyze.py +++ b/reviewer-merge-trials/analyze.py @@ -51,7 +51,7 @@ def new_findings(arch, text): "For each give its severity, the pass or report heading it sits under (such as stability, product-owner, prose, architecture, principles), its file:line or place, and a one-sentence summary. " "A finding the report raises twice, under two headings, is listed once.\n\n" f"The earlier findings:\n{numbered}\n\nThe later report:\n{text}") - args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", ""] + args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", "", "--strict-mcp-config"] for _ in range(3): rep = claude_reply(d.FX.claude, args, str(HERE), prompt, "claude") if not isinstance(rep, Failure): diff --git a/workflow-bench/advisor_bench.py b/workflow-bench/advisor_bench.py index ea294554..d8c3d0ee 100644 --- a/workflow-bench/advisor_bench.py +++ b/workflow-bench/advisor_bench.py @@ -182,7 +182,7 @@ def grade(bench, work, name): text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"), diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text()) reply = bench.process( - ["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""], + ["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"], capture_output=True, text=True, check=True).stdout verdict = verdict_of(reply) (work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]})) diff --git a/workflow-bench/test_advisor_bench.py b/workflow-bench/test_advisor_bench.py index 3ab75b44..a882f331 100644 --- a/workflow-bench/test_advisor_bench.py +++ b/workflow-bench/test_advisor_bench.py @@ -244,7 +244,7 @@ class Orchestration(unittest.TestCase): self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass") ((judge, kwargs),) = self.calls self.assertEqual(judge[:2] + judge[3:], [ - "claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""]) + "claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"]) self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True)) prompt = judge[2] self.assertIn("Do the seventh thing.", prompt)