Load no MCP server in the bench's headless claude calls that are denied tools
CI / tests (pull_request) Successful in 3m45s
CI / tests (pull_request) Successful in 3m45s
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
import ast
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from repo_checks import ROOT
|
||||
|
||||
|
||||
def denies_tools_without_strict_mcp(source: str) -> bool:
|
||||
"""Whether the source passes "--tools" but never "--strict-mcp-config", so the MCP servers of the user's config would load beside the denied tools."""
|
||||
strings = {node.value for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Constant) and isinstance(node.value, str)}
|
||||
return "--tools" in strings and "--strict-mcp-config" not in strings
|
||||
|
||||
|
||||
class HeadlessMcp(unittest.TestCase):
|
||||
def test_each_file_that_passes_tools_passes_strict_mcp_config_too(self):
|
||||
# A file not yet staged counts too, so a new call is checked before its first commit.
|
||||
listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "*.py"], capture_output=True, check=True, cwd=ROOT, text=True).stdout.split()
|
||||
self.assertIn("reviewer-merge-trials/analyze.py", listed)
|
||||
self.assertIn("bin/agent_cycle/judge.py", listed)
|
||||
files = [name for name in listed if not pathlib.PurePath(name).name.startswith("test_")]
|
||||
lacking = {name for name in files if denies_tools_without_strict_mcp((ROOT / name).read_text(encoding="utf-8"))}
|
||||
self.assertEqual(lacking, set(), "add --strict-mcp-config beside --tools, so no MCP server loads where the prompt holds text an outsider can write")
|
||||
|
||||
def test_a_file_with_tools_and_no_strict_mcp_config_is_found(self):
|
||||
self.assertTrue(denies_tools_without_strict_mcp('args = ["--tools", ""]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["--tools", "", "--strict-mcp-config"]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["-p"]'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -3219,3 +3219,20 @@ Read most (opens and searches):
|
||||
- `README.md` 4
|
||||
- `bin/do_epic/backend/contract.py` 4
|
||||
- `bin/do_epic/backend/fake.py` 4
|
||||
|
||||
## Reading cost, chunk 381d450e408f, finished 2026-11-20T17:01:28Z
|
||||
|
||||
71227 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `bin/agent_cycle/test_judge.py` 5
|
||||
- `bin/agent_cycle/test_stuck_workers.py` 3
|
||||
- `bin/do_epic/test_split.py` 2
|
||||
- `bin/review_replay/test_review_replay.py` 2
|
||||
- `README.md` 1
|
||||
- `bin/agent_cycle/judge.py` 1
|
||||
- `bin/do_epic/repo_pick.py` 1
|
||||
- `bin/do_epic/test_issue_scores.py` 1
|
||||
- `bin/do_epic/test_repo_pick.py` 1
|
||||
- `bin/model_routing/ladder.py` 1
|
||||
|
||||
@@ -51,7 +51,7 @@ def new_findings(arch, text):
|
||||
"For each give its severity, the pass or report heading it sits under (such as stability, product-owner, prose, architecture, principles), its file:line or place, and a one-sentence summary. "
|
||||
"A finding the report raises twice, under two headings, is listed once.\n\n"
|
||||
f"The earlier findings:\n{numbered}\n\nThe later report:\n{text}")
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", ""]
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", "", "--strict-mcp-config"]
|
||||
for _ in range(3):
|
||||
rep = claude_reply(d.FX.claude, args, str(HERE), prompt, "claude")
|
||||
if not isinstance(rep, Failure):
|
||||
|
||||
@@ -182,7 +182,7 @@ def grade(bench, work, name):
|
||||
text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
|
||||
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
|
||||
reply = bench.process(
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""],
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"],
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
verdict = verdict_of(reply)
|
||||
(work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]}))
|
||||
|
||||
@@ -244,7 +244,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
|
||||
((judge, kwargs),) = self.calls
|
||||
self.assertEqual(judge[:2] + judge[3:], [
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"])
|
||||
self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True))
|
||||
prompt = judge[2]
|
||||
self.assertIn("Do the seventh thing.", prompt)
|
||||
|
||||
Reference in New Issue
Block a user