Load no MCP server in the bench's headless claude calls that are denied tools #671
@@ -2,6 +2,7 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Load no MCP server in the bench's headless `claude -p` judge and in the reviewer-merge trials' new-findings call either, so text an outsider wrote cannot reach one. A new check fails any Python file that passes `--tools` more often than `--strict-mcp-config`.
|
||||
- Load no MCP server in the six headless `claude -p` calls that deny the model tools, so text an outsider wrote cannot reach one.
|
||||
- `do-epic` starts an epic's runnable issues in order of score alone, not in Jira's backlog order. It scores each open issue that holds no scores, the ones people filed included, and comments on the issue why.
|
||||
- do-epic and `todo-item add` file issues as the Jira issue types the settings `jira-finding-type` and `jira-review-type` name, Task by default, and refuse before filing where the Jira project lacks the type, naming the types it has.
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import ast
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from repo_checks import ROOT
|
||||
|
||||
|
||||
def denies_tools_without_strict_mcp(source: str) -> bool:
|
||||
"""Whether the source passes "--tools" more often than "--strict-mcp-config", so a call that denies tools would load the MCP servers of the user's or the project's config."""
|
||||
strings = [node.value for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Constant) and isinstance(node.value, str)]
|
||||
return strings.count("--tools") > strings.count("--strict-mcp-config")
|
||||
|
||||
|
||||
class HeadlessMcp(unittest.TestCase):
|
||||
def test_each_file_that_passes_tools_passes_strict_mcp_config_too(self):
|
||||
# Untracked files count too, so a new call is checked before its first commit.
|
||||
listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "*.py"], capture_output=True, check=True, cwd=ROOT, text=True).stdout.split()
|
||||
self.assertIn("reviewer-merge-trials/analyze.py", listed)
|
||||
self.assertIn("bin/agent_cycle/judge.py", listed)
|
||||
files = [name for name in listed if not pathlib.PurePath(name).name.startswith("test_")]
|
||||
lacking = {name for name in files if denies_tools_without_strict_mcp((ROOT / name).read_text(encoding="utf-8"))}
|
||||
self.assertEqual(lacking, set(), "add --strict-mcp-config beside --tools, so no MCP server loads where the prompt holds text an outsider can write")
|
||||
|
||||
def test_a_file_with_tools_and_no_strict_mcp_config_is_found(self):
|
||||
self.assertTrue(denies_tools_without_strict_mcp('args = ["--tools", ""]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["--tools", "", "--strict-mcp-config"]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["-p"]'))
|
||||
|
||||
def test_a_second_call_without_the_flag_is_found_in_a_file_where_another_call_has_it(self):
|
||||
self.assertTrue(denies_tools_without_strict_mcp('a = ["--tools", "", "--strict-mcp-config"]\nb = ["--tools", ""]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('a = ["--tools", "", "--strict-mcp-config"]\nb = ["--tools", "", "--strict-mcp-config"]'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -3219,3 +3219,20 @@ Read most (opens and searches):
|
||||
- `README.md` 4
|
||||
- `bin/do_epic/backend/contract.py` 4
|
||||
- `bin/do_epic/backend/fake.py` 4
|
||||
|
||||
## Reading cost, chunk 381d450e408f, finished 2026-11-20T17:01:28Z
|
||||
|
||||
71227 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `bin/agent_cycle/test_judge.py` 5
|
||||
- `bin/agent_cycle/test_stuck_workers.py` 3
|
||||
- `bin/do_epic/test_split.py` 2
|
||||
- `bin/review_replay/test_review_replay.py` 2
|
||||
- `README.md` 1
|
||||
- `bin/agent_cycle/judge.py` 1
|
||||
- `bin/do_epic/repo_pick.py` 1
|
||||
- `bin/do_epic/test_issue_scores.py` 1
|
||||
- `bin/do_epic/test_repo_pick.py` 1
|
||||
- `bin/model_routing/ladder.py` 1
|
||||
|
||||
@@ -51,7 +51,7 @@ def new_findings(arch, text):
|
||||
"For each give its severity, the pass or report heading it sits under (such as stability, product-owner, prose, architecture, principles), its file:line or place, and a one-sentence summary. "
|
||||
"A finding the report raises twice, under two headings, is listed once.\n\n"
|
||||
f"The earlier findings:\n{numbered}\n\nThe later report:\n{text}")
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", ""]
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", "", "--strict-mcp-config"]
|
||||
for _ in range(3):
|
||||
rep = claude_reply(d.FX.claude, args, str(HERE), prompt, "claude")
|
||||
if not isinstance(rep, Failure):
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
| 454 | 0.15.0 | | | **Start a do-epic chunk's issue as the team's board expects, by settings: assigned to the account do-epic works as, moved by a start transition and added to a board's active sprint.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 4 | 1.1 |
|
||||
| 472 | 0.15.0 | 454 | owner | **Let a project's config name a working branch per epic, such as `TW-12345`, that chunks branch off, merge into and keep level with `main`, so a person merges the finished epic into `main` in one pull request.** | 4 | 2 | 3 | 7 | 7 | 1.5 by 6, 1.6 by 7 | 1.1 |
|
||||
| 467 | 0.15.0 | 454, 472 | question | **Cut 0.15.0 as soon as items 449, 450, 453, 454, 468, 470 and 472 have merged, before its release review, and run that review in 0.15.1.** | 1 | 1 | 1 | 1 | 1 | | 0.0 |
|
||||
| 476 | 0.15.0 | | security | **Load no MCP server in the bench's headless claude calls that are denied tools, as item 473 did for the loop's six.** | 2 | 2 | 1 | 3 | 2 | 4 by 3 | 4.2 |
|
||||
| 469 | 0.15.1 | | | **Move a do-epic parent issue whose sub-tasks a person filed to Release Ready once every sub-task has landed, and to Done once every sub-task is done, by transitions its config names.** | 3 | 2 | 2 | 5 | 4 | 1.2 by 4, 1.5 by 4, 1.6 by 3 | 3.1 |
|
||||
| 446 | 0.16.0 | | question | **Kill the process a stuck check names where its verdict is `stuck`, once the human says the recorded verdicts match what they would have done.** | 2 | 2 | 3 | 5 | 3 | 1.5 by 3 | 1.0 |
|
||||
| 462 | 0.16.0 | | | **Run under `mutate-changes` the tests that today fail in mutmut's copy of the tree, so a mutant of a line only they cover is killed.** | 2 | 1 | 2 | 3 | 3 | code health by 3 | 4.7 |
|
||||
@@ -136,26 +135,6 @@ Touches:
|
||||
|
||||
- `CHANGELOG.md`: `## Unreleased` (small)
|
||||
|
||||
### 476. Load no MCP server in the bench's headless claude calls that are denied tools, as item 473 did for the loop's six.
|
||||
|
||||
Exposure: item 473 added `--strict-mcp-config`, without `--mcp-config`, beside `--tools` at six headless `claude -p` calls in `bin/` that deny the model tools while their prompt holds text an outsider can write. Without the flag, MCP servers from the user's or the project's config still load, so a prompt injection in that text can call an MCP tool. Two more calls in the repo's bench folders deny tools the same way and still lack the flag:
|
||||
|
||||
- `workflow-bench/advisor_bench.py:185`, the judge's argument list in `grade`. `--setting-sources ""` doesn't stop MCP servers from the user's config. Its prompt holds a plan item's text and two diffs of this repo, so an outsider reaches it only through a merged change.
|
||||
- `reviewer-merge-trials/analyze.py:54`, `args` in `new_findings`. Its prompt holds reviewer reports, which quote the reviewed project's code, so whoever wrote that code reaches it. `matched` in the same file calls review_replay's `match`, which item 473 already fixed.
|
||||
|
||||
Fix: pass `--strict-mcp-config` beside `--tools` at both. In `workflow-bench/test_advisor_bench.py`, extend the grade test's expected arguments in `Orchestration`. `reviewer-merge-trials/` has no test file, and no test can import `analyze.py`, since it loads the gitignored `runs.pkl` when imported. So add a check under `bin/repo_checks/` instead: every Python file of the repo, tests left out, that passes `"--tools"` passes `"--strict-mcp-config"` too. It covers both sites and item 473's six, and fails on the next call that denies tools without the flag. That needs item 473's six in place, hence the block.
|
||||
|
||||
The trials folder isn't frozen. `docs/decisions.md` → the reviewer-merge trials entry says to replay the merge with these scripts when a reviewer's definition changes, and `bin/repo_checks/test_trials_imports.py` keeps their imports from `bin/` working for that. A comprehension seat called the trial frozen at `reviewer-merge-trials/driver.py:4`; this item doesn't touch that place. The flag changes no recorded result, so `reviewer-merge-trials/README.md` stays as it is.
|
||||
|
||||
Found while shipping item 473.
|
||||
|
||||
Touches:
|
||||
|
||||
- `workflow-bench/advisor_bench.py`: `grade` (small)
|
||||
- `workflow-bench/test_advisor_bench.py`: `Orchestration` (small)
|
||||
- `reviewer-merge-trials/analyze.py`: `new_findings` (small)
|
||||
- `bin/repo_checks/test_headless_mcp.py`
|
||||
|
||||
### 469. Move a do-epic parent issue whose sub-tasks a person filed to Release Ready once every sub-task has landed, and to Done once every sub-task is done, by transitions its config names.
|
||||
|
||||
do-epic moves to done only a parent it split itself (`split_items_closed`). A parent whose sub-tasks a person filed stays as Jira holds it, and `--status` says "its sub-tasks are done; close it in Jira". Until someone moves it, the parent blocks every issue it blocks, and through it their sub-tasks. On TW-13503, TW-13505 blocks most of the critical path this way, and TW-13547's sub-tasks are all Release Ready while the parent stays In Progress. The human asked, on 2026-11-20, for deterministic code to move these parents, not a worker or a person.
|
||||
|
||||
@@ -182,7 +182,7 @@ def grade(bench, work, name):
|
||||
text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
|
||||
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
|
||||
reply = bench.process(
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""],
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"],
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
verdict = verdict_of(reply)
|
||||
(work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]}))
|
||||
|
||||
@@ -244,7 +244,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
|
||||
((judge, kwargs),) = self.calls
|
||||
self.assertEqual(judge[:2] + judge[3:], [
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"])
|
||||
self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True))
|
||||
prompt = judge[2]
|
||||
self.assertIn("Do the seventh thing.", prompt)
|
||||
|
||||
Reference in New Issue
Block a user