Merge remote-tracking branch 'origin/main' into item-454
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Load no MCP server in the bench's headless `claude -p` judge and in the reviewer-merge trials' new-findings call either, so text an outsider wrote cannot reach one. A new check fails any Python file that passes `--tools` more often than `--strict-mcp-config`.
|
||||
- Load no MCP server in the six headless `claude -p` calls that deny the model tools, so text an outsider wrote cannot reach one.
|
||||
- `do-epic` starts an epic's runnable issues in order of score alone, not in Jira's backlog order. It scores each open issue that holds no scores, the ones people filed included, and comments on the issue why.
|
||||
- do-epic and `todo-item add` file issues as the Jira issue types the settings `jira-finding-type` and `jira-review-type` name, Task by default, and refuse before filing where the Jira project lacks the type, naming the types it has.
|
||||
- do-epic names the team's own label for each repo, from the `[label-repos]` table, when it tells a chunk's worker how to file a finding or a release review, and keeps `repo:<name>` for a repo the table doesn't name.
|
||||
|
||||
@@ -208,7 +208,7 @@ def judge_args(rung: Rung, driver: str) -> list[str]:
|
||||
return [
|
||||
"--json-schema", json.dumps(judge_schema(driver)),
|
||||
*rung.flags(),
|
||||
"--tools", "",
|
||||
"--tools", "", "--strict-mcp-config",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ def command_of(pid: int, shells: list[Shell]) -> str:
|
||||
def investigated(shells: list[Shell], rung: Rung, root: str, effects: Effects) -> dict[str, typing.Any] | Failure:
|
||||
"""A headless claude's verdict on the worker holding shells, as SCHEMA has it, or why there is none. It runs without tools, so it judges only the evidence it is handed."""
|
||||
evidence = "\n\n".join(f"Shell {shell.pid}: {shell.command}\nIts processes, as pid, state, CPU and command line:\n{chr(10).join(shell.processes) or 'none'}\nThe end of its output:\n{shell.output or 'none'}" for shell in shells)
|
||||
reply = claude_reply(effects.claude, ["--json-schema", json.dumps(SCHEMA), *rung.flags(), "--tools", ""], root, PROMPT + "\n" + evidence, "the stuck check")
|
||||
reply = claude_reply(effects.claude, ["--json-schema", json.dumps(SCHEMA), *rung.flags(), "--tools", "", "--strict-mcp-config"], root, PROMPT + "\n" + evidence, "the stuck check")
|
||||
if isinstance(reply, Limited):
|
||||
return Failure(f"claude hit a limit: {reply.said}")
|
||||
if isinstance(reply, Failure):
|
||||
|
||||
@@ -139,6 +139,11 @@ class JudgePrompt(unittest.TestCase):
|
||||
"""What the judge of a-driver reads of a transcript holding entries."""
|
||||
return judge.Brief("Do it.", "## Goals", records, "the project's decisions", answer_destinations, kinds(*entries), escalated_at, "a-driver")
|
||||
|
||||
def test_runs_with_no_tools_and_loads_no_mcp_server(self):
|
||||
args = judge.judge_args(Rung("claude-opus-5-5", "medium"), "a-driver")
|
||||
self.assertEqual(args[args.index("--tools") + 1 :], ["", "--strict-mcp-config"])
|
||||
self.assertNotIn("--mcp-config", args)
|
||||
|
||||
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
||||
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
||||
prompt, grounds = judge.judge_prompt(self.brief(entries, 0))
|
||||
|
||||
@@ -36,7 +36,11 @@ class StuckWorkers(unittest.TestCase):
|
||||
if each_check:
|
||||
streamed.shells = lambda: [each_check.pop(0)]
|
||||
fake = FakeEffects(streams=[streamed], claude=replies)
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
|
||||
self.claude_folders = []
|
||||
def claude(args, cwd, stdin):
|
||||
self.claude_folders.append(cwd)
|
||||
return fake.effects().claude(args, cwd, stdin)
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0], claude=claude)
|
||||
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said, contextlib.redirect_stderr(io.StringIO()) as failed:
|
||||
archive = archive or pathlib.Path(folder, "transcripts", "project")
|
||||
worker.run_worker("/r", [], {}, archive, RUNG, effects)
|
||||
@@ -47,7 +51,8 @@ class StuckWorkers(unittest.TestCase):
|
||||
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict()])
|
||||
[(args, prompt)] = fake.made("claude")
|
||||
self.assertEqual(args[args.index("--model") + 1 :][:3], ["claude-opus-5-5", "--effort", "medium"])
|
||||
self.assertEqual(args[args.index("--tools") + 1], "")
|
||||
self.assertEqual(args[args.index("--tools") + 1 :][:2], ["", "--strict-mcp-config"])
|
||||
self.assertEqual(self.claude_folders, ["/r"])
|
||||
self.assertEqual(json.loads(args[args.index("--json-schema") + 1])["properties"]["verdict"]["enum"], ["working", "waiting on a person", "stuck"])
|
||||
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\n8 S 2.0 s python3 -m pytest\n9 S 0.0 s sh -c trap '' INT; sleep 999\nThe end of its output:\ncollected 40 items"))
|
||||
line = "stuck check: a headless claude judged the worker stuck, naming process 8 (python3 -m pytest): pytest waits on a shell that ignores SIGINT" + sign(1600, 1600)
|
||||
|
||||
@@ -33,7 +33,7 @@ class Scores:
|
||||
|
||||
def asked(issue: EpicIssue, rule: str, goals: str, language: str, rung: Rung, claude: Claude, cwd: str) -> Scores | Failure | Limited:
|
||||
"""The scores claude gives the issue. rule is the text every plan item is scored by, and goals says which goals the issue may move and how to name each. Nothing is written: the caller checks the answer against the plan's format and sets it."""
|
||||
reply = claude_reply(claude, ["--json-schema", json.dumps(SCHEMA), *rung.flags(), "--tools", ""], cwd, prompt(issue, rule, goals, language), "the scoring")
|
||||
reply = claude_reply(claude, ["--json-schema", json.dumps(SCHEMA), *rung.flags(), "--tools", "", "--strict-mcp-config"], cwd, prompt(issue, rule, goals, language), "the scoring")
|
||||
return reply if isinstance(reply, Failure | Limited) else answer_of(structured_output(reply))
|
||||
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ def picked(store: EpicStore, issue: EpicIssue, team: LabelRepos, rule: str, rung
|
||||
known = recorded(issue, team)
|
||||
if known is not None:
|
||||
return known
|
||||
reply = claude_reply(claude, ["--json-schema", json.dumps(PICK_SCHEMA), *rung.flags(), "--tools", ""], cwd, prompt(rule, issue), "the repo pick")
|
||||
reply = claude_reply(claude, ["--json-schema", json.dumps(PICK_SCHEMA), *rung.flags(), "--tools", "", "--strict-mcp-config"], cwd, prompt(rule, issue), "the repo pick")
|
||||
if isinstance(reply, Failure | Limited):
|
||||
return reply
|
||||
answer = answer_of(structured_output(reply))
|
||||
|
||||
@@ -123,7 +123,7 @@ def finished(issues: list[EpicIssue], key: str) -> bool:
|
||||
|
||||
def prediction(issue: EpicIssue, by_person: bool, rule: str, language: str, checkouts: tuple[str, ...], rung: Rung, claude: Claude, cwd: str) -> Prediction | Failure | Limited:
|
||||
"""The repos claude predicts the item changes, from the code in checkouts."""
|
||||
flags = ["--json-schema", json.dumps(SPLIT_SCHEMA), *rung.flags(), "--tools", "Read,Grep,Glob", *(arg for each in checkouts for arg in ("--add-dir", each))]
|
||||
flags = ["--json-schema", json.dumps(SPLIT_SCHEMA), *rung.flags(), "--tools", "Read,Grep,Glob", "--strict-mcp-config", *(arg for each in checkouts for arg in ("--add-dir", each))]
|
||||
reply = claude_reply(claude, flags, cwd, prompt(issue, by_person, rule, language, checkouts), "the repo split")
|
||||
return reply if isinstance(reply, Failure | Limited) else answer_of(structured_output(reply))
|
||||
|
||||
|
||||
@@ -53,12 +53,12 @@ class TheCall(unittest.TestCase):
|
||||
self.assertEqual(self.ask(claude), answer_of(ANSWER))
|
||||
args, cwd, _ = claude.calls[0]
|
||||
self.assertEqual(cwd, "/state/scoring")
|
||||
self.assertEqual(args[:9], ["-p", "--json-schema", json.dumps(SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", ""])
|
||||
self.assertEqual(args[:10], ["-p", "--json-schema", json.dumps(SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", "", "--strict-mcp-config"])
|
||||
|
||||
def test_names_the_scoring_where_claude_fails(self) -> None:
|
||||
failed = self.ask(Claude("", 1))
|
||||
assert isinstance(failed, Failure), failed
|
||||
self.assertIn("the scoring", failed.reason)
|
||||
self.assertEqual(failed.reason, "the scoring exited 1")
|
||||
|
||||
def test_puts_the_rule_the_goals_the_language_and_the_item_in_the_prompt(self) -> None:
|
||||
text = prompt(ISSUE, "L is the chance of a defect.", "An item moves only these: `epic 1`.", "Swedish")
|
||||
|
||||
@@ -57,7 +57,7 @@ class RepoPick(unittest.TestCase):
|
||||
self.pick(claude)
|
||||
args, cwd, _ = claude.calls[0]
|
||||
self.assertEqual(cwd, "/rules")
|
||||
self.assertEqual(args[:9], ["-p", "--json-schema", json.dumps(PICK_SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", ""])
|
||||
self.assertEqual(args[:10], ["-p", "--json-schema", json.dumps(PICK_SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", "", "--strict-mcp-config"])
|
||||
|
||||
def test_puts_the_rule_and_the_item_in_the_prompt(self) -> None:
|
||||
claude = says({"pick": "claude"})
|
||||
|
||||
@@ -156,8 +156,8 @@ class Split_(unittest.TestCase):
|
||||
self.run_split(claude)
|
||||
args, cwd, prompt = claude.calls[0]
|
||||
self.assertEqual(cwd, "/rules")
|
||||
self.assertEqual(args[:12], ["-p", "--json-schema", json.dumps(SPLIT_SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", "Read,Grep,Glob", "--add-dir", "/repos/api", "--add-dir"])
|
||||
self.assertEqual(args[12], "/repos/web")
|
||||
self.assertEqual(args[:13], ["-p", "--json-schema", json.dumps(SPLIT_SCHEMA), "--model", "claude-sonnet-5-5", "--effort", "medium", "--tools", "Read,Grep,Glob", "--strict-mcp-config", "--add-dir", "/repos/api", "--add-dir"])
|
||||
self.assertEqual(args[13], "/repos/web")
|
||||
self.assertIn(RULE, prompt)
|
||||
self.assertIn("## Item ABC-100: Count the runs.", prompt)
|
||||
self.assertIn("write each part's change in Swedish.", prompt)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import ast
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from repo_checks import ROOT
|
||||
|
||||
|
||||
def denies_tools_without_strict_mcp(source: str) -> bool:
|
||||
"""Whether the source passes "--tools" more often than "--strict-mcp-config", so a call that denies tools would load the MCP servers of the user's or the project's config."""
|
||||
strings = [node.value for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Constant) and isinstance(node.value, str)]
|
||||
return strings.count("--tools") > strings.count("--strict-mcp-config")
|
||||
|
||||
|
||||
class HeadlessMcp(unittest.TestCase):
|
||||
def test_each_file_that_passes_tools_passes_strict_mcp_config_too(self):
|
||||
# Untracked files count too, so a new call is checked before its first commit.
|
||||
listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "*.py"], capture_output=True, check=True, cwd=ROOT, text=True).stdout.split()
|
||||
self.assertIn("reviewer-merge-trials/analyze.py", listed)
|
||||
self.assertIn("bin/agent_cycle/judge.py", listed)
|
||||
files = [name for name in listed if not pathlib.PurePath(name).name.startswith("test_")]
|
||||
lacking = {name for name in files if denies_tools_without_strict_mcp((ROOT / name).read_text(encoding="utf-8"))}
|
||||
self.assertEqual(lacking, set(), "add --strict-mcp-config beside --tools, so no MCP server loads where the prompt holds text an outsider can write")
|
||||
|
||||
def test_a_file_with_tools_and_no_strict_mcp_config_is_found(self):
|
||||
self.assertTrue(denies_tools_without_strict_mcp('args = ["--tools", ""]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["--tools", "", "--strict-mcp-config"]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('args = ["-p"]'))
|
||||
|
||||
def test_a_second_call_without_the_flag_is_found_in_a_file_where_another_call_has_it(self):
|
||||
self.assertTrue(denies_tools_without_strict_mcp('a = ["--tools", "", "--strict-mcp-config"]\nb = ["--tools", ""]'))
|
||||
self.assertFalse(denies_tools_without_strict_mcp('a = ["--tools", "", "--strict-mcp-config"]\nb = ["--tools", "", "--strict-mcp-config"]'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -191,7 +191,7 @@ def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung,
|
||||
"A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n"
|
||||
f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}"
|
||||
)
|
||||
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""]
|
||||
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", "", "--strict-mcp-config"]
|
||||
reply = claude_reply(effects.claude, args, root, prompt, "claude")
|
||||
if isinstance(reply, Failure | Limited):
|
||||
return reply
|
||||
|
||||
@@ -179,7 +179,7 @@ class Match(unittest.TestCase):
|
||||
self.assertEqual(cwd, "/code/p")
|
||||
self.assertIn("1. 🔴 a\n\n2. 🟡 b\n\nThe other report:\nthe other report", prompt)
|
||||
self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), replay.MATCH_SCHEMA)
|
||||
self.assertEqual(args[args.index("--tools") + 1], "")
|
||||
self.assertEqual(args[args.index("--tools") + 1 :][:2], ["", "--strict-mcp-config"])
|
||||
self.assertEqual(args[args.index("--model"):args.index("--model") + 4], REFERENCE.flags())
|
||||
|
||||
def test_takes_the_numbers_in_range_and_fails_on_a_reply_of_another_shape(self):
|
||||
|
||||
@@ -3203,6 +3203,23 @@ Read most (opens and searches):
|
||||
- `bin/atlassian/test_jira.py` 3
|
||||
- `bin/do_epic/test_do_epic.py` 3
|
||||
|
||||
## Reading cost, chunk 381d450e408f, finished 2026-11-20T17:01:28Z
|
||||
|
||||
71227 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `bin/agent_cycle/test_judge.py` 5
|
||||
- `bin/agent_cycle/test_stuck_workers.py` 3
|
||||
- `bin/do_epic/test_split.py` 2
|
||||
- `bin/review_replay/test_review_replay.py` 2
|
||||
- `README.md` 1
|
||||
- `bin/agent_cycle/judge.py` 1
|
||||
- `bin/do_epic/repo_pick.py` 1
|
||||
- `bin/do_epic/test_issue_scores.py` 1
|
||||
- `bin/do_epic/test_repo_pick.py` 1
|
||||
- `bin/model_routing/ladder.py` 1
|
||||
|
||||
## Reading cost, chunk 13e4cd1bf87d, finished 2026-11-20T16:14:53Z
|
||||
|
||||
256462 new input tokens before the first commit.
|
||||
|
||||
@@ -51,7 +51,7 @@ def new_findings(arch, text):
|
||||
"For each give its severity, the pass or report heading it sits under (such as stability, product-owner, prose, architecture, principles), its file:line or place, and a one-sentence summary. "
|
||||
"A finding the report raises twice, under two headings, is listed once.\n\n"
|
||||
f"The earlier findings:\n{numbered}\n\nThe later report:\n{text}")
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", ""]
|
||||
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", "", "--strict-mcp-config"]
|
||||
for _ in range(3):
|
||||
rep = claude_reply(d.FX.claude, args, str(HERE), prompt, "claude")
|
||||
if not isinstance(rep, Failure):
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
`Bar = 0.1`
|
||||
|
||||
`Next ID = 476`
|
||||
`Next ID = 477`
|
||||
|
||||
## Items
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
| 454 | 0.15.0 | | | **Start a do-epic chunk's issue as the team's board expects, by settings: assigned to the account do-epic works as, moved by a start transition and added to a board's active sprint.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 4 | 1.1 |
|
||||
| 472 | 0.15.0 | 454 | owner | **Let a project's config name a working branch per epic, such as `TW-12345`, that chunks branch off, merge into and keep level with `main`, so a person merges the finished epic into `main` in one pull request.** | 4 | 2 | 3 | 7 | 7 | 1.5 by 6, 1.6 by 7 | 1.1 |
|
||||
| 467 | 0.15.0 | 454, 472 | question | **Cut 0.15.0 as soon as items 449, 450, 453, 454, 468, 470 and 472 have merged, before its release review, and run that review in 0.15.1.** | 1 | 1 | 1 | 1 | 1 | | 0.0 |
|
||||
| 473 | 0.15.0 | | security | **Load no MCP server in a headless claude call that is denied tools, so text an outsider wrote cannot reach one.** | 2 | 2 | 2 | 4 | 3 | 4 by 4 | 5.0 |
|
||||
| 469 | 0.15.1 | | | **Move a do-epic parent issue whose sub-tasks a person filed to Release Ready once every sub-task has landed, and to Done once every sub-task is done, by transitions its config names.** | 3 | 2 | 2 | 5 | 4 | 1.2 by 4, 1.5 by 4, 1.6 by 3 | 3.1 |
|
||||
| 446 | 0.16.0 | | question | **Kill the process a stuck check names where its verdict is `stuck`, once the human says the recorded verdicts match what they would have done.** | 2 | 2 | 3 | 5 | 3 | 1.5 by 3 | 1.0 |
|
||||
| 462 | 0.16.0 | | | **Run under `mutate-changes` the tests that today fail in mutmut's copy of the tree, so a mutant of a line only they cover is killed.** | 2 | 1 | 2 | 3 | 3 | code health by 3 | 4.7 |
|
||||
@@ -136,36 +135,6 @@ Touches:
|
||||
|
||||
- `CHANGELOG.md`: `## Unreleased` (small)
|
||||
|
||||
### 473. Load no MCP server in a headless claude call that is denied tools, so text an outsider wrote cannot reach one.
|
||||
|
||||
Exposure: six headless `claude -p` calls deny the model tools with `--tools ""`, or with a read-only list, while their prompts hold text an outsider can write. That text is a Jira issue's summary and description, or a worker's transcript. `claude --help` says `--tools` sets only the built-in tools. MCP servers from the user's or the project's config still load, so a prompt injection in that text can call an MCP tool. What it reaches is whatever those servers can do, such as reading or posting in another client's tracker or chat. `--strict-mcp-config` loads only the servers `--mcp-config` names, and none where it names none.
|
||||
|
||||
Call sites:
|
||||
|
||||
- `bin/do_epic/repo_pick.py:105`, in `picked`.
|
||||
- `bin/do_epic/issue_scores.py:36`, in `asked`.
|
||||
- `bin/do_epic/split.py:126`, in `prediction`, which allows Read, Grep and Glob.
|
||||
- `bin/agent_cycle/judge.py:211`, in `judge_args`.
|
||||
- `bin/agent_cycle/stuck_workers.py:107`, in `investigated`.
|
||||
- `bin/review_replay/replay.py:194`, in `match`.
|
||||
|
||||
Fix: pass `--strict-mcp-config`, without `--mcp-config`, beside `--tools` at each call site. Add a test per call site that the flag is passed. Found by item 470's security pass; it is older than that chunk.
|
||||
|
||||
Touches:
|
||||
|
||||
- `bin/do_epic/repo_pick.py`: `picked` (small)
|
||||
- `bin/do_epic/issue_scores.py`: `asked` (small)
|
||||
- `bin/do_epic/split.py`: `prediction` (small)
|
||||
- `bin/agent_cycle/judge.py`: `judge_args` (small)
|
||||
- `bin/agent_cycle/stuck_workers.py`: `investigated` (small)
|
||||
- `bin/review_replay/replay.py`: `match` (small)
|
||||
- `bin/do_epic/test_repo_pick.py`: `RepoPick`
|
||||
- `bin/do_epic/test_issue_scores.py`
|
||||
- `bin/do_epic/test_split.py`: `Split_`
|
||||
- `bin/agent_cycle/test_judge.py`: `JudgePrompt`
|
||||
- `bin/agent_cycle/test_stuck_workers.py`: `StuckWorkers`
|
||||
- `bin/review_replay/test_review_replay.py`
|
||||
|
||||
### 469. Move a do-epic parent issue whose sub-tasks a person filed to Release Ready once every sub-task has landed, and to Done once every sub-task is done, by transitions its config names.
|
||||
|
||||
do-epic moves to done only a parent it split itself (`split_items_closed`). A parent whose sub-tasks a person filed stays as Jira holds it, and `--status` says "its sub-tasks are done; close it in Jira". Until someone moves it, the parent blocks every issue it blocks, and through it their sub-tasks. On TW-13503, TW-13505 blocks most of the critical path this way, and TW-13547's sub-tasks are all Release Ready while the parent stays In Progress. The human asked, on 2026-11-20, for deterministic code to move these parents, not a worker or a person.
|
||||
|
||||
@@ -182,7 +182,7 @@ def grade(bench, work, name):
|
||||
text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
|
||||
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
|
||||
reply = bench.process(
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""],
|
||||
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"],
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
verdict = verdict_of(reply)
|
||||
(work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]}))
|
||||
|
||||
@@ -244,7 +244,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
|
||||
((judge, kwargs),) = self.calls
|
||||
self.assertEqual(judge[:2] + judge[3:], [
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"])
|
||||
self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True))
|
||||
prompt = judge[2]
|
||||
self.assertIn("Do the seventh thing.", prompt)
|
||||
|
||||
Reference in New Issue
Block a user