111 lines
6.7 KiB
Python
111 lines
6.7 KiB
Python
"""Replay archived review rounds separately or merged, and save each replay's reply, reports and cost; analyze.py matches them. Usage: driver.py <candidate> <sep|merged> <run> [case ids...]"""
|
|
import concurrent.futures, json, os, pathlib, pickle, re, shutil, sys, tempfile
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin"))
|
|
from claude_code.reply import claude_reply, Limited
|
|
from review_replay.replay import clone_reviewed, Copies, copy_reports, match, redirected
|
|
from failure import Failure
|
|
from review_replay.rounds import lost_inputs, Round
|
|
from review_rules.review_folder import REPORTS, review_folder
|
|
from review_replay import replay as rm
|
|
from model_routing.ladder import Rung
|
|
from loop_effects.effects import system_effects
|
|
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
HOME = pathlib.Path.home()
|
|
RUNG = Rung("claude-opus-5-5", "medium")
|
|
FX = system_effects()
|
|
PASS = {"stability-reviewer": "stability", "product-owner-reviewer": "product-owner", "prose-reviewer": "prose", "systems-architect": "architecture", "principles-auditor": "principles", "override-reviewer": "overrides"}
|
|
MERGED = {"wave3": ("wave3-reviewer", "agents-wave3.json"), "release": ("release-reviewer", "agents-release.json"), "pair": ("pair-reviewer", "agents-pair.json")}
|
|
|
|
def as_round(r):
|
|
return Round(r["reviewer"], r["checkout"], r["branch"], r["at"], r["began"], r["prompt"], tuple(r["findings"]))
|
|
|
|
def prepared(rnd, code, copies, clone=None, cutoff=None):
|
|
folder = pathlib.Path(review_folder(str(HOME), code.path))
|
|
# Every variant of a case sees only the reports written before the case's first round, so no variant is handed another pass's findings.
|
|
copy_reports(cutoff or rnd, folder, copies.reports)
|
|
p = redirected(rnd.prompt, HOME, folder, code.path, copies)
|
|
if isinstance(p, Failure): return p
|
|
# Inputs rewritten after the round: ls-files is regenerated from the clone; a places list cannot be, so its file says it is missing.
|
|
for name in lost_inputs(p, copies.reports, lambda n: (pathlib.Path(copies.reports) / n).exists()):
|
|
target = pathlib.Path(copies.reports) / name
|
|
if name == "ls-files.txt" and clone:
|
|
out = FX.git("ls-files", cwd=clone)
|
|
if not isinstance(out, Failure): target.write_text(out)
|
|
elif name.startswith("places.") and clone:
|
|
target.write_text("(The places list for this round was not archived; no places are handed to you.)\n")
|
|
if lost := lost_inputs(p, copies.reports, lambda n: (pathlib.Path(copies.reports) / n).exists()):
|
|
return Failure(f"lost inputs {lost}")
|
|
return p
|
|
|
|
def run(rounds, codes, agent_args, prompt_of, out, cutoff=None):
|
|
"""Clone at the first round's code, prepare prompts, run claude, keep reply and reports written."""
|
|
if out.exists():
|
|
return json.loads(out.read_text())
|
|
(HOME / REPORTS).mkdir(parents=True, exist_ok=True)
|
|
(HERE / "clones").mkdir(exist_ok=True)
|
|
first = rounds[0]
|
|
with tempfile.TemporaryDirectory(prefix="rm-", dir=HERE / "clones", ignore_cleanup_errors=True) as clone, \
|
|
tempfile.TemporaryDirectory(prefix="review-replay-rm-", dir=HOME / REPORTS, ignore_cleanup_errors=True) as reports:
|
|
copies = Copies(clone, reports)
|
|
if isinstance(c := clone_reviewed(first, codes[first], clone, FX), Failure): return {"error": c.reason}
|
|
prompts = []
|
|
for r in rounds:
|
|
p = prepared(r, codes[r], copies, clone, cutoff)
|
|
if isinstance(p, Failure): return {"error": p.reason}
|
|
prompts.append(p)
|
|
before = set(os.listdir(reports))
|
|
prompt = prompt_of(prompts, reports)
|
|
reply = None
|
|
for _ in range(3):
|
|
reply = claude_reply(FX.claude, [*agent_args, *RUNG.flags()], clone, prompt, "claude")
|
|
if not isinstance(reply, Failure): break
|
|
if isinstance(reply, Limited): print("LIMITED", reply.said); sys.exit(2)
|
|
if isinstance(reply, Failure): result = {"error": reply.reason}
|
|
else:
|
|
written = {n: (pathlib.Path(reports)/n).read_text(errors="replace") for n in set(os.listdir(reports)) - before if (pathlib.Path(reports)/n).is_file()}
|
|
result = {"reply": reply, "written": written, "prompt": prompt}
|
|
out.write_text(json.dumps(result))
|
|
return result
|
|
|
|
def load_cases(cand):
|
|
return json.loads((HERE / f"cases-{cand}.json").read_text())
|
|
|
|
def main():
|
|
cand, variant, rep = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
runs = pickle.load(open(HERE / "runs.pkl", "rb"))
|
|
by_meta = {r["meta"]: r for r in runs}
|
|
cases = load_cases(cand)
|
|
if sys.argv[4:]: cases = [c for c in cases if c["id"] in sys.argv[4:]]
|
|
outdir = HERE / "results" / cand / f"{variant}-{rep}"; outdir.mkdir(parents=True, exist_ok=True)
|
|
jobs = []
|
|
for case in cases:
|
|
rounds = [as_round(by_meta[m]) for m in case["metas"]]
|
|
codes = {}
|
|
for r in rounds:
|
|
c = rm.code_of(r, FX)
|
|
if isinstance(c, Failure): print(case["id"], "no code:", c.reason); break
|
|
codes[r] = c
|
|
else:
|
|
# one clone for all: the code of the first round
|
|
for r in rounds: codes[r] = codes[rounds[0]]
|
|
if variant == "sep":
|
|
for r in rounds:
|
|
jobs.append(([r], codes, ["--agent", r.reviewer], lambda ps, rep_: ps[0], outdir / f"{case['id']}--{PASS[r.reviewer]}.json", rounds[0]))
|
|
else:
|
|
name, path = MERGED[cand]
|
|
def prompt_of(ps, reports, rounds=rounds, name=name):
|
|
briefs = "\n\n".join(f"=== The worker's brief to the {PASS[r.reviewer]} pass ===\n\n{p}" for r, p in zip(rounds, ps))
|
|
return (f"You review this change once, in place of the {', '.join(PASS[r.reviewer] for r in rounds)} passes. The worker's brief to each of those passes follows; carry out all of them. "
|
|
f"Each brief names its own report file: write instead one report, holding every brief's findings under its own heading, to {reports}/{name}-round-1.md, and reply with a short summary of the findings and their severities, one line per finding under each pass's heading.\n\n{briefs}")
|
|
jobs.append((rounds, codes, ["--agents", str(HERE / path), "--agent", name], prompt_of, outdir / f"{case['id']}--merged.json", rounds[0]))
|
|
with concurrent.futures.ThreadPoolExecutor(int(os.environ.get("AT_ONCE", "4"))) as pool:
|
|
futs = {pool.submit(run, *j): j[4].name for j in jobs}
|
|
for f in concurrent.futures.as_completed(futs):
|
|
res = f.result()
|
|
cost = res.get("reply", {}).get("total_cost_usd") if "reply" in res else None
|
|
print(futs[f], "error: " + res["error"] if "error" in res else f"${cost:.2f}", flush=True)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|