Files

111 lines
6.7 KiB
Python

"""Replay archived review rounds separately or merged, and save each replay's reply, reports and cost; analyze.py matches them. Usage: driver.py <candidate> <sep|merged> <run> [case ids...]"""
import concurrent.futures, json, os, pathlib, pickle, re, shutil, sys, tempfile
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin"))
from claude_code.reply import claude_reply, Limited
from review_replay.replay import clone_reviewed, Copies, copy_reports, match, redirected
from failure import Failure
from review_replay.rounds import lost_inputs, Round
from review_rules.review_folder import REPORTS, review_folder
from review_replay import replay as rm
from model_routing.ladder import Rung
from loop_effects.effects import system_effects
HERE = pathlib.Path(__file__).resolve().parent
HOME = pathlib.Path.home()
RUNG = Rung("claude-opus-5-5", "medium")
FX = system_effects()
PASS = {"stability-reviewer": "stability", "product-owner-reviewer": "product-owner", "prose-reviewer": "prose", "systems-architect": "architecture", "principles-auditor": "principles", "override-reviewer": "overrides"}
MERGED = {"wave3": ("wave3-reviewer", "agents-wave3.json"), "release": ("release-reviewer", "agents-release.json"), "pair": ("pair-reviewer", "agents-pair.json")}
def as_round(r):
return Round(r["reviewer"], r["checkout"], r["branch"], r["at"], r["began"], r["prompt"], tuple(r["findings"]))
def prepared(rnd, code, copies, clone=None, cutoff=None):
folder = pathlib.Path(review_folder(str(HOME), code.path))
# Every variant of a case sees only the reports written before the case's first round, so no variant is handed another pass's findings.
copy_reports(cutoff or rnd, folder, copies.reports)
p = redirected(rnd.prompt, HOME, folder, code.path, copies)
if isinstance(p, Failure): return p
# Inputs rewritten after the round: ls-files is regenerated from the clone; a places list cannot be, so its file says it is missing.
for name in lost_inputs(p, copies.reports, lambda n: (pathlib.Path(copies.reports) / n).exists()):
target = pathlib.Path(copies.reports) / name
if name == "ls-files.txt" and clone:
out = FX.git("ls-files", cwd=clone)
if not isinstance(out, Failure): target.write_text(out)
elif name.startswith("places.") and clone:
target.write_text("(The places list for this round was not archived; no places are handed to you.)\n")
if lost := lost_inputs(p, copies.reports, lambda n: (pathlib.Path(copies.reports) / n).exists()):
return Failure(f"lost inputs {lost}")
return p
def run(rounds, codes, agent_args, prompt_of, out, cutoff=None):
"""Clone at the first round's code, prepare prompts, run claude, keep reply and reports written."""
if out.exists():
return json.loads(out.read_text())
(HOME / REPORTS).mkdir(parents=True, exist_ok=True)
(HERE / "clones").mkdir(exist_ok=True)
first = rounds[0]
with tempfile.TemporaryDirectory(prefix="rm-", dir=HERE / "clones", ignore_cleanup_errors=True) as clone, \
tempfile.TemporaryDirectory(prefix="review-replay-rm-", dir=HOME / REPORTS, ignore_cleanup_errors=True) as reports:
copies = Copies(clone, reports)
if isinstance(c := clone_reviewed(first, codes[first], clone, FX), Failure): return {"error": c.reason}
prompts = []
for r in rounds:
p = prepared(r, codes[r], copies, clone, cutoff)
if isinstance(p, Failure): return {"error": p.reason}
prompts.append(p)
before = set(os.listdir(reports))
prompt = prompt_of(prompts, reports)
reply = None
for _ in range(3):
reply = claude_reply(FX.claude, [*agent_args, *RUNG.flags()], clone, prompt, "claude")
if not isinstance(reply, Failure): break
if isinstance(reply, Limited): print("LIMITED", reply.said); sys.exit(2)
if isinstance(reply, Failure): result = {"error": reply.reason}
else:
written = {n: (pathlib.Path(reports)/n).read_text(errors="replace") for n in set(os.listdir(reports)) - before if (pathlib.Path(reports)/n).is_file()}
result = {"reply": reply, "written": written, "prompt": prompt}
out.write_text(json.dumps(result))
return result
def load_cases(cand):
return json.loads((HERE / f"cases-{cand}.json").read_text())
def main():
cand, variant, rep = sys.argv[1], sys.argv[2], sys.argv[3]
runs = pickle.load(open(HERE / "runs.pkl", "rb"))
by_meta = {r["meta"]: r for r in runs}
cases = load_cases(cand)
if sys.argv[4:]: cases = [c for c in cases if c["id"] in sys.argv[4:]]
outdir = HERE / "results" / cand / f"{variant}-{rep}"; outdir.mkdir(parents=True, exist_ok=True)
jobs = []
for case in cases:
rounds = [as_round(by_meta[m]) for m in case["metas"]]
codes = {}
for r in rounds:
c = rm.code_of(r, FX)
if isinstance(c, Failure): print(case["id"], "no code:", c.reason); break
codes[r] = c
else:
# one clone for all: the code of the first round
for r in rounds: codes[r] = codes[rounds[0]]
if variant == "sep":
for r in rounds:
jobs.append(([r], codes, ["--agent", r.reviewer], lambda ps, rep_: ps[0], outdir / f"{case['id']}--{PASS[r.reviewer]}.json", rounds[0]))
else:
name, path = MERGED[cand]
def prompt_of(ps, reports, rounds=rounds, name=name):
briefs = "\n\n".join(f"=== The worker's brief to the {PASS[r.reviewer]} pass ===\n\n{p}" for r, p in zip(rounds, ps))
return (f"You review this change once, in place of the {', '.join(PASS[r.reviewer] for r in rounds)} passes. The worker's brief to each of those passes follows; carry out all of them. "
f"Each brief names its own report file: write instead one report, holding every brief's findings under its own heading, to {reports}/{name}-round-1.md, and reply with a short summary of the findings and their severities, one line per finding under each pass's heading.\n\n{briefs}")
jobs.append((rounds, codes, ["--agents", str(HERE / path), "--agent", name], prompt_of, outdir / f"{case['id']}--merged.json", rounds[0]))
with concurrent.futures.ThreadPoolExecutor(int(os.environ.get("AT_ONCE", "4"))) as pool:
futs = {pool.submit(run, *j): j[4].name for j in jobs}
for f in concurrent.futures.as_completed(futs):
res = f.result()
cost = res.get("reply", {}).get("total_cost_usd") if "reply" in res else None
print(futs[f], "error: " + res["error"] if "error" in res else f"${cost:.2f}", flush=True)
if __name__ == "__main__":
main()