83 lines
4.6 KiB
Python
83 lines
4.6 KiB
Python
"""Match each variant's replay reports to the archived findings, and list new findings. Usage: analyze.py <candidate> <variant>-<run>..."""
|
|
import concurrent.futures, json, pathlib, pickle, re, sys
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin"))
|
|
from claude_code.reply import claude_reply
|
|
import driver as d
|
|
from failure import Failure
|
|
from claude_code.entries import structured_output
|
|
|
|
HERE = d.HERE
|
|
runs = {r["meta"]: r for r in pickle.load(open(HERE / "runs.pkl", "rb"))}
|
|
|
|
def archived(case):
|
|
out = []
|
|
for m in case["metas"]:
|
|
r = runs[m]
|
|
for f in r["findings"]:
|
|
sev = "🔴" if "🔴" in f.split("\n")[0][:40] else "🟡"
|
|
out.append({"pass": d.PASS[r["reviewer"]], "sev": sev, "text": f})
|
|
return out
|
|
|
|
def report_text(cand, vr, case):
|
|
texts = []; cost = 0; toks = 0; out_t = 0; ok = True
|
|
for p in sorted((HERE / "results" / cand / vr).glob(f"{case['id']}--*.json")):
|
|
res = json.loads(p.read_text())
|
|
if "error" in res: ok = False; texts.append(f"[{p.name} failed: {res['error']}]"); continue
|
|
rep = res["reply"]
|
|
cost += rep.get("total_cost_usd", 0)
|
|
for mu in (rep.get("modelUsage") or {}).values():
|
|
toks += mu.get("inputTokens", 0) + mu.get("outputTokens", 0) + mu.get("cacheReadInputTokens", 0) + mu.get("cacheCreationInputTokens", 0)
|
|
out_t += mu.get("outputTokens", 0)
|
|
body = "\n\n".join(res["written"].values()) or rep.get("result", "")
|
|
texts.append(f"### Report from {p.stem.split('--')[1]}\n\n{body}")
|
|
return "\n\n".join(texts), cost, toks, out_t, ok
|
|
|
|
def matched(arch, text):
|
|
findings = tuple(f"[{a['pass']}] {a['text']}" for a in arch)
|
|
for _ in range(3):
|
|
m = d.match(findings, text, str(HERE), d.RUNG, d.FX)
|
|
if not isinstance(m, Failure): return sorted(m)
|
|
print("match failed", m.reason[:200], file=sys.stderr)
|
|
return None
|
|
|
|
NEW_SCHEMA = {"type": "object", "properties": {"new": {"type": "array", "items": {"type": "object", "properties": {
|
|
"severity": {"type": "string", "enum": ["🔴", "🟡"]}, "pass": {"type": "string"}, "where": {"type": "string"}, "summary": {"type": "string"}},
|
|
"required": ["severity", "pass", "where", "summary"], "additionalProperties": False}}}, "required": ["new"], "additionalProperties": False}
|
|
|
|
def new_findings(arch, text):
|
|
numbered = "\n\n".join(f"{n}. [{a['pass']}] {a['text']}" for n, a in enumerate(arch, 1))
|
|
prompt = ("Below are numbered findings an earlier code review reported, then a later review report of the same change. "
|
|
"List every finding the later report raises as 🔴 or 🟡 that none of the numbered findings raises (not the same defect at the same place or from the same cause). "
|
|
"For each give its severity, the pass or report heading it sits under (such as stability, product-owner, prose, architecture, principles), its file:line or place, and a one-sentence summary. "
|
|
"A finding the report raises twice, under two headings, is listed once.\n\n"
|
|
f"The earlier findings:\n{numbered}\n\nThe later report:\n{text}")
|
|
args = ["--json-schema", json.dumps(NEW_SCHEMA), *d.RUNG.flags(), "--tools", "", "--strict-mcp-config"]
|
|
for _ in range(3):
|
|
rep = claude_reply(d.FX.claude, args, str(HERE), prompt, "claude")
|
|
if not isinstance(rep, Failure):
|
|
out = structured_output(rep)
|
|
if isinstance(out, dict) and isinstance(out.get("new"), list): return out["new"]
|
|
return None
|
|
|
|
def one(cand, vr, case):
|
|
out = HERE / "results" / cand / f"analysis-{vr}-{case['id']}.json"
|
|
if out.exists(): return json.loads(out.read_text())
|
|
arch = archived(case)
|
|
text, cost, toks, out_t, ok = report_text(cand, vr, case)
|
|
if not text: return None
|
|
res = {"case": case["id"], "vr": vr, "ok": ok, "cost": cost, "tokens": toks, "output_tokens": out_t, "archived": arch,
|
|
"found": matched(arch, text), "new": new_findings(arch, text)}
|
|
out.write_text(json.dumps(res, ensure_ascii=False, indent=1))
|
|
return res
|
|
|
|
if __name__ == "__main__":
|
|
cand = sys.argv[1]
|
|
cases = d.load_cases(cand)
|
|
only = [a for a in sys.argv[2:] if not re.match(r"(sep|merged)-\d+$", a)]
|
|
vrs = [a for a in sys.argv[2:] if a not in only]
|
|
if only: cases = [c for c in cases if c["id"] in only]
|
|
jobs = [(cand, vr, c) for vr in vrs for c in cases]
|
|
with concurrent.futures.ThreadPoolExecutor(6) as pool:
|
|
for r in pool.map(lambda j: one(*j), jobs):
|
|
if r: print(r["case"], r["vr"], "found", len(r["found"] or []), "/", len(r["archived"]), "new", len(r["new"] or []), f"${r['cost']:.2f}", flush=True)
|