Files

34 lines
1.9 KiB
Python

"""Run analyze.py's matching and new-finding count, and also sum what those calls cost. Usage: [BASELINE=<candidate>] analyze_metered.py <candidate> <variant>-<run>... [case ids...]"""
import pathlib, runpy, sys, threading
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import analyze, review_replay.replay as rm
from claude_code.reply import claude_reply
SPENT = [0.0]; LOCK = threading.Lock()
def metered(*a, **k):
r = claude_reply(*a, **k)
if isinstance(r, dict):
with LOCK: SPENT[0] += r.get("total_cost_usd", 0)
return r
rm.claude_reply = analyze.claude_reply = metered
import json, os
# BASELINE=wave3: the reports come from candidate 1, so this reuses its new-finding counts for this candidate's passes and makes no call to count them again.
BASE = os.environ.get("BASELINE")
if BASE: analyze.new_findings = lambda arch, text: []
def reused_new(res):
passes = {a["pass"] for a in res["archived"]}
old = json.loads((analyze.HERE / "results" / BASE / f"analysis-{res['vr']}-{res['case']}.json").read_text())["new"] or []
res["new"] = [f for f in old if f["pass"] in passes]
(analyze.HERE / "results" / sys.argv[1] / f"analysis-{res['vr']}-{res['case']}.json").write_text(json.dumps(res, ensure_ascii=False, indent=1))
return res
sys.argv = ["analyze.py", *sys.argv[1:]]
import concurrent.futures, re
cand = sys.argv[1]; cases = analyze.d.load_cases(cand)
only = [a for a in sys.argv[2:] if not re.match(r"(sep|merged)-\d+$", a)]
vrs = [a for a in sys.argv[2:] if a not in only]
if only: cases = [c for c in cases if c["id"] in only]
with concurrent.futures.ThreadPoolExecutor(6) as pool:
for r in pool.map(lambda j: analyze.one(*j), [(cand, vr, c) for vr in vrs for c in cases]):
if r and BASE: r = reused_new(r)
if r: print(r["case"], r["vr"], "found", len(r["found"] or []), "/", len(r["archived"]), "new", len(r["new"] or []), flush=True)
print(f"matching spend ${SPENT[0]:.2f}")