Files

35 lines
2.1 KiB
Python

import collections, json, pathlib, sys
cand = sys.argv[1]; vrs = sys.argv[2:]
R = pathlib.Path(__file__).parent / "results" / cand
A = {}
for vr in vrs:
for p in R.glob(f"analysis-{vr}-*.json"):
a = json.loads(p.read_text()); A[(vr, a["case"])] = a
cases = sorted({c for _, c in A})
cases = [c for c in cases if all((vr, c) in A for vr in vrs)]
print("cases:", len(cases), cases)
arch = {c: A[(vrs[0], c)]["archived"] for c in cases}
tot = sum(len(v) for v in arch.values())
bypass = collections.Counter((f["pass"], f["sev"]) for v in arch.values() for f in v)
print("archived:", tot, dict(bypass))
found = {vr: {c: set(A[(vr, c)]["found"] or []) for c in cases} for vr in vrs}
for vr in vrs:
n = sum(len(found[vr][c]) for c in cases)
per = collections.Counter(arch[c][i]["pass"] for c in cases for i in found[vr][c])
sev = collections.Counter(arch[c][i]["sev"] for c in cases for i in found[vr][c])
cost = sum(A[(vr, c)]["cost"] for c in cases); tok = sum(A[(vr, c)]["tokens"] for c in cases); out = sum(A[(vr, c)]["output_tokens"] for c in cases)
new = [f for c in cases for f in (A[(vr, c)]["new"] or [])]
print(f"{vr}: recovered {n}/{tot} by pass {dict(per)} by sev {dict(sev)}; new 🔴/🟡 {len(new)} ({dict(collections.Counter(f['severity'] for f in new))}); cost ${cost:.2f}; tokens {tok/1e6:.2f}M (output {out/1e3:.0f}k)")
for i, x in enumerate(vrs):
for y in vrs[i+1:]:
for a, b in ((x, y), (y, x)):
miss = [(c, j) for c in cases for j in found[a][c] - found[b][c]]
print(f"found by {a}, missed by {b}: {len(miss)} " + str(dict(collections.Counter((arch[c][j]['pass'], arch[c][j]['sev']) for c, j in miss))))
for c, j in miss: print(" ", c, arch[c][j]["pass"], arch[c][j]["text"].splitlines()[0][:150])
both = sum(len(found[x][c] & found[y][c]) for c in cases)
print(f"found by both {x} and {y}: {both}")
for vr in vrs:
print(f"--- new findings in {vr}")
for c in cases:
for f in A[(vr, c)]["new"] or []: print(" ", c, f["severity"], f["pass"], f["where"][:60], "-", f["summary"][:140])