35 lines
2.1 KiB
Python
35 lines
2.1 KiB
Python
import collections, json, pathlib, sys
|
|
cand = sys.argv[1]; vrs = sys.argv[2:]
|
|
R = pathlib.Path(__file__).parent / "results" / cand
|
|
A = {}
|
|
for vr in vrs:
|
|
for p in R.glob(f"analysis-{vr}-*.json"):
|
|
a = json.loads(p.read_text()); A[(vr, a["case"])] = a
|
|
cases = sorted({c for _, c in A})
|
|
cases = [c for c in cases if all((vr, c) in A for vr in vrs)]
|
|
print("cases:", len(cases), cases)
|
|
arch = {c: A[(vrs[0], c)]["archived"] for c in cases}
|
|
tot = sum(len(v) for v in arch.values())
|
|
bypass = collections.Counter((f["pass"], f["sev"]) for v in arch.values() for f in v)
|
|
print("archived:", tot, dict(bypass))
|
|
found = {vr: {c: set(A[(vr, c)]["found"] or []) for c in cases} for vr in vrs}
|
|
for vr in vrs:
|
|
n = sum(len(found[vr][c]) for c in cases)
|
|
per = collections.Counter(arch[c][i]["pass"] for c in cases for i in found[vr][c])
|
|
sev = collections.Counter(arch[c][i]["sev"] for c in cases for i in found[vr][c])
|
|
cost = sum(A[(vr, c)]["cost"] for c in cases); tok = sum(A[(vr, c)]["tokens"] for c in cases); out = sum(A[(vr, c)]["output_tokens"] for c in cases)
|
|
new = [f for c in cases for f in (A[(vr, c)]["new"] or [])]
|
|
print(f"{vr}: recovered {n}/{tot} by pass {dict(per)} by sev {dict(sev)}; new 🔴/🟡 {len(new)} ({dict(collections.Counter(f['severity'] for f in new))}); cost ${cost:.2f}; tokens {tok/1e6:.2f}M (output {out/1e3:.0f}k)")
|
|
for i, x in enumerate(vrs):
|
|
for y in vrs[i+1:]:
|
|
for a, b in ((x, y), (y, x)):
|
|
miss = [(c, j) for c in cases for j in found[a][c] - found[b][c]]
|
|
print(f"found by {a}, missed by {b}: {len(miss)} " + str(dict(collections.Counter((arch[c][j]['pass'], arch[c][j]['sev']) for c, j in miss))))
|
|
for c, j in miss: print(" ", c, arch[c][j]["pass"], arch[c][j]["text"].splitlines()[0][:150])
|
|
both = sum(len(found[x][c] & found[y][c]) for c in cases)
|
|
print(f"found by both {x} and {y}: {both}")
|
|
for vr in vrs:
|
|
print(f"--- new findings in {vr}")
|
|
for c in cases:
|
|
for f in A[(vr, c)]["new"] or []: print(" ", c, f["severity"], f["pass"], f["where"][:60], "-", f["summary"][:140])
|