Answer stability round 1: 0 🔴, 3 🟡 (0 misread); drop the trials' patch of authored_by, test names that sort against times

This commit is contained in:
2026-11-10 10:43:43 +01:00
parent 62d1f5edd4
commit ba903c58d2
6 changed files with 9 additions and 20 deletions
+1 -1
View File
@@ -2,7 +2,7 @@
## Unreleased
- `review-replay` now takes each replay's reviewed commit and base by the newest author time at or before the round's time, where it took the first such commit in log order, which a commit authored early but merged late, such as a dependency bump, could win.
- `review-replay` now replays each round at the commit with the newest author time at or before the round's time, and takes the clone's base the same way by the time the chunk began. It took the first such commit in log order, so a commit authored early but merged late, such as a dependency bump, could win over the commits the round reviewed.
- The shared goal every project pursues, code health, now asks that code stay easy to read, change safely and debug, tomorrow and after seven years of development.
- `rank_todo.py` now lets an item that adds a test, a gate leg or a probe name the maintain goal it guards, by how much more confident the project becomes that the goal holds, 1 to 10. Each point is worth 0.5.
- `chunk-preflight` now quotes the detail of each item that may join the chunk, so a worker no longer opens `todo.md` to read it.
+1 -1
View File
@@ -93,7 +93,7 @@ def lost_inputs(prompt: str, reports: str, held: typing.Callable[[str], bool]) -
def authored_by(log: list[tuple[str, float]], seconds: float) -> str | None:
"""The commit in log with the newest author time at or before seconds; the first in log order breaks a tie. A rebase keeps author times, so a merged branch's commits still say what it held. Log order alone would not do: a commit authored early but merged late, such as a dependency bump, sits high in it."""
"""The commit in log with the newest author time at or before seconds; the first in log order breaks a tie. The log lists commits newest first by when they were merged, so taking the first one authored by seconds would pick a commit authored early but merged late, such as a dependency bump, over the commits a round reviewed. A rebase keeps author times, so a merged branch's commits keep the times their author wrote them."""
return max(((commit, authored) for commit, authored in log if authored <= seconds), key=lambda listed: listed[1], default=(None, 0))[0]
+4 -4
View File
@@ -100,12 +100,12 @@ class AuthoredBy(unittest.TestCase):
self.assertIsNone(authored_by(log, 5))
def test_takes_the_newest_author_time_over_a_commit_authored_early_but_merged_late(self):
log = [("bump", 5), ("c3", 30), ("c2", 20)]
self.assertEqual(authored_by(log, 25), "c2")
self.assertEqual(authored_by(log, 10), "bump")
log = [("z-bump", 5), ("a-new", 30), ("m-mid", 20)]
self.assertEqual(authored_by(log, 25), "m-mid")
self.assertEqual(authored_by(log, 10), "z-bump")
def test_breaks_a_tie_by_log_order(self):
self.assertEqual(authored_by([("merged", 20), ("earlier", 20)], 25), "merged")
self.assertEqual(authored_by([("a-merged", 20), ("z-earlier", 20)], 25), "a-merged")
class Sample(unittest.TestCase):
+1 -2
View File
@@ -49,8 +49,7 @@ Candidate 4 has that. Candidate 1's merged reviewer ran once, but its prose loss
### The reviewed-commit fix
Every trial took the commit with the newest author time, log order breaking ties, which `review-replay` took too only from 2026-11-10: before, it took the first commit in log order authored by then (`driver.py` → `code_of`).
Without it, two plainpages rounds would have replayed at a commit older than their own base.
Every trial took the commit with the newest author time, log order breaking ties, as `review-replay` does from 2026-11-10. Before that it took the first commit in log order authored by then, and a trial's driver patched that in. Without the patch, two plainpages rounds would have replayed at a commit older than their own base.
## Samples
+1 -11
View File
@@ -17,16 +17,6 @@ FX = system_effects()
PASS = {"stability-reviewer": "stability", "product-owner-reviewer": "product-owner", "prose-reviewer": "prose", "systems-architect": "architecture", "principles-auditor": "principles", "override-reviewer": "overrides"}
MERGED = {"wave3": ("wave3-reviewer", "agents-wave3.json"), "release": ("release-reviewer", "agents-release.json"), "pair": ("pair-reviewer", "agents-pair.json")}
def code_of(rnd, fx):
"""review_replay.main.code_of, but picking the commit with the newest author time by then rather than the first in log order: a commit authored early and merged late (e.g. a dependency update) sits high in log order and would otherwise be taken as the reviewed commit."""
import review_replay.rounds as rr
orig = rm.authored_by
rm.authored_by = lambda log, secs: max(((a, -i, c) for i, (c, a) in enumerate(log) if a <= secs), default=(None, None, None))[2]
try:
return rm.code_of(rnd, fx)
finally:
rm.authored_by = orig
def as_round(r):
return Round(r["reviewer"], r["checkout"], r["branch"], r["at"], r["began"], r["prompt"], tuple(r["findings"]))
@@ -93,7 +83,7 @@ def main():
rounds = [as_round(by_meta[m]) for m in case["metas"]]
codes = {}
for r in rounds:
c = code_of(r, FX)
c = rm.code_of(r, FX)
if isinstance(c, Failure): print(case["id"], "no code:", c.reason); break
codes[r] = c
else:
+1 -1
View File
@@ -5,5 +5,5 @@ runs = {r["meta"]: r for r in pickle.load(open(d.HERE / "runs.pkl", "rb"))}
for cand in sys.argv[1:]:
for c in d.load_cases(cand):
r = d.as_round(runs[c["metas"][0]])
code = d.code_of(r, d.FX)
code = d.rm.code_of(r, d.FX)
print(cand, c["id"], getattr(code, "reviewed", code), getattr(code, "base", ""))