Answer stability round 1: 0 🔴, 4 🟡 (0 misread); test main's continuing past a failed run, name the denied commits in decided's tests, check the replay puts premises before records

This commit is contained in:
2026-11-08 22:58:52 +01:00
parent a3cd9796a4
commit c74ec6fc0a
6 changed files with 31 additions and 16 deletions
+1 -1
View File
@@ -114,7 +114,7 @@ def run(name):
def attempted(name):
"""run's result line, or a line naming the failure, so one failed run leaves the others' results to print."""
"""run's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
try:
return run(name)
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
+16 -1
View File
@@ -1,5 +1,8 @@
import contextlib
import io
import os
import subprocess
import sys
import unittest
import unittest.mock
@@ -7,12 +10,24 @@ with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
import bench
class Main(unittest.TestCase):
def test_prints_every_runs_line_though_an_earlier_run_failed(self):
def run(name):
if name == "a-v-1":
raise KeyError("a")
return f"{name}: done"
out = io.StringIO()
with unittest.mock.patch.object(bench, "run", side_effect=run), unittest.mock.patch.object(sys, "argv", ["bench.py", "a-v-1", "b-v-1"]), contextlib.redirect_stdout(out):
bench.main()
self.assertEqual(out.getvalue().splitlines(), ["a-v-1: FAILED: KeyError: 'a'", "b-v-1: done"])
class Attempted(unittest.TestCase):
def test_returns_the_run_result_of_a_run_that_finished(self):
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
def test_returns_a_failed_run_as_a_line_naming_it_so_the_later_runs_still_report(self):
def test_returns_a_failed_run_as_a_line_naming_the_run_and_its_error_so_later_runs_still_report(self):
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
for failure in failures:
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):