Pass the judge a Brief, decide read_seats from its inputs and return a failed bench run as a line

This commit is contained in:
2026-11-08 22:51:27 +01:00
parent 2a0eabe8d9
commit 268200de23
12 changed files with 115 additions and 47 deletions
+9 -1
View File
@@ -113,9 +113,17 @@ def run(name):
return f"{name}: FAILED after {ATTEMPTS} attempts"
def attempted(name):
"""run's result line, or a line naming the failure, so one failed run leaves the others' results to print."""
try:
return run(name)
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(run, sys.argv[1:]):
for result in pool.map(attempted, sys.argv[1:]):
print(result, flush=True)
+29
View File
@@ -0,0 +1,29 @@
import os
import subprocess
import unittest
import unittest.mock
with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
import bench
class Attempted(unittest.TestCase):
def test_returns_the_run_result_of_a_run_that_finished(self):
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
def test_returns_a_failed_run_as_a_line_naming_it_so_the_later_runs_still_report(self):
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
for failure in failures:
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):
results = [bench.attempted("a-v-1"), bench.attempted("b-v-1")]
self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0])
self.assertIn(type(failure).__name__, results[0])
self.assertEqual(results[1], "b-v-1: done")
def test_a_run_of_an_unknown_case_fails_as_a_line(self):
self.assertRegex(bench.attempted("nope-v-1"), r"^nope-v-1: FAILED: KeyError")
if __name__ == "__main__":
unittest.main()