Pass the judge a Brief, decide read_seats from its inputs and return a failed bench run as a line
This commit is contained in:
@@ -113,9 +113,17 @@ def run(name):
|
||||
return f"{name}: FAILED after {ATTEMPTS} attempts"
|
||||
|
||||
|
||||
def attempted(name):
|
||||
"""run's result line, or a line naming the failure, so one failed run leaves the others' results to print."""
|
||||
try:
|
||||
return run(name)
|
||||
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
|
||||
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
|
||||
|
||||
|
||||
def main():
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
for result in pool.map(run, sys.argv[1:]):
|
||||
for result in pool.map(attempted, sys.argv[1:]):
|
||||
print(result, flush=True)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
|
||||
import bench
|
||||
|
||||
|
||||
class Attempted(unittest.TestCase):
|
||||
def test_returns_the_run_result_of_a_run_that_finished(self):
|
||||
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
|
||||
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
|
||||
|
||||
def test_returns_a_failed_run_as_a_line_naming_it_so_the_later_runs_still_report(self):
|
||||
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
|
||||
for failure in failures:
|
||||
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):
|
||||
results = [bench.attempted("a-v-1"), bench.attempted("b-v-1")]
|
||||
self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0])
|
||||
self.assertIn(type(failure).__name__, results[0])
|
||||
self.assertEqual(results[1], "b-v-1: done")
|
||||
|
||||
def test_a_run_of_an_unknown_case_fails_as_a_line(self):
|
||||
self.assertRegex(bench.attempted("nope-v-1"), r"^nope-v-1: FAILED: KeyError")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user