Answer stability round 1: 0 🔴, 4 🟡 (0 misread); test main's continuing past a failed run, name the denied commits in decided's tests, check the replay puts premises before records
This commit is contained in:
@@ -114,7 +114,7 @@ def run(name):
|
||||
|
||||
|
||||
def attempted(name):
|
||||
"""run's result line, or a line naming the failure, so one failed run leaves the others' results to print."""
|
||||
"""run's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
|
||||
try:
|
||||
return run(name)
|
||||
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
@@ -7,12 +10,24 @@ with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
|
||||
import bench
|
||||
|
||||
|
||||
class Main(unittest.TestCase):
|
||||
def test_prints_every_runs_line_though_an_earlier_run_failed(self):
|
||||
def run(name):
|
||||
if name == "a-v-1":
|
||||
raise KeyError("a")
|
||||
return f"{name}: done"
|
||||
out = io.StringIO()
|
||||
with unittest.mock.patch.object(bench, "run", side_effect=run), unittest.mock.patch.object(sys, "argv", ["bench.py", "a-v-1", "b-v-1"]), contextlib.redirect_stdout(out):
|
||||
bench.main()
|
||||
self.assertEqual(out.getvalue().splitlines(), ["a-v-1: FAILED: KeyError: 'a'", "b-v-1: done"])
|
||||
|
||||
|
||||
class Attempted(unittest.TestCase):
|
||||
def test_returns_the_run_result_of_a_run_that_finished(self):
|
||||
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
|
||||
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
|
||||
|
||||
def test_returns_a_failed_run_as_a_line_naming_it_so_the_later_runs_still_report(self):
|
||||
def test_returns_a_failed_run_as_a_line_naming_the_run_and_its_error_so_later_runs_still_report(self):
|
||||
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
|
||||
for failure in failures:
|
||||
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):
|
||||
|
||||
Reference in New Issue
Block a user