55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
import contextlib
|
|
import functools
|
|
import io
|
|
import pathlib
|
|
import subprocess
|
|
import unittest
|
|
|
|
import bench
|
|
|
|
|
|
def answering(*answers):
|
|
"""A run that gives the next of answers on each call, raising the one that is a failure."""
|
|
left = list(answers)
|
|
|
|
def run(name):
|
|
answer = left.pop(0)
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
return answer
|
|
return run
|
|
|
|
|
|
class Main(unittest.TestCase):
|
|
def test_prints_every_runs_line_though_an_earlier_run_failed(self):
|
|
def run(name):
|
|
if name == "a-v-1":
|
|
raise KeyError("a")
|
|
return f"{name}: done"
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
bench.main(["a-v-1", "b-v-1"], run)
|
|
self.assertEqual(out.getvalue().splitlines(), ["a-v-1: FAILED: KeyError: 'a'", "b-v-1: done"])
|
|
|
|
|
|
class Attempted(unittest.TestCase):
|
|
def test_returns_the_run_result_of_a_run_that_finished(self):
|
|
self.assertEqual(bench.attempted("a-v-1", answering("a-v-1: done")), "a-v-1: done")
|
|
|
|
def test_returns_a_failed_run_as_a_line_naming_the_run_and_its_error_so_later_runs_still_report(self):
|
|
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
|
|
for failure in failures:
|
|
with self.subTest(failure=type(failure).__name__):
|
|
run = answering(failure, "b-v-1: done")
|
|
results = [bench.attempted("a-v-1", run), bench.attempted("b-v-1", run)]
|
|
self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0])
|
|
self.assertIn(type(failure).__name__, results[0])
|
|
self.assertEqual(results[1], "b-v-1: done")
|
|
|
|
def test_a_run_of_an_unknown_case_fails_as_a_line(self):
|
|
self.assertRegex(bench.attempted("nope-v-1", functools.partial(bench.run, sources=pathlib.Path("/nonexistent"))), r"^nope-v-1: FAILED: KeyError")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|