import contextlib import functools import io import pathlib import subprocess import unittest import bench def answering(*answers): """A run that gives the next of answers on each call, raising the one that is a failure.""" left = list(answers) def run(name): answer = left.pop(0) if isinstance(answer, Exception): raise answer return answer return run class Main(unittest.TestCase): def test_prints_every_runs_line_though_an_earlier_run_failed(self): def run(name): if name == "a-v-1": raise KeyError("a") return f"{name}: done" out = io.StringIO() with contextlib.redirect_stdout(out): bench.main(["a-v-1", "b-v-1"], run) self.assertEqual(out.getvalue().splitlines(), ["a-v-1: FAILED: KeyError: 'a'", "b-v-1: done"]) class Attempted(unittest.TestCase): def test_returns_the_run_result_of_a_run_that_finished(self): self.assertEqual(bench.attempted("a-v-1", answering("a-v-1: done")), "a-v-1: done") def test_returns_a_failed_run_as_a_line_naming_the_run_and_its_error_so_later_runs_still_report(self): failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")] for failure in failures: with self.subTest(failure=type(failure).__name__): run = answering(failure, "b-v-1: done") results = [bench.attempted("a-v-1", run), bench.attempted("b-v-1", run)] self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0]) self.assertIn(type(failure).__name__, results[0]) self.assertEqual(results[1], "b-v-1: done") def test_a_run_of_an_unknown_case_fails_as_a_line(self): self.assertRegex(bench.attempted("nope-v-1", functools.partial(bench.run, sources=pathlib.Path("/nonexistent"))), r"^nope-v-1: FAILED: KeyError") if __name__ == "__main__": unittest.main()