Add run-tests, which runs the suite as README → Test says and saves the whole log

This commit is contained in:
2026-11-12 08:39:35 +01:00
parent b19ecc459e
commit e4e3d8cff0
7 changed files with 177 additions and 2 deletions
+3
View File
@@ -96,10 +96,13 @@
!bin/rule_hook/
!bin/rule_hook/*.py
!bin/run-heavy
!bin/run-tests
!bin/run_heavy/
!bin/run_heavy/*.py
!bin/run_heavy/path_entry/
!bin/run_heavy/path_entry/docker
!bin/run_tests/
!bin/run_tests/*.py
!bin/rung_replay/
!bin/rung_replay/*.py
!bin/score-outcomes
+1 -1
View File
@@ -145,7 +145,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t
## Test
Test on the host with `uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
CI runs `ci.sh` on every pull request, and `main` merges nothing until it passes. The script runs the same tests on the committed HEAD in a pinned Python container; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`.
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env python3
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from run_tests import main
sys.exit(main.main(sys.argv[1:]))
View File
+72
View File
@@ -0,0 +1,72 @@
"""Run this repo's test suite as README.md → Test says, and print only what a reader needs from it.
usage: run-tests [<pytest args>...]
It runs pytest, pinned with pytest-xdist in four workers through uvx, on the arguments given,
adding the repo's bin/ folder where none of them names a path that exists. It saves the whole
output to a new file and prints pytest's short test summary, a line per failing test with its
error, then pytest's summary line, then `Full log: <path>`. Where pytest printed no summary line,
it prints the log's last lines in their place.
Exit 0 where every test passed, 1 otherwise, or where pytest cannot start, saying why.
"""
import os
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
BIN = pathlib.Path(__file__).resolve().parents[1]
# README.md → Test names these pins, and ci.sh runs this command, so they live here alone.
PYTEST = ["uvx", "--python", shutil.which("python3") or sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4"]
SUMMARY_HEADER = re.compile(r"^=+ short test summary info =+$")
# pytest's last line, such as `==== 2 failed, 1 passed in 0.31s ====`, or without the rules under -q.
SUMMARY_LINE = re.compile(r"^(=+ )?(no tests ran|\d+ \w+).* in [\d.]+s\b.*$")
LOG_TAIL = 20
def pytest_args(argv: list[str]) -> list[str]:
"""argv, with bin/ added where no argument names a path, so `run-tests -k name` still runs the suite."""
if any(os.path.exists(arg.split("::")[0]) for arg in argv):
return argv
return [*argv, str(BIN)]
def report(log: str) -> list[str]:
"""The short test summary and the summary line from pytest's output, or [] where it printed no summary line."""
lines = log.splitlines()
ends = [index for index, line in enumerate(lines) if SUMMARY_LINE.match(line)]
if not ends:
return []
end = ends[-1]
starts = [index for index, line in enumerate(lines[:end]) if SUMMARY_HEADER.match(line)]
return lines[starts[-1] + 1:end + 1] if starts else [lines[end]]
def run(argv: list[str], pytest: list[str] = PYTEST, cwd: str | None = None) -> int:
descriptor, log_path = tempfile.mkstemp(prefix="run-tests-", suffix=".log")
with os.fdopen(descriptor, "w") as log:
try:
code = subprocess.run([*pytest, *pytest_args(argv)], stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, cwd=cwd).returncode
except OSError as error:
os.unlink(log_path)
print(f"run-tests: cannot start {pytest[0]}: {error.strerror}", file=sys.stderr)
return 1
output = pathlib.Path(log_path).read_text(errors="replace")
lines = report(output)
if lines:
print("\n".join(lines))
else:
print("run-tests: pytest printed no summary line; the log ends:")
print("\n".join(output.splitlines()[-LOG_TAIL:]))
print(f"Full log: {log_path}")
return 0 if code == 0 else 1
def main(argv: list[str]) -> int:
if argv[:1] in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
return run(argv)
+90
View File
@@ -0,0 +1,90 @@
import contextlib
import importlib.util
import io
import os
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from run_tests import main
BIN = pathlib.Path(__file__).resolve().parents[1]
# The suite runs under pytest, and `python3 -m unittest` may run it where pytest is not installed.
HAS_PYTEST = importlib.util.find_spec("pytest") is not None
PYTEST_HERE = [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "no:xdist"]
class PytestArgs(unittest.TestCase):
def test_runs_bin_when_no_argument_names_a_path(self):
self.assertEqual(main.pytest_args([]), [str(BIN)])
self.assertEqual(main.pytest_args(["-k", "no_such_path_here"]), ["-k", "no_such_path_here", str(BIN)])
def test_passes_a_given_path_on_alone(self):
with tempfile.TemporaryDirectory() as folder:
test_file = pathlib.Path(folder, "test_x.py")
test_file.write_text("")
self.assertEqual(main.pytest_args([str(test_file)]), [str(test_file)])
self.assertEqual(main.pytest_args(["-x", f"{test_file}::T::test_a"]), ["-x", f"{test_file}::T::test_a"])
def test_the_pinned_command_is_readme_tests_incantation(self):
self.assertEqual(main.PYTEST[1:], ["--python", main.PYTEST[2], "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4"])
self.assertEqual(main.PYTEST[0], "uvx")
@unittest.skipUnless(HAS_PYTEST, "needs pytest importable")
class Run(unittest.TestCase):
def run_in(self, folder: str, pytest: list[str], *args: str) -> tuple[int, str]:
out = io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
code = main.run(list(args), pytest, folder)
return code, out.getvalue()
def log_named(self, output: str) -> pathlib.Path:
last = output.rstrip("\n").splitlines()[-1]
self.assertTrue(last.startswith("Full log: "), output)
path = pathlib.Path(last.removeprefix("Full log: "))
self.addCleanup(path.unlink, missing_ok=True)
return path
def test_a_passing_run_prints_the_summary_line_and_the_log_path(self):
with tempfile.TemporaryDirectory() as folder:
pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n")
code, output = self.run_in(folder, PYTEST_HERE, "-s", folder)
self.assertEqual(code, 0)
lines = output.rstrip("\n").splitlines()
self.assertEqual(len(lines), 2, output)
self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$")
self.assertIn("noise from a passing test", self.log_named(output).read_text())
def test_a_failing_run_prints_each_failing_test_with_its_error(self):
with tempfile.TemporaryDirectory() as folder:
pathlib.Path(folder, "test_bad.py").write_text("def test_ok():\n\tpass\n\ndef test_bad():\n\tassert 1 == 2, 'one is not two'\n\ndef test_worse():\n\traise KeyError('missing')\n")
code, output = self.run_in(folder, PYTEST_HERE, folder)
self.assertEqual(code, 1)
self.assertRegex(output, r"FAILED .*test_bad\.py::test_bad - AssertionError: one is not two")
self.assertRegex(output, r"FAILED .*test_bad\.py::test_worse - KeyError: 'missing'")
self.assertRegex(output, r"=+ 2 failed, 1 passed in .* =+\n")
self.assertNotIn("def test_ok", output)
self.assertIn("def test_bad", self.log_named(output).read_text())
def test_a_run_that_cannot_start_says_so_and_prints_the_logs_last_lines(self):
with tempfile.TemporaryDirectory() as folder:
code, output = self.run_in(folder, [os.path.join(folder, "no-such-uvx")])
self.assertEqual(code, 1)
self.assertIn("run-tests: cannot start", output)
self.assertNotIn("Full log: ", output)
def test_a_run_that_prints_no_summary_line_shows_the_logs_last_lines(self):
with tempfile.TemporaryDirectory() as folder:
code, output = self.run_in(folder, PYTEST_HERE, "--no-such-option", folder)
self.assertEqual(code, 1)
self.assertIn("run-tests: pytest printed no summary line; the log ends:", output)
self.assertIn("--no-such-option", output)
self.log_named(output)
if __name__ == "__main__":
unittest.main()
+2 -1
View File
@@ -9,5 +9,6 @@ git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --use
git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo
cd /tmp/repo
python3 -m pip install --quiet --disable-pip-version-check --user uv==0.12.23
PATH="$HOME/.local/bin:$PATH" uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin
# On a failure, the CI log shows the whole pytest output, which run-tests saved to its log file.
PATH="$HOME/.local/bin:$PATH" bin/run-tests || { cat /tmp/run-tests-*.log; exit 1; }
'