Add run-tests, which runs the suite as README → Test says and saves the whole log
This commit is contained in:
@@ -96,10 +96,13 @@
|
||||
!bin/rule_hook/
|
||||
!bin/rule_hook/*.py
|
||||
!bin/run-heavy
|
||||
!bin/run-tests
|
||||
!bin/run_heavy/
|
||||
!bin/run_heavy/*.py
|
||||
!bin/run_heavy/path_entry/
|
||||
!bin/run_heavy/path_entry/docker
|
||||
!bin/run_tests/
|
||||
!bin/run_tests/*.py
|
||||
!bin/rung_replay/
|
||||
!bin/rung_replay/*.py
|
||||
!bin/score-outcomes
|
||||
|
||||
@@ -145,7 +145,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t
|
||||
|
||||
## Test
|
||||
|
||||
Test on the host with `uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
|
||||
Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
|
||||
|
||||
CI runs `ci.sh` on every pull request, and `main` merges nothing until it passes. The script runs the same tests on the committed HEAD in a pinned Python container; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`.
|
||||
|
||||
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from run_tests import main
|
||||
|
||||
sys.exit(main.main(sys.argv[1:]))
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Run this repo's test suite as README.md → Test says, and print only what a reader needs from it.
|
||||
|
||||
usage: run-tests [<pytest args>...]
|
||||
|
||||
It runs pytest, pinned with pytest-xdist in four workers through uvx, on the arguments given,
|
||||
adding the repo's bin/ folder where none of them names a path that exists. It saves the whole
|
||||
output to a new file and prints pytest's short test summary, a line per failing test with its
|
||||
error, then pytest's summary line, then `Full log: <path>`. Where pytest printed no summary line,
|
||||
it prints the log's last lines in their place.
|
||||
|
||||
Exit 0 where every test passed, 1 otherwise, or where pytest cannot start, saying why.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
BIN = pathlib.Path(__file__).resolve().parents[1]
|
||||
# README.md → Test names these pins, and ci.sh runs this command, so they live here alone.
|
||||
PYTEST = ["uvx", "--python", shutil.which("python3") or sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4"]
|
||||
SUMMARY_HEADER = re.compile(r"^=+ short test summary info =+$")
|
||||
# pytest's last line, such as `==== 2 failed, 1 passed in 0.31s ====`, or without the rules under -q.
|
||||
SUMMARY_LINE = re.compile(r"^(=+ )?(no tests ran|\d+ \w+).* in [\d.]+s\b.*$")
|
||||
LOG_TAIL = 20
|
||||
|
||||
|
||||
def pytest_args(argv: list[str]) -> list[str]:
|
||||
"""argv, with bin/ added where no argument names a path, so `run-tests -k name` still runs the suite."""
|
||||
if any(os.path.exists(arg.split("::")[0]) for arg in argv):
|
||||
return argv
|
||||
return [*argv, str(BIN)]
|
||||
|
||||
|
||||
def report(log: str) -> list[str]:
|
||||
"""The short test summary and the summary line from pytest's output, or [] where it printed no summary line."""
|
||||
lines = log.splitlines()
|
||||
ends = [index for index, line in enumerate(lines) if SUMMARY_LINE.match(line)]
|
||||
if not ends:
|
||||
return []
|
||||
end = ends[-1]
|
||||
starts = [index for index, line in enumerate(lines[:end]) if SUMMARY_HEADER.match(line)]
|
||||
return lines[starts[-1] + 1:end + 1] if starts else [lines[end]]
|
||||
|
||||
|
||||
def run(argv: list[str], pytest: list[str] = PYTEST, cwd: str | None = None) -> int:
|
||||
descriptor, log_path = tempfile.mkstemp(prefix="run-tests-", suffix=".log")
|
||||
with os.fdopen(descriptor, "w") as log:
|
||||
try:
|
||||
code = subprocess.run([*pytest, *pytest_args(argv)], stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, cwd=cwd).returncode
|
||||
except OSError as error:
|
||||
os.unlink(log_path)
|
||||
print(f"run-tests: cannot start {pytest[0]}: {error.strerror}", file=sys.stderr)
|
||||
return 1
|
||||
output = pathlib.Path(log_path).read_text(errors="replace")
|
||||
lines = report(output)
|
||||
if lines:
|
||||
print("\n".join(lines))
|
||||
else:
|
||||
print("run-tests: pytest printed no summary line; the log ends:")
|
||||
print("\n".join(output.splitlines()[-LOG_TAIL:]))
|
||||
print(f"Full log: {log_path}")
|
||||
return 0 if code == 0 else 1
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if argv[:1] in (["-h"], ["--help"]):
|
||||
print(__doc__.strip())
|
||||
return 0
|
||||
return run(argv)
|
||||
@@ -0,0 +1,90 @@
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from run_tests import main
|
||||
|
||||
BIN = pathlib.Path(__file__).resolve().parents[1]
|
||||
# The suite runs under pytest, and `python3 -m unittest` may run it where pytest is not installed.
|
||||
HAS_PYTEST = importlib.util.find_spec("pytest") is not None
|
||||
PYTEST_HERE = [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "no:xdist"]
|
||||
|
||||
|
||||
class PytestArgs(unittest.TestCase):
|
||||
def test_runs_bin_when_no_argument_names_a_path(self):
|
||||
self.assertEqual(main.pytest_args([]), [str(BIN)])
|
||||
self.assertEqual(main.pytest_args(["-k", "no_such_path_here"]), ["-k", "no_such_path_here", str(BIN)])
|
||||
|
||||
def test_passes_a_given_path_on_alone(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
test_file = pathlib.Path(folder, "test_x.py")
|
||||
test_file.write_text("")
|
||||
self.assertEqual(main.pytest_args([str(test_file)]), [str(test_file)])
|
||||
self.assertEqual(main.pytest_args(["-x", f"{test_file}::T::test_a"]), ["-x", f"{test_file}::T::test_a"])
|
||||
|
||||
def test_the_pinned_command_is_readme_tests_incantation(self):
|
||||
self.assertEqual(main.PYTEST[1:], ["--python", main.PYTEST[2], "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4"])
|
||||
self.assertEqual(main.PYTEST[0], "uvx")
|
||||
|
||||
|
||||
@unittest.skipUnless(HAS_PYTEST, "needs pytest importable")
|
||||
class Run(unittest.TestCase):
|
||||
def run_in(self, folder: str, pytest: list[str], *args: str) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
||||
code = main.run(list(args), pytest, folder)
|
||||
return code, out.getvalue()
|
||||
|
||||
def log_named(self, output: str) -> pathlib.Path:
|
||||
last = output.rstrip("\n").splitlines()[-1]
|
||||
self.assertTrue(last.startswith("Full log: "), output)
|
||||
path = pathlib.Path(last.removeprefix("Full log: "))
|
||||
self.addCleanup(path.unlink, missing_ok=True)
|
||||
return path
|
||||
|
||||
def test_a_passing_run_prints_the_summary_line_and_the_log_path(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n")
|
||||
code, output = self.run_in(folder, PYTEST_HERE, "-s", folder)
|
||||
self.assertEqual(code, 0)
|
||||
lines = output.rstrip("\n").splitlines()
|
||||
self.assertEqual(len(lines), 2, output)
|
||||
self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$")
|
||||
self.assertIn("noise from a passing test", self.log_named(output).read_text())
|
||||
|
||||
def test_a_failing_run_prints_each_failing_test_with_its_error(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
pathlib.Path(folder, "test_bad.py").write_text("def test_ok():\n\tpass\n\ndef test_bad():\n\tassert 1 == 2, 'one is not two'\n\ndef test_worse():\n\traise KeyError('missing')\n")
|
||||
code, output = self.run_in(folder, PYTEST_HERE, folder)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertRegex(output, r"FAILED .*test_bad\.py::test_bad - AssertionError: one is not two")
|
||||
self.assertRegex(output, r"FAILED .*test_bad\.py::test_worse - KeyError: 'missing'")
|
||||
self.assertRegex(output, r"=+ 2 failed, 1 passed in .* =+\n")
|
||||
self.assertNotIn("def test_ok", output)
|
||||
self.assertIn("def test_bad", self.log_named(output).read_text())
|
||||
|
||||
def test_a_run_that_cannot_start_says_so_and_prints_the_logs_last_lines(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
code, output = self.run_in(folder, [os.path.join(folder, "no-such-uvx")])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("run-tests: cannot start", output)
|
||||
self.assertNotIn("Full log: ", output)
|
||||
|
||||
def test_a_run_that_prints_no_summary_line_shows_the_logs_last_lines(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
code, output = self.run_in(folder, PYTEST_HERE, "--no-such-option", folder)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("run-tests: pytest printed no summary line; the log ends:", output)
|
||||
self.assertIn("--no-such-option", output)
|
||||
self.log_named(output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -9,5 +9,6 @@ git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --use
|
||||
git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo
|
||||
cd /tmp/repo
|
||||
python3 -m pip install --quiet --disable-pip-version-check --user uv==0.12.23
|
||||
PATH="$HOME/.local/bin:$PATH" uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin
|
||||
# On a failure, the CI log shows the whole pytest output, which run-tests saved to its log file.
|
||||
PATH="$HOME/.local/bin:$PATH" bin/run-tests || { cat /tmp/run-tests-*.log; exit 1; }
|
||||
'
|
||||
|
||||
Reference in New Issue
Block a user