163 lines
7.6 KiB
Python
163 lines
7.6 KiB
Python
import contextlib
|
|
import io
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from run_tests import main
|
|
|
|
BIN = pathlib.Path(__file__).resolve().parents[1]
|
|
# run-tests' own pytest command, without its worker count. uvx fetches pytest for it, so the Run tests need no pytest installed.
|
|
PYTEST_PINNED = [*main.PYTEST[:main.PYTEST.index("-n")], "-p", "no:cacheprovider"]
|
|
|
|
|
|
class PytestArgs(unittest.TestCase):
|
|
def test_runs_bin_when_no_argument_names_a_path(self):
|
|
self.assertEqual(main.pytest_args([]), [str(BIN)])
|
|
self.assertEqual(main.pytest_args(["-k", "no_such_path_here"]), ["-k", "no_such_path_here", str(BIN)])
|
|
|
|
def test_passes_a_given_path_on_alone(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
test_file = pathlib.Path(folder, "test_x.py")
|
|
test_file.write_text("")
|
|
self.assertEqual(main.pytest_args([str(test_file)]), [str(test_file)])
|
|
self.assertEqual(main.pytest_args(["-x", f"{test_file}::T::test_a"]), ["-x", f"{test_file}::T::test_a"])
|
|
|
|
|
|
def errors(count: int) -> str:
|
|
return "".join(f"E line {n}\n" for n in range(count))
|
|
|
|
|
|
class Report(unittest.TestCase):
|
|
def test_prints_each_failure_and_error_with_its_errors_then_the_summary(self):
|
|
log = (
|
|
"===== test session starts =====\nE not in a section\n"
|
|
f"===== FAILURES =====\n_____ test_a _____\n\n def test_a():\n{errors(main.ERROR_LINES)}a.py:2: AssertionError\n"
|
|
f"===== ERRORS =====\n_____ ERROR at setup of test_b _____\n{errors(main.ERROR_LINES + 3)}b.py:4: ValueError\n"
|
|
"===== short test summary info =====\nFAILED a.py::test_a\nERROR b.py::test_b\n===== 1 failed, 1 error in 0.10s =====\n"
|
|
)
|
|
self.assertEqual(main.report(log), [
|
|
"_____ test_a _____", *errors(main.ERROR_LINES).splitlines(), "a.py:2: AssertionError",
|
|
"_____ ERROR at setup of test_b _____", *errors(main.ERROR_LINES).splitlines(), "E … 3 more lines in the log", "b.py:4: ValueError",
|
|
"FAILED a.py::test_a", "ERROR b.py::test_b", "===== 1 failed, 1 error in 0.10s =====",
|
|
])
|
|
|
|
def test_prints_nothing_where_pytest_printed_no_summary_line(self):
|
|
self.assertEqual(main.report("usage: pytest\nerror: unrecognized arguments\n"), [])
|
|
|
|
|
|
class Main(unittest.TestCase):
|
|
def test_prints_the_help_for_h_or_help_first_and_runs_anything_else(self):
|
|
for argv in (["-h"], ["--help"], ["--help", "bin/x"]):
|
|
with self.subTest(argv=argv):
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
self.assertEqual(main.main(argv, self.fail), 0)
|
|
self.assertEqual(out.getvalue(), main.__doc__.strip() + "\n")
|
|
ran = []
|
|
self.assertEqual(main.main(["bin/x", "-h"], lambda argv: ran.append(argv) or 1), 1)
|
|
self.assertEqual(ran, [["bin/x", "-h"]])
|
|
|
|
|
|
class RunWithAStandIn(unittest.TestCase):
|
|
"""run with a stand-in for pytest: a Python script given the arguments run passes on."""
|
|
|
|
def run_script(self, script: str, cwd: str | None = None) -> tuple[int, str, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = main.run(["-x"], [sys.executable, "-c", script], cwd)
|
|
lines = out.getvalue().splitlines()
|
|
if lines and lines[-1].startswith("Full log: "):
|
|
self.addCleanup(os.unlink, lines[-1].removeprefix("Full log: "))
|
|
return code, out.getvalue(), err.getvalue()
|
|
|
|
def test_saves_the_log_as_run_tests_star_log_in_the_temp_folder_which_ci_sh_prints(self):
|
|
code, out, _ = self.run_script("print('== 1 passed in 0.1s ==')")
|
|
self.assertEqual(code, 0)
|
|
log = pathlib.Path(out.splitlines()[-1].removeprefix("Full log: "))
|
|
self.assertEqual(log.parent, pathlib.Path(tempfile.gettempdir()))
|
|
self.assertRegex(log.name, r"^run-tests-.+\.log$")
|
|
|
|
def test_runs_in_the_folder_given_and_shows_the_last_lines_of_a_log_without_a_summary(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
script = "import os, sys; sys.stdout.buffer.write(b'bad \\xff byte\\n'); print('\\n'.join(map(str, range(30)))); print(os.getcwd())"
|
|
code, out, _ = self.run_script(script, folder)
|
|
self.assertEqual(code, 0)
|
|
tail = [str(n) for n in range(30 - main.LOG_TAIL + 1, 30)] + [os.path.realpath(folder)]
|
|
self.assertEqual(out.splitlines()[:-1], ["run-tests: pytest printed no summary line; the log ends:", *tail])
|
|
|
|
def test_reads_a_log_that_is_not_utf_8(self):
|
|
code, out, _ = self.run_script("import sys; sys.stdout.buffer.write(b'\\xff\\n== 1 failed in 0.1s ==\\n'); sys.exit(1)")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("== 1 failed in 0.1s ==", out)
|
|
|
|
def test_says_on_stderr_that_a_command_cannot_start(self):
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = main.run([], ["/no/such/uvx"])
|
|
self.assertEqual((code, out.getvalue()), (1, ""))
|
|
self.assertEqual(err.getvalue(), "run-tests: cannot start /no/such/uvx: No such file or directory\n")
|
|
|
|
|
|
class Run(unittest.TestCase):
|
|
def run_in(self, folder: str, pytest: list[str], *args: str) -> tuple[int, str]:
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
|
code = main.run(list(args), pytest, folder)
|
|
return code, out.getvalue()
|
|
|
|
def log_named(self, output: str) -> pathlib.Path:
|
|
last = output.rstrip("\n").splitlines()[-1]
|
|
self.assertTrue(last.startswith("Full log: "), output)
|
|
path = pathlib.Path(last.removeprefix("Full log: "))
|
|
self.addCleanup(path.unlink, missing_ok=True)
|
|
return path
|
|
|
|
def test_a_passing_run_prints_the_summary_line_and_the_log_path(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n")
|
|
code, output = self.run_in(folder, PYTEST_PINNED, "-s", folder)
|
|
self.assertEqual(code, 0, output)
|
|
lines = output.rstrip("\n").splitlines()
|
|
self.assertEqual(len(lines), 2, output)
|
|
self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$")
|
|
self.assertIn("noise from a passing test", self.log_named(output).read_text())
|
|
|
|
def test_a_failing_run_prints_each_failing_test_with_its_error(self):
|
|
# pytest-xdist drops the error from the short summary's lines, so the run takes it from the FAILURES section.
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
pathlib.Path(folder, "test_bad.py").write_text("def test_ok():\n\tpass\n\ndef test_bad():\n\tassert 1 == 2, 'one is not two'\n\ndef test_long():\n\traise ValueError('\\n'.join(map(str, range(30))))\n")
|
|
code, output = self.run_in(folder, PYTEST_PINNED, "-n", "2", folder)
|
|
self.assertEqual(code, 1)
|
|
self.assertRegex(output, r"(?m)^_+ test_bad _+\nE +AssertionError: one is not two\n(E .*\n)*.*test_bad\.py:5: AssertionError$")
|
|
self.assertRegex(output, r"(?m)^FAILED .*test_bad\.py::test_bad\b")
|
|
self.assertRegex(output, r"(?m)^E +… \d+ more lines in the log$")
|
|
self.assertEqual(len(re.findall(r"(?m)^E ", output)), 2 + main.ERROR_LINES + 1)
|
|
self.assertRegex(output, r"=+ 2 failed, 1 passed in .* =+\n")
|
|
self.assertNotIn("def test_bad", output)
|
|
self.assertIn("def test_bad", self.log_named(output).read_text())
|
|
|
|
def test_a_run_that_cannot_start_says_so(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
code, output = self.run_in(folder, [os.path.join(folder, "no-such-uvx")])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("run-tests: cannot start", output)
|
|
self.assertNotIn("Full log: ", output)
|
|
|
|
def test_a_run_that_prints_no_summary_line_shows_the_logs_last_lines(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
code, output = self.run_in(folder, PYTEST_PINNED, "--no-such-option", folder)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("run-tests: pytest printed no summary line; the log ends:", output)
|
|
self.assertIn("--no-such-option", output)
|
|
self.log_named(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|