diff --git a/README.md b/README.md index bec11763..c23cfa32 100644 --- a/README.md +++ b/README.md @@ -240,9 +240,9 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a ## Test -Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in one worker per CPU of a heavy slot (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`. +Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy, a Docker daemon, Linux, and a user other than root; a test that lacks one fails, naming it. pytest-xdist runs the tests in one worker per CPU of a heavy slot (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time, with the same needs. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`. -CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container, which reaches the host's docker daemon for the tests that run the adf-codec shim; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`. +CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container. Run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH` and its socket at `/var/run/docker.sock`, which the container shares. To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout. diff --git a/bin/run_heavy/test_run_heavy.py b/bin/run_heavy/test_run_heavy.py index af147f8d..c64b55b7 100644 --- a/bin/run_heavy/test_run_heavy.py +++ b/bin/run_heavy/test_run_heavy.py @@ -94,7 +94,7 @@ class RunHeavy(unittest.TestCase): def test_a_docker_run_without_limits_gets_three_quarters_of_the_memory_split_across_the_slots(self): memory = read_memory() - assert memory is not None, "this test reads the share from /proc/meminfo, which only Linux has" + assert memory is not None, "this test needs MemTotal and MemAvailable from /proc/meminfo: run it on Linux" share = f"{memory.total * 3 // 4 // slot_count(usable_cpus()) // 1024}m" result = self.run_tool("docker", "run", "--rm", "img", "true", env=self.fake_docker()) self.assertEqual((result.returncode, result.stdout), (0, f"run --memory {share} --memory-swap {share} --rm img true\n")) diff --git a/bin/run_tests/test_run_tests.py b/bin/run_tests/test_run_tests.py index 9e31a7e7..538385b1 100644 --- a/bin/run_tests/test_run_tests.py +++ b/bin/run_tests/test_run_tests.py @@ -13,7 +13,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from run_tests import main BIN = pathlib.Path(__file__).resolve().parents[1] -# The pytest and pytest-xdist run-tests pins, without its worker count, so the tests run them whether or not the interpreter running the tests has pytest. +# run-tests' own pytest command, without its worker count. uvx fetches pytest for it, so the Run tests need no pytest installed. PYTEST_PINNED = [*main.PYTEST[:main.PYTEST.index("-n")], "-p", "no:cacheprovider"] @@ -124,7 +124,7 @@ class Run(unittest.TestCase): with tempfile.TemporaryDirectory() as folder: pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n") code, output = self.run_in(folder, PYTEST_PINNED, "-s", folder) - self.assertEqual(code, 0) + self.assertEqual(code, 0, output) lines = output.rstrip("\n").splitlines() self.assertEqual(len(lines), 2, output) self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$") diff --git a/bin/test_env.py b/bin/test_env.py index ab231493..09106e0d 100644 --- a/bin/test_env.py +++ b/bin/test_env.py @@ -8,6 +8,6 @@ def isolated_env() -> dict[str, str]: return {name: value for name, value in os.environ.items() if not name.startswith(("GIT_", "LOOP_CLAUDE_", "PYTHON")) and name != "CLAUDE_CODE_SANDBOXED"} -# Root ignores file permissions, so a test that makes a file unreadable or a folder read-only needs another user. +# Call it first in a test that takes a file's or a folder's permissions away. def assert_not_root(test: unittest.TestCase) -> None: - test.assertNotEqual(os.geteuid(), 0, "this test needs file permissions to bind it, which they don't for root: run the tests as another user") + test.assertNotEqual(os.geteuid(), 0, "root ignores file permissions, which this test checks: run the tests as a user other than root") diff --git a/ci.sh b/ci.sh index d5d44325..1551c2bc 100755 --- a/ci.sh +++ b/ci.sh @@ -1,14 +1,14 @@ #!/bin/sh -# Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, so the tests that need file permissions to bind them run. +# Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, since root ignores file permissions and the tests that check them would fail. # It is sh, not Python, since it runs on the CI runner's host, which needs only git and docker. -# The repo goes in as a git bundle rather than a bind mount, so the script works whether the CI runner shares the docker daemon's filesystem or not. -# The adf-codec tests run their shim's container, so the test container holds a docker client and reaches the daemon through its socket, which the daemon mounts from its own host. +# The repo goes in as a git bundle rather than a bind mount, so the runner need not share the docker daemon's filesystem. +# The adf-codec tests, which convert Jira descriptions, start a container of their own. So the test container holds a docker client and the daemon's socket. +# The daemon mounts the socket from its own host, while stat reads its group on the runner: both need it at /var/run/docker.sock. set -eu cd "$(dirname "$0")" socket=/var/run/docker.sock -image="claude-config-ci:$$" -printf 'FROM docker:29.1.3-cli AS docker\nFROM python:3.14.8\nCOPY --from=docker /usr/local/bin/docker /usr/local/bin/docker\n' | docker build --quiet --memory 1g --memory-swap 1g --tag "$image" - >/dev/null -trap 'docker image rm "$image" >/dev/null' EXIT +# An untagged image, which docker image prune removes, since a cancelled run skips any cleanup. +image=$(printf 'FROM docker:29.1.3-cli AS docker\nFROM python:3.14.8\nCOPY --from=docker /usr/local/bin/docker /usr/local/bin/docker\n' | docker build --quiet --memory 1g --memory-swap 1g -) git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --user "$(id -u):$(id -g)" --group-add "$(stat -c %g "$socket")" -v "$socket:$socket" -e HOME=/tmp "$image" sh -euc ' cat >/tmp/head.bundle git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo