Answer stability round 1: 0 🔴, 2 🟡 (0 misread); run CI's image untagged and state where the socket must be

This commit is contained in:
2026-11-18 18:05:39 +01:00
parent 59c319ebb4
commit aa62b081fb
5 changed files with 13 additions and 13 deletions
+2 -2
View File
@@ -240,9 +240,9 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
## Test ## Test
Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in one worker per CPU of a heavy slot (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`. Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy, a Docker daemon, Linux, and a user other than root; a test that lacks one fails, naming it. pytest-xdist runs the tests in one worker per CPU of a heavy slot (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time, with the same needs. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container, which reaches the host's docker daemon for the tests that run the adf-codec shim; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`. CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container. Run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH` and its socket at `/var/run/docker.sock`, which the container shares.
To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout. To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout.
+1 -1
View File
@@ -94,7 +94,7 @@ class RunHeavy(unittest.TestCase):
def test_a_docker_run_without_limits_gets_three_quarters_of_the_memory_split_across_the_slots(self): def test_a_docker_run_without_limits_gets_three_quarters_of_the_memory_split_across_the_slots(self):
memory = read_memory() memory = read_memory()
assert memory is not None, "this test reads the share from /proc/meminfo, which only Linux has" assert memory is not None, "this test needs MemTotal and MemAvailable from /proc/meminfo: run it on Linux"
share = f"{memory.total * 3 // 4 // slot_count(usable_cpus()) // 1024}m" share = f"{memory.total * 3 // 4 // slot_count(usable_cpus()) // 1024}m"
result = self.run_tool("docker", "run", "--rm", "img", "true", env=self.fake_docker()) result = self.run_tool("docker", "run", "--rm", "img", "true", env=self.fake_docker())
self.assertEqual((result.returncode, result.stdout), (0, f"run --memory {share} --memory-swap {share} --rm img true\n")) self.assertEqual((result.returncode, result.stdout), (0, f"run --memory {share} --memory-swap {share} --rm img true\n"))
+2 -2
View File
@@ -13,7 +13,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from run_tests import main from run_tests import main
BIN = pathlib.Path(__file__).resolve().parents[1] BIN = pathlib.Path(__file__).resolve().parents[1]
# The pytest and pytest-xdist run-tests pins, without its worker count, so the tests run them whether or not the interpreter running the tests has pytest. # run-tests' own pytest command, without its worker count. uvx fetches pytest for it, so the Run tests need no pytest installed.
PYTEST_PINNED = [*main.PYTEST[:main.PYTEST.index("-n")], "-p", "no:cacheprovider"] PYTEST_PINNED = [*main.PYTEST[:main.PYTEST.index("-n")], "-p", "no:cacheprovider"]
@@ -124,7 +124,7 @@ class Run(unittest.TestCase):
with tempfile.TemporaryDirectory() as folder: with tempfile.TemporaryDirectory() as folder:
pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n") pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n")
code, output = self.run_in(folder, PYTEST_PINNED, "-s", folder) code, output = self.run_in(folder, PYTEST_PINNED, "-s", folder)
self.assertEqual(code, 0) self.assertEqual(code, 0, output)
lines = output.rstrip("\n").splitlines() lines = output.rstrip("\n").splitlines()
self.assertEqual(len(lines), 2, output) self.assertEqual(len(lines), 2, output)
self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$") self.assertRegex(lines[0], r"^=+ 1 passed in .* =+$")
+2 -2
View File
@@ -8,6 +8,6 @@ def isolated_env() -> dict[str, str]:
return {name: value for name, value in os.environ.items() if not name.startswith(("GIT_", "LOOP_CLAUDE_", "PYTHON")) and name != "CLAUDE_CODE_SANDBOXED"} return {name: value for name, value in os.environ.items() if not name.startswith(("GIT_", "LOOP_CLAUDE_", "PYTHON")) and name != "CLAUDE_CODE_SANDBOXED"}
# Root ignores file permissions, so a test that makes a file unreadable or a folder read-only needs another user. # Call it first in a test that takes a file's or a folder's permissions away.
def assert_not_root(test: unittest.TestCase) -> None: def assert_not_root(test: unittest.TestCase) -> None:
test.assertNotEqual(os.geteuid(), 0, "this test needs file permissions to bind it, which they don't for root: run the tests as another user") test.assertNotEqual(os.geteuid(), 0, "root ignores file permissions, which this test checks: run the tests as a user other than root")
+6 -6
View File
@@ -1,14 +1,14 @@
#!/bin/sh #!/bin/sh
# Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, so the tests that need file permissions to bind them run. # Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, since root ignores file permissions and the tests that check them would fail.
# It is sh, not Python, since it runs on the CI runner's host, which needs only git and docker. # It is sh, not Python, since it runs on the CI runner's host, which needs only git and docker.
# The repo goes in as a git bundle rather than a bind mount, so the script works whether the CI runner shares the docker daemon's filesystem or not. # The repo goes in as a git bundle rather than a bind mount, so the runner need not share the docker daemon's filesystem.
# The adf-codec tests run their shim's container, so the test container holds a docker client and reaches the daemon through its socket, which the daemon mounts from its own host. # The adf-codec tests, which convert Jira descriptions, start a container of their own. So the test container holds a docker client and the daemon's socket.
# The daemon mounts the socket from its own host, while stat reads its group on the runner: both need it at /var/run/docker.sock.
set -eu set -eu
cd "$(dirname "$0")" cd "$(dirname "$0")"
socket=/var/run/docker.sock socket=/var/run/docker.sock
image="claude-config-ci:$$" # An untagged image, which docker image prune removes, since a cancelled run skips any cleanup.
printf 'FROM docker:29.1.3-cli AS docker\nFROM python:3.14.8\nCOPY --from=docker /usr/local/bin/docker /usr/local/bin/docker\n' | docker build --quiet --memory 1g --memory-swap 1g --tag "$image" - >/dev/null image=$(printf 'FROM docker:29.1.3-cli AS docker\nFROM python:3.14.8\nCOPY --from=docker /usr/local/bin/docker /usr/local/bin/docker\n' | docker build --quiet --memory 1g --memory-swap 1g -)
trap 'docker image rm "$image" >/dev/null' EXIT
git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --user "$(id -u):$(id -g)" --group-add "$(stat -c %g "$socket")" -v "$socket:$socket" -e HOME=/tmp "$image" sh -euc ' git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --user "$(id -u):$(id -g)" --group-add "$(stat -c %g "$socket")" -v "$socket:$socket" -e HOME=/tmp "$image" sh -euc '
cat >/tmp/head.bundle cat >/tmp/head.bundle
git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo