Fail the tests that skipped at run time, naming what they miss, and give CI's tests docker
This commit is contained in:
@@ -242,7 +242,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
|
||||
|
||||
Test on the host with `bin/run-tests`, which runs pytest through `uvx`, pinned in `bin/run_tests/main.py`. It prints each failing test with its error and pytest's summary line, and saves the whole output to a file whose path it prints. It passes on any test paths or pytest options it is given, such as `bin/run-tests bin/chunk_pr -x`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in one worker per CPU of a heavy slot (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
|
||||
|
||||
CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`.
|
||||
CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until it passes. The script runs the same tests on the committed HEAD in a pinned Python container, which reaches the host's docker daemon for the tests that run the adf-codec shim; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`.
|
||||
|
||||
To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
@@ -296,10 +295,10 @@ def test_container_run_passes_an_image_failure_through(tmp_path):
|
||||
assert convert.container_run("to-adf", "text", scripted(down), shim_dir(tmp_path)) == down
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True, timeout=20).returncode != 0, reason="needs a docker daemon")
|
||||
def test_round_trip_through_the_real_container():
|
||||
doc = convert.markdown_to_adf("# Title\n\nSome **bold** text.\n")
|
||||
assert isinstance(doc, dict) and doc["type"] == "doc" and doc["content"][0]["type"] == "heading"
|
||||
assert isinstance(doc, dict), doc
|
||||
assert doc["type"] == "doc" and doc["content"][0]["type"] == "heading"
|
||||
markdown = convert.adf_to_markdown(doc)
|
||||
assert isinstance(markdown, str) and markdown.startswith("# Title") and "**bold**" in markdown
|
||||
refused = convert.adf_to_markdown({"type": "nope"})
|
||||
@@ -324,10 +323,9 @@ JIRA_DESCRIPTION = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True, timeout=20).returncode != 0, reason="needs a docker daemon")
|
||||
def test_the_lossless_pair_gives_back_a_jira_description_identical_and_the_portable_pair_does_not():
|
||||
markdown = convert.adf_to_markdown_lossless(JIRA_DESCRIPTION)
|
||||
assert isinstance(markdown, str)
|
||||
assert isinstance(markdown, str), markdown
|
||||
assert convert.markdown_to_adf_lossless(markdown) == JIRA_DESCRIPTION
|
||||
assert convert.adfs_to_markdown_lossless([JIRA_DESCRIPTION, JIRA_DESCRIPTION]) == [markdown, markdown]
|
||||
portable = convert.markdown_to_adf(convert.adf_to_markdown(JIRA_DESCRIPTION))
|
||||
|
||||
@@ -18,7 +18,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from test_env import isolated_env
|
||||
from test_env import assert_not_root, isolated_env
|
||||
from project_config.test_project_config import write_config
|
||||
|
||||
BIN = pathlib.Path(__file__).resolve().parents[1]
|
||||
@@ -753,8 +753,8 @@ class Cycles(unittest.TestCase):
|
||||
self.assertIn("past the noise", run.stdout)
|
||||
self.assertIn("past the noise", run.calls[1]["stdin"])
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
||||
def test_exits_1_when_the_transcript_cannot_be_read(self):
|
||||
assert_not_root(self)
|
||||
checkout = Checkout(self)
|
||||
run = checkout.run([worker(unreadable=True)])
|
||||
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
||||
@@ -1000,8 +1000,8 @@ class Interrupt(unittest.TestCase):
|
||||
self.assertFalse(run.calls[2]["ctrl_c_ignored"])
|
||||
self.assertIn("[human] B", run.calls[3]["stdin"])
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
||||
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
|
||||
assert_not_root(self)
|
||||
checkout = Checkout(self)
|
||||
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="/talk\n", before_typing=ANSWERING)
|
||||
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
|
||||
@@ -1082,8 +1082,8 @@ class Models(unittest.TestCase):
|
||||
self.assertIn("no model catalog cached", run.stderr)
|
||||
self.assertEqual(argument(run.calls[0], "--model"), "1")
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
||||
def test_exits_1_when_the_models_file_cannot_be_written(self):
|
||||
assert_not_root(self)
|
||||
checkout = Checkout(self)
|
||||
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
||||
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
|
||||
|
||||
@@ -18,6 +18,7 @@ import uuid
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from test_env import assert_not_root
|
||||
from routing_figures import archive, state
|
||||
from claude_code.output_limits import BASH_CEILING_CHARACTERS
|
||||
from routing_figures.calls import Call
|
||||
@@ -519,8 +520,8 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
[next_chunk] = set(self.archive.iterdir()) - {chunk}
|
||||
self.assertEqual(sorted(path.read_text() for path in (next_chunk / "session" / str(self.session.id)).glob("custom-title.json*")), ["third"])
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable folder")
|
||||
def test_stops_on_a_session_folder_it_cannot_read_whole(self):
|
||||
assert_not_root(self)
|
||||
locked = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents"
|
||||
locked.mkdir(parents=True)
|
||||
locked.chmod(0)
|
||||
@@ -689,8 +690,8 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
self.assertIn(f"\n{STAMP} Which one?\a\n", said.getvalue())
|
||||
self.assertNotIn("Use ... A.", said.getvalue())
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
||||
def test_stops_keeping_the_transcript_when_the_finished_chunks_session_cannot_be_deleted(self):
|
||||
assert_not_root(self)
|
||||
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
|
||||
(folder / "subagents").mkdir(parents=True)
|
||||
folder.chmod(0o500)
|
||||
@@ -967,8 +968,8 @@ class RoutingCheck(unittest.TestCase):
|
||||
stop, _ = self.check(FakeEffects(on_terminal_claude=[OSError(2, "No such file or directory")], git=[UNREAD] * 3))
|
||||
self.assertEqual(stop, Stop(1, f"could not start claude for the routing calls in {self.home / 'code' / 'claude-config'}, the setup's development clone: No such file or directory"))
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
||||
def test_says_a_figure_it_cannot_write_and_goes_on(self):
|
||||
assert_not_root(self)
|
||||
self.chunks(5, 4)
|
||||
(self.root / "routing-figures.lock").touch()
|
||||
self.root.chmod(0o555)
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
@@ -22,9 +20,6 @@ Touches:
|
||||
More prose, with `bin/b.py` named.
|
||||
"""
|
||||
|
||||
needs_docker = unittest.skipIf(shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True, timeout=20).returncode != 0, "adf_codec runs in a container, which needs a docker daemon")
|
||||
|
||||
|
||||
def text(words: str, code: bool = False) -> dict:
|
||||
return {"type": "text", "text": words, **({"marks": [{"type": "code"}]} if code else {})}
|
||||
|
||||
@@ -90,15 +85,14 @@ class PredictedTest(unittest.TestCase):
|
||||
def test_a_longer_run_with_a_space_at_one_end_only_keeps_it(self) -> None:
|
||||
self.assertEqual(predicted("Touches:\n- `a.md`: `` x``, ``y ``, `` a ``\n"), {"a.md": (Part(" x"), Part("y "), Part("a"))})
|
||||
|
||||
@needs_docker
|
||||
def test_a_list_written_in_markdown_and_read_back_from_a_jira_description_reads_as_written(self) -> None:
|
||||
description = convert.markdown_to_adf(DETAIL)
|
||||
assert isinstance(description, dict)
|
||||
assert isinstance(description, dict), description
|
||||
self.assertEqual(predicted(convert.adf_to_markdown(description)), predicted(DETAIL))
|
||||
|
||||
@needs_docker
|
||||
def test_a_list_typed_in_jiras_editor_reads_as_its_files_and_parts(self) -> None:
|
||||
read = convert.adf_to_markdown(JIRA_TOUCHES)
|
||||
assert isinstance(read, str), read
|
||||
self.assertEqual(predicted(read), {
|
||||
"bin/a.py": (Part("main"), Part("Plan.branch", small=True)),
|
||||
"a`b.md": (Part("## The `foo` tool", small=True), Part("`x`")),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import dataclasses
|
||||
import decimal
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -8,6 +7,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from test_env import assert_not_root
|
||||
from project_config.settings import DEFAULT_FLOOR, Epic, Finish, JiraScoring, ProjectConfig, SETTINGS
|
||||
from failure import Failure
|
||||
from record_store.stores import FileStore, goals_store, store as record_store
|
||||
@@ -136,8 +136,8 @@ class FilePlan(FilePlanContract):
|
||||
self.assertEqual(store.edit(id_, {}, "### Why\n"), Failure(split))
|
||||
self.assertEqual((self.folder / "todo.md").read_text(), before)
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
||||
def test_a_file_it_cannot_write_fails_naming_it(self):
|
||||
assert_not_root(self)
|
||||
store = self.new_store()
|
||||
self.folder.chmod(0o555)
|
||||
self.addCleanup(self.folder.chmod, 0o755)
|
||||
|
||||
@@ -25,7 +25,7 @@ from routing_figures.state import due
|
||||
from claude_code.entries import entry_from
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from test_env import isolated_env
|
||||
from test_env import assert_not_root, isolated_env
|
||||
from plan.todo_table import Item
|
||||
from transcript_archive import outcomes, write
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
@@ -495,8 +495,8 @@ class Archive(unittest.TestCase):
|
||||
self.assertIsNone(state.mark_raised(self.root, [a], "2026-10-15T02:00:00Z"))
|
||||
self.assertEqual(state.raised(self.root), state.Raised("2026-10-15T01:00:00Z", frozenset({a})))
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
||||
def test_fails_when_it_cannot_keep_the_state(self):
|
||||
assert_not_root(self)
|
||||
self.root.chmod(0o555)
|
||||
self.addCleanup(self.root.chmod, 0o755)
|
||||
self.assertIsInstance(state.mark_measured(self.root, 40), Failure)
|
||||
|
||||
@@ -94,8 +94,7 @@ class RunHeavy(unittest.TestCase):
|
||||
|
||||
def test_a_docker_run_without_limits_gets_three_quarters_of_the_memory_split_across_the_slots(self):
|
||||
memory = read_memory()
|
||||
if memory is None:
|
||||
self.skipTest("no /proc/meminfo to read the share from")
|
||||
assert memory is not None, "this test reads the share from /proc/meminfo, which only Linux has"
|
||||
share = f"{memory.total * 3 // 4 // slot_count(usable_cpus()) // 1024}m"
|
||||
result = self.run_tool("docker", "run", "--rm", "img", "true", env=self.fake_docker())
|
||||
self.assertEqual((result.returncode, result.stdout), (0, f"run --memory {share} --memory-swap {share} --rm img true\n"))
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
from unittest import mock
|
||||
import os
|
||||
@@ -14,10 +13,8 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
from run_tests import main
|
||||
|
||||
BIN = pathlib.Path(__file__).resolve().parents[1]
|
||||
# The suite runs under pytest, and `python3 -m unittest` may run it where pytest is not installed.
|
||||
HAS_PYTEST = importlib.util.find_spec("pytest") is not None
|
||||
HAS_XDIST = importlib.util.find_spec("xdist") is not None
|
||||
PYTEST_HERE = [sys.executable, "-m", "pytest", "-p", "no:cacheprovider"]
|
||||
# The pytest and pytest-xdist run-tests pins, without its worker count, so the tests run them whether or not the interpreter running the tests has pytest.
|
||||
PYTEST_PINNED = [*main.PYTEST[:main.PYTEST.index("-n")], "-p", "no:cacheprovider"]
|
||||
|
||||
|
||||
class PytestArgs(unittest.TestCase):
|
||||
@@ -109,7 +106,6 @@ class RunWithAStandIn(unittest.TestCase):
|
||||
self.assertEqual(err.getvalue(), "run-tests: cannot start /no/such/uvx: No such file or directory\n")
|
||||
|
||||
|
||||
@unittest.skipUnless(HAS_PYTEST, "needs pytest importable")
|
||||
class Run(unittest.TestCase):
|
||||
def run_in(self, folder: str, pytest: list[str], *args: str) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
@@ -127,7 +123,7 @@ class Run(unittest.TestCase):
|
||||
def test_a_passing_run_prints_the_summary_line_and_the_log_path(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
pathlib.Path(folder, "test_ok.py").write_text("def test_ok():\n\tprint('noise from a passing test')\n")
|
||||
code, output = self.run_in(folder, PYTEST_HERE, "-s", folder)
|
||||
code, output = self.run_in(folder, PYTEST_PINNED, "-s", folder)
|
||||
self.assertEqual(code, 0)
|
||||
lines = output.rstrip("\n").splitlines()
|
||||
self.assertEqual(len(lines), 2, output)
|
||||
@@ -136,10 +132,9 @@ class Run(unittest.TestCase):
|
||||
|
||||
def test_a_failing_run_prints_each_failing_test_with_its_error(self):
|
||||
# pytest-xdist drops the error from the short summary's lines, so the run takes it from the FAILURES section.
|
||||
workers = ["-n", "2"] if HAS_XDIST else []
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
pathlib.Path(folder, "test_bad.py").write_text("def test_ok():\n\tpass\n\ndef test_bad():\n\tassert 1 == 2, 'one is not two'\n\ndef test_long():\n\traise ValueError('\\n'.join(map(str, range(30))))\n")
|
||||
code, output = self.run_in(folder, PYTEST_HERE, *workers, folder)
|
||||
code, output = self.run_in(folder, PYTEST_PINNED, "-n", "2", folder)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertRegex(output, r"(?m)^_+ test_bad _+\nE +AssertionError: one is not two\n(E .*\n)*.*test_bad\.py:5: AssertionError$")
|
||||
self.assertRegex(output, r"(?m)^FAILED .*test_bad\.py::test_bad\b")
|
||||
@@ -158,7 +153,7 @@ class Run(unittest.TestCase):
|
||||
|
||||
def test_a_run_that_prints_no_summary_line_shows_the_logs_last_lines(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
code, output = self.run_in(folder, PYTEST_HERE, "--no-such-option", folder)
|
||||
code, output = self.run_in(folder, PYTEST_PINNED, "--no-such-option", folder)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("run-tests: pytest printed no summary line; the log ends:", output)
|
||||
self.assertIn("--no-such-option", output)
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import os
|
||||
import unittest
|
||||
|
||||
|
||||
# A git hook's GIT_DIR, the host's LOOP_CLAUDE_* or PYTHONPATH would point the program under test at the host's own state,
|
||||
# and the host's CLAUDE_CODE_SANDBOXED would hide whether loop-claude sets it.
|
||||
def isolated_env() -> dict[str, str]:
|
||||
return {name: value for name, value in os.environ.items() if not name.startswith(("GIT_", "LOOP_CLAUDE_", "PYTHON")) and name != "CLAUDE_CODE_SANDBOXED"}
|
||||
|
||||
|
||||
# Root ignores file permissions, so a test that makes a file unreadable or a folder read-only needs another user.
|
||||
def assert_not_root(test: unittest.TestCase) -> None:
|
||||
test.assertNotEqual(os.geteuid(), 0, "this test needs file permissions to bind it, which they don't for root: run the tests as another user")
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
#!/bin/sh
|
||||
# Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, so the tests that skip under root still run.
|
||||
# Runs the test suite on the committed HEAD as CI does: in a pinned Python container, as the caller's uid, so the tests that need file permissions to bind them run.
|
||||
# It is sh, not Python, since it runs on the CI runner's host, which needs only git and docker.
|
||||
# The repo goes in as a git bundle rather than a bind mount, so the script works whether the CI runner shares the docker daemon's filesystem or not.
|
||||
# The adf-codec tests run their shim's container, so the test container holds a docker client and reaches the daemon through its socket, which the daemon mounts from its own host.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --user "$(id -u):$(id -g)" -e HOME=/tmp python:3.14.8 sh -euc '
|
||||
socket=/var/run/docker.sock
|
||||
image="claude-config-ci:$$"
|
||||
printf 'FROM docker:29.1.3-cli AS docker\nFROM python:3.14.8\nCOPY --from=docker /usr/local/bin/docker /usr/local/bin/docker\n' | docker build --quiet --memory 1g --memory-swap 1g --tag "$image" - >/dev/null
|
||||
trap 'docker image rm "$image" >/dev/null' EXIT
|
||||
git bundle create - HEAD | docker run --rm -i --memory 4g --memory-swap 4g --user "$(id -u):$(id -g)" --group-add "$(stat -c %g "$socket")" -v "$socket:$socket" -e HOME=/tmp "$image" sh -euc '
|
||||
cat >/tmp/head.bundle
|
||||
git -c init.defaultBranch=main -c advice.detachedHead=false clone --quiet /tmp/head.bundle /tmp/repo
|
||||
cd /tmp/repo
|
||||
|
||||
Reference in New Issue
Block a user