Answer stability round 1: 1 🔴, 3 🟡 (0 misread); kill each test shell's whole process group, check the CPU sum without timing, and file the headless calls' missing time limit as item 447

This commit is contained in:
2026-11-19 09:56:43 +01:00
parent dbfe87d36f
commit ed241f3ab5
2 changed files with 43 additions and 18 deletions
+30 -17
View File
@@ -27,6 +27,13 @@ def clocked():
return dataclasses.replace(system_effects(), local_time=lambda seconds: f"<{seconds}>", now=lambda: NOW)
def ticks(pid):
"""The user and the kernel clock ticks a process has used, read from /proc apart from the code under test."""
with open(f"/proc/{pid}/stat") as handle:
fields = handle.read().rpartition(") ")[2].split()
return int(fields[11]), int(fields[12])
def wait_until(condition):
"""Polls up to 5 s for condition, so a test doesn't depend on how fast a process starts."""
for _ in range(100):
@@ -150,12 +157,18 @@ class PythonEffect(unittest.TestCase):
class ChildShells(unittest.TestCase):
def started(self, command, **options):
"""A shell running command, once a process runs below it."""
proc = subprocess.Popen(["bash", "-c", command], **options)
self.addCleanup(lambda: (proc.kill(), proc.wait()))
"""A shell running command, once a process runs below it. The shell leads a process group of its own, which the cleanup kills, so that no process it started outlives the test."""
proc = subprocess.Popen(["bash", "-c", command], start_new_session=True, **options)
self.addCleanup(self.killed, proc)
wait_until(lambda: child_shells.descendants(proc.pid))
return proc
def killed(self, proc):
# A test that stopped the shell has emptied its group already.
with contextlib.suppress(ProcessLookupError):
os.killpg(proc.pid, signal.SIGKILL)
proc.wait()
def shell_of(self, proc):
[found] = [each for each in child_shells.shells_of(os.getpid()) if each.pid == proc.pid]
return found
@@ -172,14 +185,14 @@ class ChildShells(unittest.TestCase):
self.assertEqual((self.shell_of(naming).runs_busy_tool, self.shell_of(busy).runs_busy_tool), (False, True))
self.assertLess(abs(self.shell_of(naming).started - time.time()), 5)
def test_a_shell_below_which_a_process_holds_a_heavy_slot_file_runs_a_busy_tool(self):
def test_a_shell_below_which_a_process_holds_a_heavy_slot_file_holds_a_slot_and_runs_no_busy_tool(self):
# run-heavy replaces itself with its command, which keeps the slot's file open.
with tempfile.TemporaryDirectory() as state, unittest.mock.patch.dict(os.environ, {"XDG_STATE_HOME": state}):
slot = pathlib.Path(state, "run-heavy", "slot-0")
slot.parent.mkdir()
holding = self.started(f"exec 9>{slot}; sleep 60; true")
plain = self.started("sleep 60; true")
self.assertEqual((self.shell_of(holding).runs_busy_tool, self.shell_of(plain).runs_busy_tool), (True, False))
self.assertEqual([(self.shell_of(each).holds_slot, self.shell_of(each).runs_busy_tool) for each in (holding, plain)], [(True, False), (False, False)])
def test_a_shell_written_output_is_the_modified_time_of_its_output_file_and_else_its_start(self):
with tempfile.NamedTemporaryFile() as output:
@@ -189,7 +202,7 @@ class ChildShells(unittest.TestCase):
self.assertEqual(self.shell_of(to_file).written, 1000)
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
def test_reads_the_end_of_a_shells_output_file_and_nothing_from_a_pipe(self):
def test_reads_the_end_of_a_shells_output_file_and_no_output_from_a_pipe(self):
for written, want in ((b"x" * child_shells.OUTPUT_BYTES + b"last line\n", "x" * (child_shells.OUTPUT_BYTES - 10) + "last line\n"), (b"short \xff\n", "short \ufffd\n")):
with self.subTest(want=want[-12:]), tempfile.NamedTemporaryFile() as output:
output.write(written)
@@ -197,24 +210,24 @@ class ChildShells(unittest.TestCase):
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=output)).output, want)
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=subprocess.PIPE)).output, "")
def test_reads_nothing_of_a_process_that_is_gone(self):
def test_reads_no_cpu_state_or_output_of_a_process_that_is_gone(self):
gone = subprocess.Popen(["true"])
gone.wait()
self.assertEqual((child_shells.cpu_and_state(gone.pid), child_shells.output_tail(gone.pid)), ((0.0, "gone"), ""))
def test_reads_each_process_below_a_shell_and_the_user_and_kernel_cpu_the_whole_tree_used(self):
# dd from /dev/zero spends its time in the kernel, and the sleep after it uses none, so the total must add both modes of every process.
busy = self.started("dd if=/dev/zero of=/dev/null bs=1M & sleep 60; true")
# The shell counts in user mode before it starts dd, which spends its time in the kernel. Stopping dd freezes every figure, so the total must equal the ticks of all three processes, each mode counted once.
busy = self.started("i=0; while [ $i -lt 100000 ]; do i=$((i+1)); done; dd if=/dev/zero of=/dev/null bs=1M & sleep 60; true")
wait_until(lambda: len(child_shells.descendants(busy.pid)) == 2)
before, started = self.shell_of(busy).cpu, time.monotonic()
time.sleep(1)
used = self.shell_of(busy).cpu - before
self.assertGreater(used, 0.2 * (time.monotonic() - started))
# Under 2 × the kernel's share alone, so a sum that counts one mode twice fails.
self.assertLess(used, 1.15 * (time.monotonic() - started))
dd, sleep = child_shells.descendants(busy.pid)
wait_until(lambda: ticks(dd)[1] > 0)
os.kill(dd, signal.SIGSTOP)
wait_until(lambda: child_shells.cpu_and_state(dd)[1] == "T")
self.assertGreater(ticks(busy.pid)[0], 0)
self.assertAlmostEqual(self.shell_of(busy).cpu, sum(sum(ticks(pid)) for pid in (busy.pid, dd, sleep)) / os.sysconf("SC_CLK_TCK"))
processes = self.shell_of(busy).processes
self.assertEqual([line.split(" ", 4)[4] for line in processes], ["dd if=/dev/zero of=/dev/null bs=1M", "sleep 60"])
self.assertEqual([line.split(" ")[1] for line in processes], ["R", "S"])
self.assertEqual([line.split(" ")[:2] for line in processes], [[str(dd), "T"], [str(sleep), "S"]])
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
shell = self.started("sleep 60; true")
@@ -246,7 +259,7 @@ class ChildShells(unittest.TestCase):
wait_until(lambda: len(child_shells.descendants(shell.pid)) == 2)
def test_a_process_that_is_gone_holds_no_slot_and_a_vanished_child_does_not_hide_the_shells_after_it(self):
self.assertFalse(child_shells.holds_slot(2**30, pathlib.Path("/")))
self.assertFalse(child_shells.has_slot(2**30, pathlib.Path("/")))
shell = self.started("sleep 60; true")
real = child_shells.children_of
with unittest.mock.patch.object(child_shells, "children_of", side_effect=lambda pid: [2**30, shell.pid] if pid == os.getpid() else real(pid)):
+13 -1
View File
@@ -6,7 +6,7 @@
`Bar = 0.1`
`Next ID = 447`
`Next ID = 448`
## Items
@@ -26,6 +26,7 @@
| 298 | 0.13.0 | | | **Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind.** | 3 | 3 | 3 | 7 | 4 | 1.4 by 2, 1.5 by 1 | 0.9 |
| 395 | 0.13.0 | | decision | **Keep each block a save didn't change, with its localId, and the spacing paragraphs beside it as Jira or Confluence holds them.** | 3 | 2 | 2 | 5 | 5 | 1.6 by 4 | 0.8 |
| 425 | 0.13.0 | | | **Show on loop-claude's terminal why it doesn't start a chunk beside the next item, as chunk-preflight prints it.** | 1 | 1 | 2 | 2 | 2 | 1.5 by 2 | 0.7 |
| 447 | 0.13.0 | | | **Stop a headless claude call that runs past a time limit, and report it as a failure, so a hung judge or stuck check can't hold the loop.** | 2 | 2 | 2 | 4 | 3 | 1.5 by 2 | 0.7 |
| 367 | 0.13.0 | | | **Check that every write to a Jira plan leaves it ranked, as rule-hook's `todo-rank-check` checks `todo.md` today.** | 2 | 2 | 3 | 5 | 3 | 1.6 by 2 | 0.7 |
| 423 | 0.13.0 | | | **Skip the judge's model call on a worker's final message that the offline classifier decides is finished.** | 3 | 3 | 3 | 7 | 5 | 1.4 by 2 | 0.4 |
| 77 | 0.14.0 | | | **Teach `bin/metric-bench` Go and TypeScript, rerun it once two projects' comprehension histories name places, and bring into review only a metric that tracks the panel.** | 4 | 2 | 1 | 5 | 6 | 1.1 by 7, 1.4 by 7 | 2.1 |
@@ -207,6 +208,17 @@ Today a Jira detail edited in one line, or a Confluence record with one entry ap
`chunks_beside` names each item it refuses and why, such as an unreadable `Touches:` list, and chunk-preflight prints those lines, but loop-claude's `chunk_windows.may_start` keeps only `picked` and drops them, so idle slots show no reason. Found by item 424's product-owner review; a list a person edits in Jira's UI, or one filed before item 424, still reaches loop-claude unchecked.
### 447. Stop a headless claude call that runs past a time limit, and report it as a failure, so a hung judge or stuck check can't hold the loop.
Every headless claude that `claude_reply` runs, the judge's and the stuck check's included, runs without a time limit, and the stuck check runs while the worker's own claude waits for its output to be read. A call that hangs holds the loop, and the worker with it, until a person notices. Found by item 422's stability review.
Give the `claude` effect a timeout, and have `claude_reply` turn its expiry into a Failure that names who timed out, as it names a non-zero exit. Pick the limit from the longest such call the transcript archive shows, with room to spare, and say where it comes from in a comment at the constant.
Touches:
- `bin/loop_effects/effects.py`: `system_effects` (small)
- `bin/claude_code/reply.py`: `claude_reply`
### 367. Check that every write to a Jira plan leaves it ranked, as rule-hook's `todo-rank-check` checks `todo.md` today.
One of 0.13.0's history ports, per lilleman's call of 2026-11-12.