Run each table-driven test case in its own subTest and split the tests that check several behaviours

This commit is contained in:
2026-11-08 23:01:40 +01:00
parent d8e43bc0fc
commit 40ada0788a
8 changed files with 52 additions and 29 deletions
+14 -5
View File
@@ -36,8 +36,9 @@ class CaseFrom(unittest.TestCase):
def test_reads_each_kind_of_verdict(self):
for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED):
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT)
self.assertEqual(found.archived_kind, verdict)
with self.subTest(verdict=verdict):
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT)
self.assertEqual(found.archived_kind, verdict)
def test_skips_a_copy_with_no_verdict(self):
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
@@ -137,7 +138,8 @@ class Main(unittest.TestCase):
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
self.assertTrue(out.endswith("returns every verdict the reference judge reproduces: claude-opus-5-5 at medium effort\n"))
for said in ("The task the worker was given:\nDo it.", "1. Ship.", "From ~/.claude/CLAUDE.md:\nBe kind."):
self.assertIn(said, self.prompts[0])
with self.subTest(said=said):
self.assertIn(said, self.prompts[0])
# The premises head the prompt and the records follow them.
self.assertRegex(self.prompts[0], r"(?s)From README\.md:.*1\. Ship\..*The records:.*Be kind\.")
# The replay shows the judge the whole transcript, with no step-up marked in it.
@@ -171,15 +173,22 @@ class Main(unittest.TestCase):
self.assertIn("claude-opus-5-5 at medium effort missed 0.", out)
self.assertTrue(out.endswith(": claude-opus-5-5 at medium effort\n"))
def test_exits_1_on_a_usage_limit_bad_arguments_or_an_unreadable_models_file(self):
def test_exits_1_on_a_usage_limit(self):
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
self.assertEqual(code, 1)
self.assertIn("usage limit: You've hit your limit", err)
def test_exits_1_on_bad_arguments(self):
for argv in ([], ["off-ladder"], ["claude-opus-5-5", "x"]):
self.assertEqual(self.run_main(argv, unused)[0], 1)
with self.subTest(argv=argv):
self.assertEqual(self.run_main(argv, unused)[0], 1)
def test_exits_1_for_a_model_no_project_runs_on(self):
code, _, err = self.run_main(["claude-fable-5-1"], unused)
self.assertEqual(code, 1)
self.assertIn("no archived verdicts of projects on claude-fable-5-1", err)
def test_exits_1_on_an_unreadable_models_file(self):
(self.home / ".claude" / "loop-claude-models.json").write_text("[")
self.assertEqual(self.run_main(["claude-opus-5-5"], unused)[0], 1)
+8 -5
View File
@@ -690,7 +690,8 @@ class Limits(unittest.TestCase):
def test_reads_only_a_rejecting_events_whole_seconds(self):
for info, want in [({"status": "rejected", "resetsAt": NOW}, NOW), ({"status": "allowed", "resetsAt": NOW}, None), ({"status": "rejected", "resetsAt": True}, None), ({"status": "rejected"}, None), ("rejected", None)]:
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want, info)
with self.subTest(info=info):
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want)
class MemoryWait(unittest.TestCase):
@@ -1910,8 +1911,9 @@ class ExitContract(unittest.TestCase):
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
self.assertEqual(len(found), len(clauses), found)
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
self.assertEqual((path, function, code), (want_path, want_function, want_code))
self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}")
with self.subTest(path=path, function=function):
self.assertEqual((path, function, code), (want_path, want_function, want_code))
self.assertTrue(reason.startswith(reason_start), reason)
help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split())
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
@@ -1939,8 +1941,9 @@ class JudgeFailures(unittest.TestCase):
found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected"))
self.assertEqual(len(found), len(clauses), found)
for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses):
self.assertEqual(function, want_function)
self.assertTrue(reason.startswith(reason_start), f"{function}: {reason}")
with self.subTest(function=function):
self.assertEqual(function, want_function)
self.assertTrue(reason.startswith(reason_start), reason)
judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split())
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
+6 -2
View File
@@ -17,7 +17,8 @@ from machine_load import slots
class SlotCount(unittest.TestCase):
def test_one_slot_per_four_cpus_and_never_none(self):
for cpus, slots in ((16, 4), (8, 2), (7, 1), (4, 1), (1, 1), (None, 1)):
self.assertEqual(slot_count(cpus), slots, cpus)
with self.subTest(cpus=cpus):
self.assertEqual(slot_count(cpus), slots)
class MemoryShare(unittest.TestCase):
@@ -28,8 +29,11 @@ class MemoryShare(unittest.TestCase):
def test_reads_available_and_total_memory_from_meminfo(self):
meminfo = "MemTotal: 16387072 kB\nMemFree: 981236 kB\nMemAvailable: 4096000 kB\n"
self.assertEqual(memory.parsed(meminfo), memory.Memory(available=4096000, total=16387072))
def test_reads_nothing_from_a_meminfo_missing_a_figure_or_holding_a_bad_one(self):
for unread in ("MemTotal: 16 kB\n", "MemTotal: x kB\nMemAvailable: 1 kB\n", ""):
self.assertIsNone(memory.parsed(unread), unread)
with self.subTest(unread=unread):
self.assertIsNone(memory.parsed(unread))
def test_reads_a_meminfo_file_and_none_where_it_cannot(self):
with tempfile.TemporaryDirectory() as temp:
+2 -1
View File
@@ -843,7 +843,8 @@ class Hook(unittest.TestCase):
def test_help_lists_every_rule(self):
helped = subprocess.run([str(HOOK), "--help"], capture_output=True, check=True, env=isolated_env(), text=True).stdout
for rule in ["action-pin", "commit-one-line", "docker-run-memory", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check"]:
self.assertIn(f" {rule}\n The rule: ", helped)
with self.subTest(rule):
self.assertIn(f" {rule}\n The rule: ", helped)
self.assertIn("reviewer-report\n The rule: a reviewer writes nothing but its report", helped)
self.assertRegex(helped, r" reviewer-report\n( .*\n){2} Takes no override\.\n")
self.assertRegex(helped, r" image-pin\n( .*\n){2} \S", "a rule a marker waives says nothing of overrides")
+15 -6
View File
@@ -31,20 +31,29 @@ class SubagentRung(unittest.TestCase):
def test_refuses_an_l_outside_1_to_4_or_a_wrong_count_of_arguments(self):
for argv in (["claude-opus-5-5", "0"], ["claude-opus-5-5", "01"], ["claude-opus-5-5", "5"], ["claude-opus-5-5", "x"], ["claude-opus-5-5"], ["3", "claude-opus-5-5"], ["claude-opus-5-5", "3", "--plan"], ["--planning", "claude-opus-5-5", "3"]):
result = self.run_tool(*argv)
self.assertEqual((result.returncode, result.stdout), (1, ""), argv)
self.assertIn("usage: subagent-rung [--plan] <model> <L>", result.stderr)
with self.subTest(argv=argv):
result = self.run_tool(*argv)
self.assertEqual((result.returncode, result.stdout), (1, ""))
self.assertIn("usage: subagent-rung [--plan] <model> <L>", result.stderr)
def test_the_launcher_prints_the_rung_and_exits_0_on_and_off_the_ladder(self):
def test_the_launcher_prints_the_rung_and_exits_0_on_the_ladder(self):
on = self.run_tool("claude-opus-5-5", "4")
self.assertEqual((on.returncode, on.stdout, on.stderr), (0, "model opus, effort high\n", ""))
def test_the_launcher_prints_nothing_and_says_why_and_exits_0_off_the_ladder(self):
off = self.run_tool("claude-opus-5", "4")
self.assertEqual((off.returncode, off.stdout), (0, ""))
self.assertEqual(off.stderr, "subagent-rung: claude-opus-5 is not on the routing ladder; pass the Agent tool no model or effort\n")
def test_the_launcher_accepts_the_bounds_of_l(self):
for bound, rung in (("1", "model sonnet, effort high\n"), ("4", "model opus, effort high\n")):
self.assertEqual(self.run_tool("claude-opus-5-5", bound).stdout, rung)
with self.subTest(bound=bound):
self.assertEqual(self.run_tool("claude-opus-5-5", bound).stdout, rung)
def test_the_launcher_prints_the_usage_for_each_help_flag(self):
for flag in ("-h", "--help"):
self.assertIn("usage: subagent-rung [--plan] <model> <L>", self.run_tool(flag).stdout)
with self.subTest(flag=flag):
self.assertIn("usage: subagent-rung [--plan] <model> <L>", self.run_tool(flag).stdout)
if __name__ == "__main__":
unittest.main()
+4 -3
View File
@@ -26,9 +26,10 @@ class Clean(unittest.TestCase):
def test_a_file_that_holds_no_json_object_exits_1_naming_the_tool(self):
for written in ("{", "[]"):
ran = subprocess.run([sys.executable, FILTER], input=written, capture_output=True, text=True)
self.assertEqual((ran.returncode, ran.stdout), (1, ""), written)
self.assertIn("settings_filter.py: ", ran.stderr)
with self.subTest(written=written):
ran = subprocess.run([sys.executable, FILTER], input=written, capture_output=True, text=True)
self.assertEqual((ran.returncode, ran.stdout), (1, ""))
self.assertIn("settings_filter.py: ", ran.stderr)
class Checkout(unittest.TestCase):
+3 -2
View File
@@ -100,8 +100,9 @@ class Script(unittest.TestCase):
def test_warns_on_the_status_line_for_input_that_is_not_a_json_object(self):
for stdin in ("not json", "[]"):
ran = self.run_script(stdin)
self.assertEqual((ran.returncode, ran.stdout, ran.stderr), (0, f"{RED}warning: statusline.py's input is not a JSON object{RESET}", ""))
with self.subTest(stdin=stdin):
ran = self.run_script(stdin)
self.assertEqual((ran.returncode, ran.stdout, ran.stderr), (0, f"{RED}warning: statusline.py's input is not a JSON object{RESET}", ""))
def run_on_branch(self, branch):
with tempfile.TemporaryDirectory() as checkout:
-5
View File
@@ -13,7 +13,6 @@
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | 0.7.0 | | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 1 | 2 | 1 | 2 | 1 | | 0.0 |
| 242 | 0.7.0 | | principle | **Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case.** | 2 | 1 | 1 | 2 | 3 | code health by 2 | 3.6 |
| 113 | 0.7.0 | | | **Compute an item's S from the code it names, where it names any, and show it beside the finished items `score-outcomes` lists.** | 3 | 2 | 2 | 5 | 4 | 1.1 by 5, 1.4 by 5 | 3.4 |
| 263 | 0.7.0 | | | **Have the judge reject an interrupt that re-asks what the human answered earlier in the chunk, unless the ask quotes what that answer left open.** | 2 | 2 | 3 | 5 | 3 | 1.2 by 6, 1.5 by 2 | 3.2 |
| 162 | 0.7.0 | | owner | **Script each step that 0.2.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.** | 2 | 2 | 1 | 3 | 3 | 1.4 by 4, 1.5 by 4 | 3.1 |
@@ -72,10 +71,6 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost
The README line covers the bench's transcripts in `workflow-bench/logs/` too (goal 3).
### 242. Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case.
"Tests" in `principles/technical-principles.md`. The principles audit of PR #380 found test bodies that loop over their cases without `subTest`, some checking several behaviours at once: `bin/judge_replay/test_judge_replay.py:38`, `:139`, `:176`; `bin/loop_claude/test_loop_claude.py:590`; `bin/machine_load/test_machine_load.py:19`, `:31`; `bin/rule_hook/test_rule_hook.py:480`; `bin/subagent_rung/test_subagent_rung.py:32`, `:43`, `:45`; `bin/test_settings_filter.py:28`; `bin/test_statusline.py:102`. The first failing case stops the loop, and without a message the failure doesn't say which case broke. Split the tests that check several behaviours at once, `test_judge_replay.py:176` and `test_subagent_rung.py:43-46`, into one test per behaviour ("One behaviour per test").
### 113. Compute an item's S from the code it names, where it names any, and show it beside the finished items `score-outcomes` lists.
`score-outcomes` grounds L, E, I and S in finished chunks only. Where an item names files or units, their size bounds what the change can touch.