diff --git a/bin/judge_replay/test_judge_replay.py b/bin/judge_replay/test_judge_replay.py index c957a4a1..f1925426 100644 --- a/bin/judge_replay/test_judge_replay.py +++ b/bin/judge_replay/test_judge_replay.py @@ -36,8 +36,9 @@ class CaseFrom(unittest.TestCase): def test_reads_each_kind_of_verdict(self): for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED): - found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT) - self.assertEqual(found.archived_kind, verdict) + with self.subTest(verdict=verdict): + found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT) + self.assertEqual(found.archived_kind, verdict) def test_skips_a_copy_with_no_verdict(self): self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT)) @@ -137,7 +138,8 @@ class Main(unittest.TestCase): self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out) self.assertTrue(out.endswith("returns every verdict the reference judge reproduces: claude-opus-5-5 at medium effort\n")) for said in ("The task the worker was given:\nDo it.", "1. Ship.", "From ~/.claude/CLAUDE.md:\nBe kind."): - self.assertIn(said, self.prompts[0]) + with self.subTest(said=said): + self.assertIn(said, self.prompts[0]) # The premises head the prompt and the records follow them. self.assertRegex(self.prompts[0], r"(?s)From README\.md:.*1\. Ship\..*The records:.*Be kind\.") # The replay shows the judge the whole transcript, with no step-up marked in it. @@ -171,15 +173,22 @@ class Main(unittest.TestCase): self.assertIn("claude-opus-5-5 at medium effort missed 0.", out) self.assertTrue(out.endswith(": claude-opus-5-5 at medium effort\n")) - def test_exits_1_on_a_usage_limit_bad_arguments_or_an_unreadable_models_file(self): + def test_exits_1_on_a_usage_limit(self): code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"}) self.assertEqual(code, 1) self.assertIn("usage limit: You've hit your limit", err) + + def test_exits_1_on_bad_arguments(self): for argv in ([], ["off-ladder"], ["claude-opus-5-5", "x"]): - self.assertEqual(self.run_main(argv, unused)[0], 1) + with self.subTest(argv=argv): + self.assertEqual(self.run_main(argv, unused)[0], 1) + + def test_exits_1_for_a_model_no_project_runs_on(self): code, _, err = self.run_main(["claude-fable-5-1"], unused) self.assertEqual(code, 1) self.assertIn("no archived verdicts of projects on claude-fable-5-1", err) + + def test_exits_1_on_an_unreadable_models_file(self): (self.home / ".claude" / "loop-claude-models.json").write_text("[") self.assertEqual(self.run_main(["claude-opus-5-5"], unused)[0], 1) diff --git a/bin/loop_claude/test_loop_claude.py b/bin/loop_claude/test_loop_claude.py index d324575a..5b8c8dc7 100644 --- a/bin/loop_claude/test_loop_claude.py +++ b/bin/loop_claude/test_loop_claude.py @@ -690,7 +690,8 @@ class Limits(unittest.TestCase): def test_reads_only_a_rejecting_events_whole_seconds(self): for info, want in [({"status": "rejected", "resetsAt": NOW}, NOW), ({"status": "allowed", "resetsAt": NOW}, None), ({"status": "rejected", "resetsAt": True}, None), ({"status": "rejected"}, None), ("rejected", None)]: - self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want, info) + with self.subTest(info=info): + self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want) class MemoryWait(unittest.TestCase): @@ -1910,8 +1911,9 @@ class ExitContract(unittest.TestCase): found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)] self.assertEqual(len(found), len(clauses), found) for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses): - self.assertEqual((path, function, code), (want_path, want_function, want_code)) - self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}") + with self.subTest(path=path, function=function): + self.assertEqual((path, function, code), (want_path, want_function, want_code)) + self.assertTrue(reason.startswith(reason_start), reason) help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split()) self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], []) @@ -1939,8 +1941,9 @@ class JudgeFailures(unittest.TestCase): found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected")) self.assertEqual(len(found), len(clauses), found) for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses): - self.assertEqual(function, want_function) - self.assertTrue(reason.startswith(reason_start), f"{function}: {reason}") + with self.subTest(function=function): + self.assertEqual(function, want_function) + self.assertTrue(reason.startswith(reason_start), reason) judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split()) self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], []) diff --git a/bin/machine_load/test_machine_load.py b/bin/machine_load/test_machine_load.py index a73a720d..728c41be 100644 --- a/bin/machine_load/test_machine_load.py +++ b/bin/machine_load/test_machine_load.py @@ -17,7 +17,8 @@ from machine_load import slots class SlotCount(unittest.TestCase): def test_one_slot_per_four_cpus_and_never_none(self): for cpus, slots in ((16, 4), (8, 2), (7, 1), (4, 1), (1, 1), (None, 1)): - self.assertEqual(slot_count(cpus), slots, cpus) + with self.subTest(cpus=cpus): + self.assertEqual(slot_count(cpus), slots) class MemoryShare(unittest.TestCase): @@ -28,8 +29,11 @@ class MemoryShare(unittest.TestCase): def test_reads_available_and_total_memory_from_meminfo(self): meminfo = "MemTotal: 16387072 kB\nMemFree: 981236 kB\nMemAvailable: 4096000 kB\n" self.assertEqual(memory.parsed(meminfo), memory.Memory(available=4096000, total=16387072)) + + def test_reads_nothing_from_a_meminfo_missing_a_figure_or_holding_a_bad_one(self): for unread in ("MemTotal: 16 kB\n", "MemTotal: x kB\nMemAvailable: 1 kB\n", ""): - self.assertIsNone(memory.parsed(unread), unread) + with self.subTest(unread=unread): + self.assertIsNone(memory.parsed(unread)) def test_reads_a_meminfo_file_and_none_where_it_cannot(self): with tempfile.TemporaryDirectory() as temp: diff --git a/bin/rule_hook/test_rule_hook.py b/bin/rule_hook/test_rule_hook.py index 98f3086e..4bab0f0f 100644 --- a/bin/rule_hook/test_rule_hook.py +++ b/bin/rule_hook/test_rule_hook.py @@ -843,7 +843,8 @@ class Hook(unittest.TestCase): def test_help_lists_every_rule(self): helped = subprocess.run([str(HOOK), "--help"], capture_output=True, check=True, env=isolated_env(), text=True).stdout for rule in ["action-pin", "commit-one-line", "docker-run-memory", "ignorefile-sorted", "image-pin", "no-ai-attribution", "repeat-read", "reviewer-report", "reviewer-rung", "round-cap", "round-title-counts", "run-heavy", "todo-rank-check"]: - self.assertIn(f" {rule}\n The rule: ", helped) + with self.subTest(rule): + self.assertIn(f" {rule}\n The rule: ", helped) self.assertIn("reviewer-report\n The rule: a reviewer writes nothing but its report", helped) self.assertRegex(helped, r" reviewer-report\n( .*\n){2} Takes no override\.\n") self.assertRegex(helped, r" image-pin\n( .*\n){2} \S", "a rule a marker waives says nothing of overrides") diff --git a/bin/subagent_rung/test_subagent_rung.py b/bin/subagent_rung/test_subagent_rung.py index e5c83e46..e6628345 100644 --- a/bin/subagent_rung/test_subagent_rung.py +++ b/bin/subagent_rung/test_subagent_rung.py @@ -31,20 +31,29 @@ class SubagentRung(unittest.TestCase): def test_refuses_an_l_outside_1_to_4_or_a_wrong_count_of_arguments(self): for argv in (["claude-opus-5-5", "0"], ["claude-opus-5-5", "01"], ["claude-opus-5-5", "5"], ["claude-opus-5-5", "x"], ["claude-opus-5-5"], ["3", "claude-opus-5-5"], ["claude-opus-5-5", "3", "--plan"], ["--planning", "claude-opus-5-5", "3"]): - result = self.run_tool(*argv) - self.assertEqual((result.returncode, result.stdout), (1, ""), argv) - self.assertIn("usage: subagent-rung [--plan] ", result.stderr) + with self.subTest(argv=argv): + result = self.run_tool(*argv) + self.assertEqual((result.returncode, result.stdout), (1, "")) + self.assertIn("usage: subagent-rung [--plan] ", result.stderr) - def test_the_launcher_prints_the_rung_and_exits_0_on_and_off_the_ladder(self): + def test_the_launcher_prints_the_rung_and_exits_0_on_the_ladder(self): on = self.run_tool("claude-opus-5-5", "4") self.assertEqual((on.returncode, on.stdout, on.stderr), (0, "model opus, effort high\n", "")) + + def test_the_launcher_prints_nothing_and_says_why_and_exits_0_off_the_ladder(self): off = self.run_tool("claude-opus-5", "4") self.assertEqual((off.returncode, off.stdout), (0, "")) self.assertEqual(off.stderr, "subagent-rung: claude-opus-5 is not on the routing ladder; pass the Agent tool no model or effort\n") + + def test_the_launcher_accepts_the_bounds_of_l(self): for bound, rung in (("1", "model sonnet, effort high\n"), ("4", "model opus, effort high\n")): - self.assertEqual(self.run_tool("claude-opus-5-5", bound).stdout, rung) + with self.subTest(bound=bound): + self.assertEqual(self.run_tool("claude-opus-5-5", bound).stdout, rung) + + def test_the_launcher_prints_the_usage_for_each_help_flag(self): for flag in ("-h", "--help"): - self.assertIn("usage: subagent-rung [--plan] ", self.run_tool(flag).stdout) + with self.subTest(flag=flag): + self.assertIn("usage: subagent-rung [--plan] ", self.run_tool(flag).stdout) if __name__ == "__main__": unittest.main() diff --git a/bin/test_settings_filter.py b/bin/test_settings_filter.py index ca140e6a..04160591 100644 --- a/bin/test_settings_filter.py +++ b/bin/test_settings_filter.py @@ -26,9 +26,10 @@ class Clean(unittest.TestCase): def test_a_file_that_holds_no_json_object_exits_1_naming_the_tool(self): for written in ("{", "[]"): - ran = subprocess.run([sys.executable, FILTER], input=written, capture_output=True, text=True) - self.assertEqual((ran.returncode, ran.stdout), (1, ""), written) - self.assertIn("settings_filter.py: ", ran.stderr) + with self.subTest(written=written): + ran = subprocess.run([sys.executable, FILTER], input=written, capture_output=True, text=True) + self.assertEqual((ran.returncode, ran.stdout), (1, "")) + self.assertIn("settings_filter.py: ", ran.stderr) class Checkout(unittest.TestCase): diff --git a/bin/test_statusline.py b/bin/test_statusline.py index 3d99af01..c38dd5d6 100644 --- a/bin/test_statusline.py +++ b/bin/test_statusline.py @@ -100,8 +100,9 @@ class Script(unittest.TestCase): def test_warns_on_the_status_line_for_input_that_is_not_a_json_object(self): for stdin in ("not json", "[]"): - ran = self.run_script(stdin) - self.assertEqual((ran.returncode, ran.stdout, ran.stderr), (0, f"{RED}warning: statusline.py's input is not a JSON object{RESET}", "")) + with self.subTest(stdin=stdin): + ran = self.run_script(stdin) + self.assertEqual((ran.returncode, ran.stdout, ran.stderr), (0, f"{RED}warning: statusline.py's input is not a JSON object{RESET}", "")) def run_on_branch(self, branch): with tempfile.TemporaryDirectory() as checkout: diff --git a/todo.md b/todo.md index 55991022..d2c8b361 100644 --- a/todo.md +++ b/todo.md @@ -13,7 +13,6 @@ | ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score | |---|---|---|---|---|---|---|---|---|---|---|---| | 1 | 0.7.0 | | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 1 | 2 | 1 | 2 | 1 | | 0.0 | -| 242 | 0.7.0 | | principle | **Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case.** | 2 | 1 | 1 | 2 | 3 | code health by 2 | 3.6 | | 113 | 0.7.0 | | | **Compute an item's S from the code it names, where it names any, and show it beside the finished items `score-outcomes` lists.** | 3 | 2 | 2 | 5 | 4 | 1.1 by 5, 1.4 by 5 | 3.4 | | 263 | 0.7.0 | | | **Have the judge reject an interrupt that re-asks what the human answered earlier in the chunk, unless the ask quotes what that answer left open.** | 2 | 2 | 3 | 5 | 3 | 1.2 by 6, 1.5 by 2 | 3.2 | | 162 | 0.7.0 | | owner | **Script each step that 0.2.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.** | 2 | 2 | 1 | 3 | 3 | 1.4 by 4, 1.5 by 4 | 3.1 | @@ -72,10 +71,6 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost The README line covers the bench's transcripts in `workflow-bench/logs/` too (goal 3). -### 242. Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case. - -"Tests" in `principles/technical-principles.md`. The principles audit of PR #380 found test bodies that loop over their cases without `subTest`, some checking several behaviours at once: `bin/judge_replay/test_judge_replay.py:38`, `:139`, `:176`; `bin/loop_claude/test_loop_claude.py:590`; `bin/machine_load/test_machine_load.py:19`, `:31`; `bin/rule_hook/test_rule_hook.py:480`; `bin/subagent_rung/test_subagent_rung.py:32`, `:43`, `:45`; `bin/test_settings_filter.py:28`; `bin/test_statusline.py:102`. The first failing case stops the loop, and without a message the failure doesn't say which case broke. Split the tests that check several behaviours at once, `test_judge_replay.py:176` and `test_subagent_rung.py:43-46`, into one test per behaviour ("One behaviour per test"). - ### 113. Compute an item's S from the code it names, where it names any, and show it beside the finished items `score-outcomes` lists. `score-outcomes` grounds L, E, I and S in finished chunks only. Where an item names files or units, their size bounds what the change can touch.