Files
claude-config/bin/loop_claude/loop/test_usage.py
T

123 lines
8.9 KiB
Python

import ast
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
from claude_code.output_limits import BASH_CEILING_CHARACTERS
from chunk_driver import chunk_run
from agent_cycle import judge, retry, states, worker
from loop_claude.loop import main, usage
from loop_effects.effects import PYTHON_TIMEOUT
from chunk_driver.project import records
from claude_code import reply
from routing_figures import state
class HelpLimits(unittest.TestCase):
def test_help_wraps_at_100_columns(self):
self.assertEqual([line for line in usage.HELP.splitlines() if len(line) > 100], [])
def test_help_states_the_limits_the_code_holds(self):
said = " ".join(usage.HELP.split())
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
# The retry messages say these limits in words too.
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
self.assertIn("fails three times running, retried a minute apart", said)
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
self.assertIn(f"still running after {PYTHON_TIMEOUT} s", said)
self.assertIn(f"Once {state.CYCLES_PER_CHECK} cycles have closed across all projects", said)
self.assertIn(f"run with a {worker.COMPACT_WINDOW // 1000}k auto-compact window", said)
self.assertIn(f"{BASH_CEILING_CHARACTERS:,} characters", said)
self.assertIn(f"unfinished {chunk_run.UNFINISHED_LIMIT} cycles running", said)
self.assertIn(f"rerun to give it {chunk_run.UNFINISHED_LIMIT} more", said)
class ExitContract(unittest.TestCase):
def test_every_exit_1_stop_has_its_clause_in_help(self):
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "no config names the checkout, its config cannot be read, or the goals file holds no premises"
models_ = "the models file cannot be read or written"
names = (*records.RECORD_FILES, records.HUMAN_RULE_NAMES[0], f"{records.HUMAN_RULE_NAMES[1]}*.md", records.HUMAN_RULE_NAMES[2])
unreadable = f"the goals file, or {', '.join(names[:-1])} or {names[-1]} where they exist, cannot be read or is not UTF-8"
# Every Stop whose code is not a literal 0, 2 or 130, in source order: path, function, code, start of reason, its --help clause.
clauses = [
("agent_cycle/interrupt.py", "claude_open", "1", "f'could not start claude for the interrupt:", "claude cannot start for an interrupt"),
("agent_cycle/interrupt.py", "human_typed", "1", "entries.reason", read),
("agent_cycle/interrupt.py", "human_typed", "1", "f'could not tell whether you typed anything:", read),
("agent_cycle/retry.py", "retried", "1", "f'the {who} failed three times running:", "the worker or the judge fails three times running"),
("agent_cycle/run.py", "archived_stop", "1", "f'{failure.reason}", read),
("agent_cycle/run.py", "work", "1", "f'claude left no transcript for session", "claude leaves no transcript"),
("agent_cycle/run.py", "rule", "1", "entries.reason", read),
("agent_cycle/run.py", "rule", "1", "annotation.reason", read),
("agent_cycle/run.py", "rule", "1", "failure.reason", read),
("agent_cycle/run.py", "keep_rejection", "1", "unkept.reason", read),
("agent_cycle/run.py", "close", "1", "annotation.reason", read),
("agent_cycle/run.py", "close", "1", "failure.reason", read),
("agent_cycle/run.py", "close", "1", "failure.reason", "a finished chunk's session cannot be deleted"),
("chunk_driver/chunk_run.py", "closed_cycle", "1", "failure.reason", read),
("chunk_driver/cycle_start.py", "project_routing", "1", "ladder_file.reason", premises_),
("chunk_driver/cycle_start.py", "routing_on", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"),
("chunk_driver/project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
("chunk_driver/project/model_choice.py", "get_or_ask_model", "1", "failure.reason", models_),
("loop_claude/loop/checkout.py", "checkout_root", "1", "found.reason", git),
("loop_claude/loop/main.py", "run_loop", "1", "reason", "tmux is missing or older than 3.2"),
("loop_claude/loop/main.py", "run_loop", "1", "f'{CHUNK_FLAG} is how loop-claude starts itself", "--chunk is given outside a tmux window"),
("loop_claude/loop/main.py", "windows_opened", "1", "started.reason", "a chunk's window cannot be started"),
("loop_claude/loop/main.py", "windows_opened", "1", "f\"`tmux {' '.join(shown)}` failed", "the tmux session cannot be put on the terminal"),
("loop_claude/loop/main.py", "handed_on", "1", "f'the chunk finished, but the next could not start", "a chunk's window cannot be started"),
("loop_claude/loop/main.py", "next_cycle", "1", "premises.reason", premises_),
("loop_claude/loop/main.py", "next_cycle", "1", "records.reason", unreadable),
("loop_claude/loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
("loop_claude/loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
("loop_claude/loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"),
("loop_claude/loop/routing_check.py", "raise_calls", "1", "f\"could not start claude for the routing calls in {loop.paths.config_clone}", "claude cannot start for an interrupt"),
]
root = pathlib.Path(main.__file__).parents[2]
found = []
for path in sorted(path for package in ("agent_cycle", "chunk_driver", "loop_claude") for path in (root / package).rglob("*.py") if not path.name.startswith("test_")):
functions = [node for node in ast.walk(ast.parse(path.read_text())) if isinstance(node, ast.FunctionDef)]
stops = [(call.lineno, function.name, ast.unparse(call.args[0]), ast.unparse(call.args[1])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Stop" and not (isinstance(call.args[0], ast.Constant) and call.args[0].value in (0, 2, 130))]
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
self.assertEqual(len(found), len(clauses), found)
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
with self.subTest(path=path, function=function):
self.assertEqual((path, function, code), (want_path, want_function, want_code))
self.assertTrue(reason.startswith(reason_start), reason)
help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split())
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
class JudgeFailures(unittest.TestCase):
def test_every_failed_judge_run_has_its_clause_in_help(self):
started, exited, unparsed = "claude cannot start", "exits non-zero", "prints no json reply"
field, unasked = "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
# Every Failure the judge's run returns, in source order of judge.py, then of the claude run it calls: function, start of reason, its --help clause.
clauses = [
("verdict_from", "'the judge returned no reply'", field),
("verdict_from", 'f"the judge\'s reply has no {name}', field),
("verdict_from", "'the judge interrupted without a question'", unasked),
("verdict_from", "'the judge quoted an answer without a question'", unasked),
("run_judge", "verdict.reason", field),
("run_judge", "checked.reason", "quotes a passage that neither the task nor the transcript holds"),
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the task or the transcript", "quotes a passage that neither the task nor the transcript holds"),
("claude_reply", "f'could not start claude:", started),
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
("claude_reply", 'f"{who} printed no json reply', unparsed),
]
found = []
for module in (judge, reply):
functions = [node for node in ast.walk(ast.parse(pathlib.Path(module.__file__).read_text())) if isinstance(node, ast.FunctionDef)]
found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected"))
self.assertEqual(len(found), len(clauses), found)
for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses):
with self.subTest(function=function):
self.assertEqual(function, want_function)
self.assertTrue(reason.startswith(reason_start), reason)
judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split())
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
if __name__ == "__main__":
unittest.main()