123 lines
8.9 KiB
Python
123 lines
8.9 KiB
Python
import ast
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
|
|
from claude_code.output_limits import BASH_CEILING_CHARACTERS
|
|
from chunk_driver import chunk_run
|
|
from agent_cycle import judge, retry, states, worker
|
|
from loop_claude.loop import main, usage
|
|
from loop_effects.effects import PYTHON_TIMEOUT
|
|
from chunk_driver.project import records
|
|
from claude_code import reply
|
|
from routing_figures import state
|
|
|
|
|
|
class HelpLimits(unittest.TestCase):
|
|
def test_help_wraps_at_100_columns(self):
|
|
self.assertEqual([line for line in usage.HELP.splitlines() if len(line) > 100], [])
|
|
|
|
def test_help_states_the_limits_the_code_holds(self):
|
|
said = " ".join(usage.HELP.split())
|
|
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
|
|
# The retry messages say these limits in words too.
|
|
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
|
|
self.assertIn("fails three times running, retried a minute apart", said)
|
|
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
|
|
self.assertIn(f"still running after {PYTHON_TIMEOUT} s", said)
|
|
self.assertIn(f"Once {state.CYCLES_PER_CHECK} cycles have closed across all projects", said)
|
|
self.assertIn(f"run with a {worker.COMPACT_WINDOW // 1000}k auto-compact window", said)
|
|
self.assertIn(f"{BASH_CEILING_CHARACTERS:,} characters", said)
|
|
self.assertIn(f"unfinished {chunk_run.UNFINISHED_LIMIT} cycles running", said)
|
|
self.assertIn(f"rerun to give it {chunk_run.UNFINISHED_LIMIT} more", said)
|
|
|
|
|
|
class ExitContract(unittest.TestCase):
|
|
def test_every_exit_1_stop_has_its_clause_in_help(self):
|
|
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "no config names the checkout, its config cannot be read, or the goals file holds no premises"
|
|
models_ = "the models file cannot be read or written"
|
|
names = (*records.RECORD_FILES, records.HUMAN_RULE_NAMES[0], f"{records.HUMAN_RULE_NAMES[1]}*.md", records.HUMAN_RULE_NAMES[2])
|
|
unreadable = f"the goals file, or {', '.join(names[:-1])} or {names[-1]} where they exist, cannot be read or is not UTF-8"
|
|
# Every Stop whose code is not a literal 0, 2 or 130, in source order: path, function, code, start of reason, its --help clause.
|
|
clauses = [
|
|
("agent_cycle/interrupt.py", "claude_open", "1", "f'could not start claude for the interrupt:", "claude cannot start for an interrupt"),
|
|
("agent_cycle/interrupt.py", "human_typed", "1", "entries.reason", read),
|
|
("agent_cycle/interrupt.py", "human_typed", "1", "f'could not tell whether you typed anything:", read),
|
|
("agent_cycle/retry.py", "retried", "1", "f'the {who} failed three times running:", "the worker or the judge fails three times running"),
|
|
("agent_cycle/run.py", "archived_stop", "1", "f'{failure.reason}", read),
|
|
("agent_cycle/run.py", "work", "1", "f'claude left no transcript for session", "claude leaves no transcript"),
|
|
("agent_cycle/run.py", "rule", "1", "entries.reason", read),
|
|
("agent_cycle/run.py", "rule", "1", "annotation.reason", read),
|
|
("agent_cycle/run.py", "rule", "1", "failure.reason", read),
|
|
("agent_cycle/run.py", "keep_rejection", "1", "unkept.reason", read),
|
|
("agent_cycle/run.py", "close", "1", "annotation.reason", read),
|
|
("agent_cycle/run.py", "close", "1", "failure.reason", read),
|
|
("agent_cycle/run.py", "close", "1", "failure.reason", "a finished chunk's session cannot be deleted"),
|
|
("chunk_driver/chunk_run.py", "closed_cycle", "1", "failure.reason", read),
|
|
("chunk_driver/cycle_start.py", "project_routing", "1", "ladder_file.reason", premises_),
|
|
("chunk_driver/cycle_start.py", "routing_on", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"),
|
|
("chunk_driver/project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
|
|
("chunk_driver/project/model_choice.py", "get_or_ask_model", "1", "failure.reason", models_),
|
|
("loop_claude/loop/checkout.py", "checkout_root", "1", "found.reason", git),
|
|
("loop_claude/loop/main.py", "run_loop", "1", "reason", "tmux is missing or older than 3.2"),
|
|
("loop_claude/loop/main.py", "run_loop", "1", "f'{CHUNK_FLAG} is how loop-claude starts itself", "--chunk is given outside a tmux window"),
|
|
("loop_claude/loop/main.py", "windows_opened", "1", "started.reason", "a chunk's window cannot be started"),
|
|
("loop_claude/loop/main.py", "windows_opened", "1", "f\"`tmux {' '.join(shown)}` failed", "the tmux session cannot be put on the terminal"),
|
|
("loop_claude/loop/main.py", "handed_on", "1", "f'the chunk finished, but the next could not start", "a chunk's window cannot be started"),
|
|
("loop_claude/loop/main.py", "next_cycle", "1", "premises.reason", premises_),
|
|
("loop_claude/loop/main.py", "next_cycle", "1", "records.reason", unreadable),
|
|
("loop_claude/loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
|
|
("loop_claude/loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
|
|
("loop_claude/loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"),
|
|
("loop_claude/loop/routing_check.py", "raise_calls", "1", "f\"could not start claude for the routing calls in {loop.paths.config_clone}", "claude cannot start for an interrupt"),
|
|
]
|
|
root = pathlib.Path(main.__file__).parents[2]
|
|
found = []
|
|
for path in sorted(path for package in ("agent_cycle", "chunk_driver", "loop_claude") for path in (root / package).rglob("*.py") if not path.name.startswith("test_")):
|
|
functions = [node for node in ast.walk(ast.parse(path.read_text())) if isinstance(node, ast.FunctionDef)]
|
|
stops = [(call.lineno, function.name, ast.unparse(call.args[0]), ast.unparse(call.args[1])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Stop" and not (isinstance(call.args[0], ast.Constant) and call.args[0].value in (0, 2, 130))]
|
|
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
|
|
self.assertEqual(len(found), len(clauses), found)
|
|
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
|
|
with self.subTest(path=path, function=function):
|
|
self.assertEqual((path, function, code), (want_path, want_function, want_code))
|
|
self.assertTrue(reason.startswith(reason_start), reason)
|
|
help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split())
|
|
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
|
|
|
|
|
|
class JudgeFailures(unittest.TestCase):
|
|
def test_every_failed_judge_run_has_its_clause_in_help(self):
|
|
started, exited, unparsed = "claude cannot start", "exits non-zero", "prints no json reply"
|
|
field, unasked = "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
|
|
# Every Failure the judge's run returns, in source order of judge.py, then of the claude run it calls: function, start of reason, its --help clause.
|
|
clauses = [
|
|
("verdict_from", "'the judge returned no reply'", field),
|
|
("verdict_from", 'f"the judge\'s reply has no {name}', field),
|
|
("verdict_from", "'the judge interrupted without a question'", unasked),
|
|
("verdict_from", "'the judge quoted an answer without a question'", unasked),
|
|
("run_judge", "verdict.reason", field),
|
|
("run_judge", "checked.reason", "quotes a passage the transcript does not hold"),
|
|
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
|
|
("claude_reply", "f'could not start claude:", started),
|
|
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
|
|
("claude_reply", 'f"{who} printed no json reply', unparsed),
|
|
]
|
|
found = []
|
|
for module in (judge, reply):
|
|
functions = [node for node in ast.walk(ast.parse(pathlib.Path(module.__file__).read_text())) if isinstance(node, ast.FunctionDef)]
|
|
found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected"))
|
|
self.assertEqual(len(found), len(clauses), found)
|
|
for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses):
|
|
with self.subTest(function=function):
|
|
self.assertEqual(function, want_function)
|
|
self.assertTrue(reason.startswith(reason_start), reason)
|
|
judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split())
|
|
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|