Answer stability round 1: 1 🔴, 5 🟡 (0 misread); catch a broken HTTP answer, drop typed keys before the DM, DM every stop but Ctrl-C and a crash, inject home and http, and test the gaps

This commit is contained in:
2026-11-11 22:37:52 +01:00
parent f10170bf8b
commit 23af106b47
11 changed files with 131 additions and 46 deletions
+2 -2
View File
@@ -69,7 +69,7 @@ Then pick a model with `/model`.
Link `loop-claude` and `loop-inbox` onto `PATH`: `ln -s ~/.claude/bin/loop-claude ~/.local/bin/loop-claude` and `ln -s ~/.claude/bin/loop-inbox ~/.local/bin/loop-inbox`. Every claude they start, the interactive ones included, skips every permission prompt.
To get loop-claude's questions and stops as Mattermost DMs from a bot, write the server, the bot's token and your username into `~/.claude/.env`, in the keys `bin/mattermost-dm --help` names, and run `chmod 600 ~/.claude/.env`. Without that file, loop-claude sends no DM.
To get loop-claude's questions and stops as Mattermost DMs from a bot, write the server, the bot's token and your username into `~/.claude/.env`, in the keys `~/.claude/bin/mattermost-dm --help` names, and run `chmod 600 ~/.claude/.env`. Without that file, loop-claude sends no DM.
## Update
@@ -105,7 +105,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t
- `larv-review <command>` holds larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's round count, the scoring run's trigger; the comprehension history's format, its latest places and its count of how often seats agree; and each reviewer's prompt header. `larv-review --help` states them.
- `loop-claude <task>` runs a project's next chunk cycle after cycle; `loop-claude --help` states its contract.
- `loop-inbox` shows the interrupts of every loop-claude on the machine, oldest first, takes each one's answer, `/talk` included, and lists every running loop with the `todo.md` items its chunk takes; `loop-inbox --help` states how.
- `mattermost-dm [<message>]` sends a message, or else stdin, from the Mattermost bot to the human, configured in `~/.claude/.env`; `loop-claude` sends one for each question and each stop with exit 1, and `mattermost-dm --help` names the file's keys.
- `mattermost-dm [<message>]` sends a message, or else stdin, from the Mattermost bot to the human; `~/.claude/.env` names the server, the bot's token and the human, and `mattermost-dm --help` names its keys. `loop-claude --help` says when loop-claude sends one.
- `metric-bench [history]` benches cheap static metrics of a project's Python against the places its comprehension history names as hardest; `metric-bench --help` states the metrics and the scores.
- `mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]` mutation-tests the lines a branch changes and prints each mutant its tests don't kill; `mutate-changes --help` says which tool it runs for each language.
- `rank_todo.py [--check] [todo.md]` scores and ranks a project's `todo.md`; `rank_todo.py --help` states the format.
+6 -4
View File
@@ -43,12 +43,14 @@ class Talked:
def interrupt(question: str, answer_here: bool, session: WorkerSession, questions: pathlib.Path, effects: Effects) -> str | Talked | Stop:
"""Prints question for the human and returns their answer: where answer_here, the first line typed at loop-claude's terminal or in loop-inbox, stripped, or Talked once they typed in the worker's claude session, which TALK opens; else Talked once they typed in that session, which opens at once on the terminal. loop-inbox lists the question meanwhile. A Stop where loop-claude is off a terminal, or they type nothing, or the input ends."""
say(f"\n{question}", effects)
# Before the terminal check, so a loop that cannot ask still tells the human what it would have asked.
on_terminal = effects.on_terminal()
if on_terminal:
# Keys typed while the worker ran wait on the terminal, and would answer a question no one has read, here or in claude's session, which reads them too. Of a pasted answer, the first line answers; the rest wait, and the next question drops them. A longer answer goes through TALK.
effects.discard_typed()
# After the discard, so an answer typed while the DM sends counts; before the terminal check, so a loop that cannot ask still tells the human what it would have asked.
tell(f"loop-claude in {session.root} asks:\n\n{question}", effects)
if stop := needs_terminal(ANSWER_QUESTION, effects.on_terminal()):
if stop := needs_terminal(ANSWER_QUESTION, on_terminal):
return stop
# Keys typed while the worker ran wait on the terminal, and would answer a question no one has read, here or in claude's session, which reads them too. Of a pasted answer, the first line answers; the rest wait, and the next question drops them. A longer answer goes through TALK.
effects.discard_typed()
ask = Ask(effects.new_id(), str(session.id), session.root, question, effects.now(), Asking(session.claude_command()) if answer_here else OpenOnTerminal())
with listed(questions, ask, effects):
answer = first_answer(ask, questions, effects) if answer_here else OpenOnTerminal()
+1 -1
View File
@@ -2,6 +2,6 @@ from loop_effects.effects import Effects, say
def tell(text: str, effects: Effects) -> None:
"""DMs text to the human, saying on the terminal why where the DM fails; the DM only copies what the terminal shows, so its failure never stops the loop."""
"""DMs text to the human. Where the DM fails, it says why on stderr and goes on: the DM only copies what the terminal shows, so its failure never stops the loop."""
if failed := effects.dm(text):
say(f"could not DM you on Mattermost: {failed.reason}", effects)
+5 -2
View File
@@ -31,10 +31,13 @@ def main(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launcher:
stop = run_loop(argv, env, paths, launcher, cwd, effects)
except KeyboardInterrupt:
stop = Stop(130, "")
except Exception as crash:
tell(f"loop-claude in {cwd} crashed: {crash!r}", effects)
raise
if stop.reason:
say(stop.reason, effects, stdout=not stop.code)
# Exit 2 is a stop the human chose or must answer at the terminal, and 130 is Ctrl-C; a loop given no task stops before it runs.
if stop.code == 1 and argv:
# Every stop but the human's own Ctrl-C and the help is DMed, so a loop left overnight never ends unseen. A loop given no task prints the help.
if stop.code != 130 and stop.reason != HELP:
tell(f"loop-claude in {cwd} stopped: {stop.reason}", effects)
return stop.code
+2
View File
@@ -14,6 +14,7 @@ from routing_figures.render import render
from model_routing.ladder import Rung
from routing_figures import state
from loop_claude.stop import Stop
from loop_claude.direct_message import tell
from records.utc_stamp import utc
@@ -54,6 +55,7 @@ def raise_calls(root: pathlib.Path, new: list[Call], rung: Rung, loop: Loop) ->
bullets = "".join(f"- {call.sentence()}\n" for call in new)
question = f"loop-claude's routing figures call for a routing change:\n{bullets}The figures: {root / archive.FIGURES}"
say(f"\n{question}", loop.effects)
tell(f"loop-claude in {loop.root} asks:\n\n{question}", loop.effects)
if not loop.effects.on_terminal():
say("Not on a terminal: the calls are raised again at the next measurement.", loop.effects)
return None
+3 -3
View File
@@ -33,9 +33,9 @@ interrupts of every loop on the machine. The first answer counts, from either pl
- Enter alone, on loop-claude's terminal: stops the loop.
A release ready to cut opens that session on the terminal at once, since only the human may cut it;
loop-inbox lists it too. While claude is open, Ctrl-C reaches it alone, and the loop goes on.
Where ~/.claude/.env exists, each question, and each stop with exit 1, also goes to the human as a
Mattermost DM, as bin/mattermost-dm sends it; a DM that fails is said on the terminal, and the loop
goes on.
Where ~/.claude/.env exists, loop-claude also sends the human each question, each release ready
to cut and each stop as a Mattermost DM, as bin/mattermost-dm sends it. Two stops are left out:
Ctrl-C, and printing this help. A DM that fails is said on stderr, and the loop goes on.
While it runs, loop-claude also lists itself in loop-inbox, with the todo.md items its chunk takes,
as bin/loop_state/running_loops.py describes.
+35 -6
View File
@@ -94,7 +94,7 @@ class FakeEffects:
def __init__(self, claude=(), on_terminal_claude=(), git=(), inputs=(), python=(), streams=(), on_terminal=True, free_memory=(), dm_failures=()):
self.calls = []
# Each direct message sent, apart from calls, so tests that list every call need not list them. Without a script, each is sent.
# Each direct message sent. They stay out of self.calls, so a test that lists every call doesn't list them. Without a script, each is sent.
self.dms = []
self.dm_failures = list(dm_failures)
self.scripts = {"claude": list(claude), "claude_on_terminal": list(on_terminal_claude), "git": list(git), "input": list(inputs), "python": list(python), "stream_claude": list(streams)}
@@ -735,13 +735,20 @@ class Interrupts(unittest.TestCase):
self.assertEqual(self.put(fake), (Stop(2, "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"), f"\n{STAMP} Which one?\n"))
self.assertEqual(fake.calls, [])
def test_dms_the_question_naming_the_project_before_asking_on_a_terminal_or_off_one(self):
def test_dms_the_question_naming_the_project_on_a_terminal_or_off_one(self):
for on_terminal in (True, False):
with self.subTest(on_terminal=on_terminal):
fake = FakeEffects(inputs=["Use B."], on_terminal=on_terminal)
self.put(fake)
self.assertEqual(fake.dms, [f"loop-claude in {self.home} asks:\n\nWhich one?"])
def test_drops_the_keys_typed_before_the_question_ahead_of_the_dm_so_an_answer_typed_while_it_sends_counts(self):
fake = FakeEffects(inputs=["Use B."])
seen = []
effects = dataclasses.replace(fake.effects(), dm=lambda text: seen.append(list(fake.calls)))
self.assertEqual(self.put(fake, effects=effects)[0], "Use B.")
self.assertEqual(seen, [[("discard_typed", None)]])
def test_says_why_a_dm_failed_and_still_takes_the_answer(self):
fake = FakeEffects(inputs=["Use B."], dm_failures=[Failure("cannot reach https://chat.example: timed out")])
self.assertEqual(self.put(fake), ("Use B.", f"\n{STAMP} Which one?\n{STAMP} could not DM you on Mattermost: cannot reach https://chat.example: timed out\n"))
@@ -983,6 +990,8 @@ class RoutingCheck(unittest.TestCase):
self.assertIn(f"change nothing until they answer:\n- {new[0].sentence()}\n- {new[1].sentence()}\nThe figures", args[6])
self.assertIn(f"call for a routing change:\n- {new[0].sentence()}\n- {new[1].sentence()}\n", said.getvalue())
self.assertIn("routing figures: full", said.getvalue())
[sent] = fake.dms
self.assertTrue(sent.startswith(f"loop-claude in {self.loop(fake).root} asks:\n\nloop-claude's routing figures call for a routing change:\n- {new[0].sentence()}\n"), sent)
class ModelPrompt(unittest.TestCase):
@@ -1812,6 +1821,24 @@ class StopDms(unittest.TestCase):
code = main.main(argv, {}, paths.paths_from(pathlib.Path(home)), pathlib.Path(home, "loop-claude"), CWD, fake.effects())
return code, said.getvalue()
def test_dms_the_reason_of_a_stop_with_exit_2_such_as_a_chunk_left_unfinished_too_often(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", return_value=Stop(2, "the human is needed to look at the chunk, but it stayed unfinished")):
self.assertEqual(self.run_main(["do it"], fake)[0], 2)
self.assertEqual(fake.dms, [f"loop-claude in {CWD} stopped: the human is needed to look at the chunk, but it stayed unfinished"])
def test_skips_the_dm_where_ctrl_c_stopped_the_loop(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", side_effect=KeyboardInterrupt):
self.assertEqual(self.run_main(["do it"], fake)[0], 130)
self.assertEqual(fake.dms, [])
def test_dms_a_crash_before_it_reaches_the_terminal(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", side_effect=RuntimeError("boom")), self.assertRaises(RuntimeError):
self.run_main(["do it"], fake)
self.assertEqual(fake.dms, [f"loop-claude in {CWD} crashed: RuntimeError('boom')"])
def test_dms_the_reason_where_a_loop_given_a_task_stops_with_exit_1(self):
fake = FakeEffects()
code, said = self.run_main([main.RELOAD_FLAG, "3"], fake)
@@ -1822,10 +1849,12 @@ class StopDms(unittest.TestCase):
fake = FakeEffects(dm_failures=[Failure("cannot reach https://chat.example: timed out")])
self.assertTrue(self.run_main([main.RELOAD_FLAG, "3"], fake)[1].endswith(f"Handed: 3\n{STAMP} could not DM you on Mattermost: cannot reach https://chat.example: timed out\n"))
def test_sends_nothing_where_no_task_is_given(self):
fake = FakeEffects()
self.assertEqual(self.run_main([], fake)[0], 1)
self.assertEqual(fake.dms, [])
def test_skips_the_dm_where_it_prints_its_help(self):
for argv, code in (([], 1), (["--help"], 0)):
with self.subTest(argv=argv):
fake = FakeEffects()
self.assertEqual(self.run_main(argv, fake)[0], code)
self.assertEqual(fake.dms, [])
class HelpLimits(unittest.TestCase):
+3 -2
View File
@@ -4,6 +4,7 @@ A change here, such as to how input reads a line, changes every tool that runs o
"""
import dataclasses
import os
import pathlib
import select
import signal
import subprocess
@@ -13,7 +14,7 @@ import time
import typing
import uuid
from mattermost_dm.dm import dm
from mattermost_dm.dm import dm, http_json
from failure import Failure
from git_cli import git_repo
from machine_load.memory import Memory, read_memory
@@ -84,7 +85,7 @@ def system_effects() -> Effects:
claude=lambda args, cwd, stdin: subprocess.run(["claude", *args], capture_output=True, cwd=cwd, input=stdin, text=True),
claude_on_terminal=claude_on_terminal,
discard_typed=lambda: termios.tcflush(sys.stdin, termios.TCIFLUSH),
dm=dm,
dm=lambda text: dm(text, pathlib.Path.home(), http_json),
execv=os.execv,
git=git_repo.git,
input=input,
+11 -8
View File
@@ -1,5 +1,6 @@
"""Sends a direct message from a Mattermost bot to one user, through Mattermost's REST API v4."""
import dataclasses
import http.client
import json
import pathlib
import typing
@@ -10,15 +11,15 @@ import urllib.request
from failure import Failure
KEYS = ("MATTERMOST_URL", "MATTERMOST_TOKEN", "MATTERMOST_RECIPIENT")
# Seconds a request may take, so an unreachable server holds loop-claude's interrupt only briefly.
# Seconds each connect and each read may wait, so an unreachable server holds loop-claude's interrupt only briefly.
TIMEOUT = 10
# Sends a request, the body as JSON where there is one, and returns the JSON answer.
# Sends a request, the body as JSON where there is one, and returns the JSON answer, or a Failure saying why there is none.
Http = typing.Callable[[str, str, str, object], object]
class NoRedirect(urllib.request.HTTPRedirectHandler):
"""Follows no redirect: urllib resends the token to the new location, even one off https, and Mattermost's API answers without one."""
"""Refuses every redirect, which Mattermost's API never sends: urllib would resend the token to the new location, even one off https."""
def redirect_request(self, *args: typing.Any, **kwargs: typing.Any) -> None:
return None
@@ -94,7 +95,7 @@ def id_of(answer: object, request: str) -> str | Failure:
def http_json(method: str, url: str, token: str, body: object) -> object:
"""The JSON that url answers; a Failure naming the request where it cannot be reached, answers an error or answers something other than JSON."""
"""The JSON that url answers; a Failure naming the server where it cannot be reached, and the request where it answers an error, breaks HTTP or answers something other than JSON."""
request_name = f"{method} {urllib.parse.urlsplit(url).path}"
data = None if body is None else json.dumps(body).encode()
request = urllib.request.Request(url, data, {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, method=method)
@@ -104,6 +105,8 @@ def http_json(method: str, url: str, token: str, body: object) -> object:
except urllib.error.HTTPError as failure:
with failure:
return Failure(f"{request_name} answered {failure.code}: {error_message(failure.read()) or failure.reason}")
except http.client.HTTPException as failure:
return Failure(f"{request_name} got a broken answer: {failure!r}")
except (OSError, ValueError) as failure:
server = urllib.parse.urlunsplit(urllib.parse.urlsplit(url)[:2] + ("", "", ""))
return Failure(f"cannot reach {server}: {getattr(failure, 'reason', failure)}")
@@ -122,10 +125,10 @@ def error_message(raw: bytes) -> str:
return message if isinstance(message, str) else ""
def dm(text: str) -> Failure | None:
"""Sends text as configured in ~/.claude/.env; sends nothing on a machine without that file, which hasn't asked for DMs."""
path = env_file(pathlib.Path.home())
def dm(text: str, home: pathlib.Path, http: Http) -> Failure | None:
"""Sends text as home's .claude/.env configures it. A machine without that file hasn't asked for DMs, so there the DM is skipped."""
path = env_file(home)
if not path.exists():
return None
config = read_config(path)
return config if isinstance(config, Failure) else send(config, text, http_json)
return config if isinstance(config, Failure) else send(config, text, http)
+4 -6
View File
@@ -9,12 +9,10 @@ holds three KEY=value lines, and only its owner may read it, since it holds the
MATTERMOST_TOKEN=<the bot's access token>
MATTERMOST_RECIPIENT=<the recipient's username>
Where that file exists, loop-claude sends the human a DM when it asks a question, a release is
ready to cut, or the loop stops with exit 1.
Exit 0 once sent, 1, saying why, when the send failed, and 2 on misuse: bad arguments, an empty
message, or a ~/.claude/.env that is missing, lacks a key, names a server off https or that
others may read.
Exit 0 once the message is sent.
Exit 1, saying why, when Mattermost cannot be reached or refuses the message.
Exit 2, saying why, on misuse: bad arguments, an empty message, or a ~/.claude/.env that is
missing or unreadable, lacks a key, names a server off https, or that others may read.
"""
import typing
+59 -12
View File
@@ -2,6 +2,7 @@ import http.server
import io
import json
import pathlib
import socket
import subprocess
import sys
import tempfile
@@ -93,11 +94,18 @@ class Send(unittest.TestCase):
self.assertEqual(dm.send(CONFIG, "Which one?", server), Failure("GET /api/v4/users/username/lilleman answered 404: Unable to find an existing account matching your username."))
self.assertEqual(len(server.requests), 2)
def test_fails_on_an_answer_without_an_id(self):
def test_fails_on_an_answer_without_an_id_or_one_that_is_not_an_object(self):
for answer in ({"status": "OK"}, ["channel-id"], {"id": ""}):
with self.subTest(answer=answer):
server = mattermost()
server.answers[("POST", "/api/v4/channels/direct")] = answer
self.assertEqual(dm.send(CONFIG, "Which one?", server), Failure("POST /api/v4/channels/direct answered without an id"))
self.assertEqual(len(server.requests), 3)
def test_fails_where_the_post_itself_fails(self):
server = mattermost()
server.answers[("POST", "/api/v4/channels/direct")] = {"status": "OK"}
self.assertEqual(dm.send(CONFIG, "Which one?", server), Failure("POST /api/v4/channels/direct answered without an id"))
self.assertEqual(len(server.requests), 3)
server.answers[("POST", "/api/v4/posts")] = Failure("POST /api/v4/posts answered 403: You do not have the appropriate permissions.")
self.assertEqual(dm.send(CONFIG, "Which one?", server), Failure("POST /api/v4/posts answered 403: You do not have the appropriate permissions."))
class Recorded(http.server.BaseHTTPRequestHandler):
@@ -113,6 +121,11 @@ class Recorded(http.server.BaseHTTPRequestHandler):
length = int(self.headers.get("Content-Length") or 0)
self.server.got.append((self.command, self.path, dict(self.headers), self.rfile.read(length))) # type: ignore[attr-defined]
status, body = self.server.script.pop(0) # type: ignore[attr-defined]
if status is None:
# Raw bytes, as a server that breaks HTTP sends them.
self.wfile.write(body)
self.close_connection = True
return
self.send_response(status)
if 300 <= status < 400:
self.send_header("Location", "/elsewhere")
@@ -150,6 +163,28 @@ class HttpJson(unittest.TestCase):
self.assertEqual(dm.http_json("GET", f"{base}/api/v4/users/me", "t", None), Failure("GET /api/v4/users/me answered 302: Found"))
self.assertEqual(len(self.server.got), 1) # type: ignore[attr-defined]
def test_sends_no_body_with_a_get(self):
base = self.serve((200, b'{"id": "bot-id"}'))
dm.http_json("GET", f"{base}/api/v4/users/me", "t", None)
[(_, _, headers, body)] = self.server.got # type: ignore[attr-defined]
self.assertEqual((headers.get("Content-Length"), body), (None, b""))
def test_fails_on_an_answer_that_breaks_http(self):
for raw in (b"garbage\r\n\r\n", b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{}"):
with self.subTest(raw=raw):
base = self.serve((None, raw))
failed = dm.http_json("GET", f"{base}/api/v4/users/me", "t", None)
self.assertTrue(isinstance(failed, Failure) and failed.reason.startswith("GET /api/v4/users/me got a broken answer: "), failed)
def test_gives_up_on_a_server_that_never_answers_after_the_timeout(self):
silent = socket.socket()
self.addCleanup(silent.close)
silent.bind(("127.0.0.1", 0))
silent.listen(1)
base = f"http://127.0.0.1:{silent.getsockname()[1]}"
with unittest.mock.patch.object(dm, "TIMEOUT", 0.2):
self.assertEqual(dm.http_json("GET", f"{base}/api/v4/users/me", "t", None), Failure(f"cannot reach {base}: timed out"))
def test_fails_on_an_answer_that_is_not_json(self):
base = self.serve((200, b"<html>"))
self.assertEqual(dm.http_json("GET", f"{base}/api/v4/users/me", "t", None), Failure("GET /api/v4/users/me answered with something other than JSON"))
@@ -163,16 +198,28 @@ class HttpJson(unittest.TestCase):
class Dm(unittest.TestCase):
def test_sends_nothing_and_fails_nothing_on_a_machine_without_the_env_file(self):
with tempfile.TemporaryDirectory() as home, unittest.mock.patch.dict("os.environ", {"HOME": home}):
self.assertIsNone(dm.dm("Which one?"))
def home_with(self, home: str, text: str) -> pathlib.Path:
folder = pathlib.Path(home, ".claude")
folder.mkdir()
return env_file(str(folder), text)
def test_sends_through_the_env_file_under_home(self):
with tempfile.TemporaryDirectory() as home:
self.home_with(home, "MATTERMOST_URL=https://chat.example\nMATTERMOST_TOKEN=secret-token\nMATTERMOST_RECIPIENT=lilleman\n")
server = mattermost()
self.assertIsNone(dm.dm("Which one?", pathlib.Path(home), server))
self.assertEqual(server.requests[-1], ("POST", "https://chat.example/api/v4/posts", "secret-token", {"channel_id": "channel-id", "message": "Which one?"}))
def test_skips_the_dm_on_a_machine_without_the_env_file(self):
with tempfile.TemporaryDirectory() as home:
server = mattermost()
self.assertIsNone(dm.dm("Which one?", pathlib.Path(home), server))
self.assertEqual(server.requests, [])
def test_fails_naming_the_mistake_where_the_env_file_is_there_but_wrong(self):
with tempfile.TemporaryDirectory() as home, unittest.mock.patch.dict("os.environ", {"HOME": home}):
path = pathlib.Path(home, ".claude")
path.mkdir()
env_file(str(path), "MATTERMOST_URL=https://chat.example\n")
self.assertEqual(dm.dm("Which one?"), Failure(f"{path}/.env lacks MATTERMOST_TOKEN and MATTERMOST_RECIPIENT"))
with tempfile.TemporaryDirectory() as home:
path = self.home_with(home, "MATTERMOST_URL=https://chat.example\n")
self.assertEqual(dm.dm("Which one?", pathlib.Path(home), mattermost()), Failure(f"{path} lacks MATTERMOST_TOKEN and MATTERMOST_RECIPIENT"))
class Main(unittest.TestCase):