Answer stability round 1: 0 🔴, 1 🟡 (0 misread); the comment on a usage limit's reset time says the stream holds it, and the no-json message quotes the last line only
This commit is contained in:
@@ -17,22 +17,23 @@ class Limited:
|
||||
said: str
|
||||
|
||||
|
||||
# stream-json, whose last line claude_reply parses and whose every line a caller may keep as the run's transcript, which --no-session-persistence has Claude Code not write; -p needs --verbose with it. No permission prompt, as no one answers it.
|
||||
# --no-session-persistence keeps the runs out of the session picker, and Claude Code then writes no transcript; stream-json prints one, so a caller may keep it, and claude_reply reads the reply from its last line. -p needs --verbose with stream-json. No permission prompt, as no one answers it.
|
||||
HEADLESS = ["--no-session-persistence", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions"]
|
||||
|
||||
|
||||
def claude_reply(claude: Claude, args: list[str], cwd: str, prompt: str, who: str) -> dict | Failure | Limited:
|
||||
"""claude's reply to `claude -p <args>` and HEADLESS, the result event that ends its stream, or why there is none; who, such as "the judge", names the run in the message for a non-zero exit or a stream that ends in no json object."""
|
||||
"""claude's reply to `claude -p <args>` and HEADLESS, the result event that ends its stream, or why there is none; who, such as "the judge", names the run in the message for a non-zero exit or a last line that isn't a json object."""
|
||||
try:
|
||||
proc = claude(["-p", *args, *HEADLESS], cwd, prompt)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not start claude: {failure.strerror}")
|
||||
reply = json_object(proc.stdout.strip().rpartition("\n")[2])
|
||||
# The result event names the reset only in prose, such as "resets 7:40pm (Europe/Stockholm)", so its limit carries no reset time.
|
||||
last = proc.stdout.strip().rpartition("\n")[2]
|
||||
reply = json_object(last)
|
||||
# The result event names the reset only in prose, such as "resets 7:40pm (Europe/Stockholm)", so its limit carries no reset time; the stream's rate_limit_event holds it as a time, which nothing reads here yet.
|
||||
if reply is not None and hit_limit(reply):
|
||||
return Limited(None, limit_said(reply))
|
||||
if proc.returncode != 0:
|
||||
return Failure(proc.stderr.strip() or f"{who} exited {proc.returncode}")
|
||||
if reply is None:
|
||||
return Failure(f"{who} printed no json reply; it printed: {proc.stdout.strip() or 'nothing'}")
|
||||
return Failure(f"{who} printed no json reply; its last line was: {last or 'nothing'}")
|
||||
return reply
|
||||
|
||||
@@ -205,10 +205,13 @@ class ClaudeReply(unittest.TestCase):
|
||||
stream = "\n".join(json.dumps(event) for event in ({"type": "system", "subtype": "init"}, {"type": "assistant"}, {"type": "result", "result": "ok"}))
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, stream + "\n", "")), [], "/r", "p", "the judge"), {"type": "result", "result": "ok"})
|
||||
|
||||
def test_a_stream_that_ends_in_no_json_names_only_its_last_line(self):
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, '{"type": "system"}\nWarning: bye\n', "")), [], "/r", "p", "the judge"), Failure("the judge printed no json reply; its last line was: Warning: bye"))
|
||||
|
||||
def test_returns_the_json_reply_a_usage_limit_whatever_the_exit_code_or_why_there_is_none(self):
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, '{"result": "ok"}', "")), [], "/r", "p", "the judge"), {"result": "ok"})
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, "Warning: hook said hi\n", "")), [], "/r", "p", "the judge"), Failure("the judge printed no json reply; it printed: Warning: hook said hi"))
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, "", "")), [], "/r", "p", "the judge"), Failure("the judge printed no json reply; it printed: nothing"))
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, "Warning: hook said hi\n", "")), [], "/r", "p", "the judge"), Failure("the judge printed no json reply; its last line was: Warning: hook said hi"))
|
||||
self.assertEqual(reply.claude_reply(self.claude((0, "", "")), [], "/r", "p", "the judge"), Failure("the judge printed no json reply; its last line was: nothing"))
|
||||
limit = json.dumps({"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
||||
self.assertEqual(reply.claude_reply(self.claude((1, limit, "")), [], "/r", "p", "the judge"), reply.Limited(None, "You've hit your limit"))
|
||||
self.assertEqual(reply.claude_reply(self.claude((2, "", " ")), [], "/r", "p", "the judge"), Failure("the judge exited 2"))
|
||||
|
||||
@@ -699,7 +699,7 @@ class Cycles(unittest.TestCase):
|
||||
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
|
||||
self.assertEqual(run.sleeps, [60, 60])
|
||||
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
|
||||
self.assertIn("retrying the judge in a minute: the judge printed no json reply; it printed: nothing\n", run.stderr)
|
||||
self.assertIn("retrying the judge in a minute: the judge printed no json reply; its last line was: nothing\n", run.stderr)
|
||||
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string')
|
||||
|
||||
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
|
||||
|
||||
@@ -16,7 +16,7 @@ RESULT = "result.md"
|
||||
|
||||
|
||||
def replay_folder(root: pathlib.Path, tool: str, argv: list[str], now: float) -> pathlib.Path:
|
||||
"""Where a replay of tool run with argv at now keeps its runs and its result, under the archive at root; no reader of loop-claude's archive takes a folder there for a project's."""
|
||||
"""The folder a replay of tool, run with argv at now, keeps its runs and its result in, under the archive at root. It holds no outcomes file, so no reader of loop-claude's archive takes it for a chunk's."""
|
||||
name = "-".join([tool, *(arg.lstrip("-") for arg in argv), time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(now))])
|
||||
return root / "replays" / re.sub(r"[^A-Za-z0-9._-]", "_", name)
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@ class ArchivedRuns(unittest.TestCase):
|
||||
self.assertEqual(out.getvalue(), "The report.\n")
|
||||
self.assertIn(f"could not keep the result in {self.folder}: ", err.getvalue())
|
||||
|
||||
def test_names_the_folder_by_the_tool_its_arguments_and_the_minute_in_utc_under_replays(self):
|
||||
def test_names_the_folder_by_the_tool_its_arguments_and_the_second_in_utc_under_replays(self):
|
||||
root = pathlib.Path("/archive")
|
||||
self.assertEqual(replay_folder(root, "review-replay", ["stability", "claude-opus-5-5", "--omit-claude-md"], 1790000000.0), root / "replays" / "review-replay-stability-claude-opus-5-5-omit-claude-md-20260921T141320Z")
|
||||
self.assertEqual(replay_folder(root, "judge-replay", ["a/b c"], 0.0), root / "replays" / "judge-replay-a_b_c-19700101T000000Z")
|
||||
|
||||
Reference in New Issue
Block a user