Answer stability round 2: 0 🔴, 1 🟡 (0 misread); the run-pairing test holds each prompt with its reply again, and the task heading test now kills its mutant
This commit is contained in:
@@ -398,8 +398,8 @@ Worker: {OFF_LADDER}
|
||||
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
|
||||
self.assertEqual(len(named), 3, kept)
|
||||
runs = [json.loads((chunk / name).read_text()) for name in named]
|
||||
self.assertEqual(sorted(found["prompt"] for found in runs), sorted(prompts))
|
||||
self.assertEqual(sorted(json.dumps(found["reply"]) for found in runs), sorted(json.dumps(json.loads(reply.stdout)) for reply in (rejected, interrupted, finished)))
|
||||
replies = [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)]
|
||||
self.assertEqual(sorted((found["prompt"], json.dumps(found["reply"])) for found in runs), sorted((prompt, json.dumps(reply)) for prompt, reply in zip(prompts, replies)))
|
||||
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
|
||||
|
||||
def test_stops_when_a_judge_run_cannot_be_archived(self):
|
||||
|
||||
Reference in New Issue
Block a user