Answer product-owner round 1: 0 🔴, 2 🟡 (0 misread); print the report before exiting 1 on an unreadable archive, naming the file
This commit is contained in:
+12
-10
@@ -78,13 +78,14 @@ that has any, then how many have none: the defects that reappear in what already
|
||||
other half of that counter-signal.
|
||||
|
||||
Then, newest first, how many of the human's typed answers each finished chunk's judge quoted as
|
||||
reversing a choice the worker took on its own, for every chunk loop-claude finished that has any,
|
||||
then how many have none: goal 1.2's counter-signal. It reads loop-claude's archive, and counts only
|
||||
chunks whose verdicts record a Config line, since earlier judges didn't look for reversals.
|
||||
reversing a choice the worker took on its own recommendation, for every chunk loop-claude finished
|
||||
that has any, then how many have none: how often a choice the worker took on its own recommendation
|
||||
fails to hold. It reads loop-claude's archive, and counts only chunks whose verdicts record a
|
||||
Config line, since earlier judges didn't look for reversals.
|
||||
|
||||
Exit 1, saying why, on bad arguments, when git cannot start, finds no checkout or reads a shallow
|
||||
clone, the default branch or loop-claude's archive cannot be read; or, after the rest of the report, when the anchors are
|
||||
left out.
|
||||
clone, or the default branch cannot be read; or, after the rest of the report, when the anchors or
|
||||
loop-claude's archive cannot be read.
|
||||
"""
|
||||
import functools
|
||||
import os
|
||||
@@ -135,13 +136,14 @@ def main(argv: list[str]) -> int:
|
||||
return failed(results)
|
||||
anchored = read_anchors()
|
||||
reversal = read_reversals()
|
||||
if isinstance(reversal, Failure):
|
||||
return failed(reversal)
|
||||
print(report_text(results, anchored, dict(zip(GIVEN, map(int, argv))) if argv else None, reversal))
|
||||
return failed(anchored) if isinstance(anchored, Failure) else 0
|
||||
unread = [found for found in (anchored, reversal) if isinstance(found, Failure)]
|
||||
for failure in unread:
|
||||
failed(failure)
|
||||
return 1 if unread else 0
|
||||
|
||||
|
||||
def report_text(results: list[Result], anchored: Anchors | Failure, given: dict[str, int] | None, reversal: list[str]) -> str:
|
||||
def report_text(results: list[Result], anchored: Anchors | Failure, given: dict[str, int] | None, reversal: list[str] | Failure) -> str:
|
||||
"""The report: the finished items nearest the given scores, where any are given, then the anchors, then the medians, then each chunk's misread findings, then its later fixes, then its reversals, as reversal_lines gives them."""
|
||||
sections = []
|
||||
if given is not None:
|
||||
@@ -152,7 +154,7 @@ def report_text(results: list[Result], anchored: Anchors | Failure, given: dict[
|
||||
sections.append(["Median outcome at each guessed score, among chunks that shipped one item, never-replayed ones left out:", "", *report(results)])
|
||||
sections.append(["🔴 and 🟡 findings, with how many a reviewer marked `[misread]`, per finished chunk whose round titles count them, newest first:", "", *misreads(results)])
|
||||
sections.append(["Later fixes: the chunks that fixed code each finished chunk shipped, newest first:", "", *fixed_later(results)])
|
||||
sections.append(["Reversals: the human's typed answers that reversed a choice the worker took on its own, per chunk loop-claude finished, newest first:", "", *reversal])
|
||||
sections.append(["Reversals: the human's typed answers that reversed a choice the worker took on its own recommendation, per chunk loop-claude finished, newest first:", "", *([f"- None: {reversal.reason}"] if isinstance(reversal, Failure) else reversal)])
|
||||
if any(result.landing != "merge" for result in results):
|
||||
sections.append(UNMERGED)
|
||||
return "\n\n".join("\n".join(section) for section in sections)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The human's reversals of choices workers took on their own, per finished chunk in loop-claude's archive: the counter-signal docs/decisions.md → `loop-claude: a judge decides each interrupt and each quoted answer…`, its entry of 2026-11-08, is valid under."""
|
||||
"""The human's reversals of choices workers took on their own recommendation, per finished chunk in loop-claude's archive. docs/decisions.md → `loop-claude: a judge decides each interrupt and each quoted answer…`, in its entry of 2026-11-08, is valid while they stay few."""
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
@@ -15,10 +15,12 @@ def reversal_lines(project_archive: pathlib.Path) -> list[str] | Failure:
|
||||
return chunks
|
||||
counted = []
|
||||
for chunk in reversed(chunks):
|
||||
try:
|
||||
texts = [path.read_text(encoding="utf-8") for path in chunk_outcomes(project_archive / chunk.folder)]
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
return Failure(f"could not read the archive {project_archive}: {failure}")
|
||||
texts = []
|
||||
for path in chunk_outcomes(project_archive / chunk.folder):
|
||||
try:
|
||||
texts.append(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
return Failure(f"could not read the archive file {path}: {failure}")
|
||||
if any(f"\n{CONFIG}" in text for text in texts):
|
||||
# The judge quotes the latest reversal at each ruling, so one answer may be quoted many times.
|
||||
found = dict.fromkeys(quote for text in texts for quote in reversals(text))
|
||||
|
||||
@@ -222,7 +222,7 @@ Later fixes: the chunks that fixed code each finished chunk shipped, newest firs
|
||||
|
||||
- None: no later chunk fixed code a finished chunk shipped.
|
||||
|
||||
Reversals: the human's typed answers that reversed a choice the worker took on its own, per chunk loop-claude finished, newest first:
|
||||
Reversals: the human's typed answers that reversed a choice the worker took on its own recommendation, per chunk loop-claude finished, newest first:
|
||||
|
||||
- None: loop-claude's archive holds no finished chunk whose verdicts record a Config line.""")
|
||||
|
||||
@@ -315,16 +315,19 @@ class ScoreOutcomesTool(unittest.TestCase):
|
||||
def test_counts_the_reversals_in_loop_claudes_archive_of_the_main_checkout_run_from_a_worktree(self):
|
||||
archive = pathlib.Path(self.env["HOME"], "archive")
|
||||
self.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(archive)
|
||||
chunk_file(project_archive(str(self.checkout), pathlib.Path(self.env["HOME"]), self.env), "a", ("2026-11-02T10:00:00Z", outcomes.finished('{"model": "m", "item": 4}', "e", "j", record=outcomes.Record("c", reversed="No."))))
|
||||
chunk_file(project_archive(str(self.checkout), pathlib.Path(self.env["HOME"]), self.env), "a", ("2026-11-02T10:00:00Z", outcomes.finished('{"model": "m", "item": 4}', "e", "j", annotation=outcomes.Annotation("c", reversed="No."))))
|
||||
worktree = pathlib.Path(self.env["HOME"], "proj-item-4")
|
||||
self.git("worktree", "add", "-q", str(worktree))
|
||||
ran = self.run_tool(cwd=worktree)
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
self.assertTrue(ran.stdout.endswith("newest first:\n\n- 2026-11-02T10:00:00Z, item 4: 1 reversed\n"), ran.stdout[-400:])
|
||||
(archive.rglob("a.outcomes.md").__next__()).write_bytes(b"\xff")
|
||||
unreadable = next(archive.rglob("a.outcomes.md"))
|
||||
unreadable.write_bytes(b"\xff")
|
||||
ran = self.run_tool()
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertIn("could not read the archive", ran.stderr)
|
||||
self.assertIn("Later fixes:", ran.stdout)
|
||||
self.assertTrue(ran.stdout.endswith(f"newest first:\n\n- None: could not read the archive file {unreadable}: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte\n"), ran.stdout[-400:])
|
||||
self.assertEqual(ran.stderr, f"score-outcomes: could not read the archive file {unreadable}: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte\n")
|
||||
|
||||
def test_measures_each_merged_chunk_that_removed_an_item(self):
|
||||
self.git("switch", "-q", "-c", "chunk")
|
||||
@@ -500,24 +503,24 @@ class Reversals(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
archive = pathlib.Path(root)
|
||||
self.assertEqual(reversal_lines(archive), ["- None: loop-claude's archive holds no finished chunk whose verdicts record a Config line."])
|
||||
chunk_file(archive, "old", ("2026-11-01T10:00:00Z", outcomes.finished(worker, "e", "j", record=outcomes.Record(reversed="No."))))
|
||||
chunk_file(archive, "reversed", ("2026-11-02T10:00:00Z", outcomes.unfinished(worker, "e", "j", "", record=outcomes.Record("c", reversed="No."))), ("2026-11-02T11:00:00Z", outcomes.unfinished(worker, "e", "j", "", record=outcomes.Record("c", reversed="No."))), ("2026-11-02T12:00:00Z", outcomes.finished(worker, "e", "j", record=outcomes.Record("c", reversed="Use X."))))
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished('{"model": "m"}', "e", "j", record=outcomes.Record("c"))))
|
||||
chunk_file(archive, "old", ("2026-11-01T10:00:00Z", outcomes.finished(worker, "e", "j", annotation=outcomes.Annotation(reversed="No."))))
|
||||
chunk_file(archive, "reversed", ("2026-11-02T10:00:00Z", outcomes.unfinished(worker, "e", "j", "", annotation=outcomes.Annotation("c", reversed="No."))), ("2026-11-02T11:00:00Z", outcomes.unfinished(worker, "e", "j", "", annotation=outcomes.Annotation("c", reversed="No."))), ("2026-11-02T12:00:00Z", outcomes.finished(worker, "e", "j", annotation=outcomes.Annotation("c", reversed="Use X."))))
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished('{"model": "m"}', "e", "j", annotation=outcomes.Annotation("c"))))
|
||||
self.assertEqual(reversal_lines(archive), ["- 2026-11-02T12:00:00Z, item 7: 2 reversed", "- The other 1 finished chunk: none."])
|
||||
|
||||
def test_adds_no_line_for_the_others_where_every_chunk_holds_a_reversal(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
chunk_file(pathlib.Path(root), "a", ("2026-11-02T10:00:00Z", outcomes.finished('{"model": "m", "item": 7}', "e", "j", record=outcomes.Record("c", reversed="No."))))
|
||||
chunk_file(pathlib.Path(root), "a", ("2026-11-02T10:00:00Z", outcomes.finished('{"model": "m", "item": 7}', "e", "j", annotation=outcomes.Annotation("c", reversed="No."))))
|
||||
self.assertEqual(reversal_lines(pathlib.Path(root)), ["- 2026-11-02T10:00:00Z, item 7: 1 reversed"])
|
||||
|
||||
def test_says_all_chunks_have_none_and_names_no_item_the_worker_line_lacks(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
archive = pathlib.Path(root)
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished('{"model": "m", "item": 7}', "e", "j", record=outcomes.Record("c"))))
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished('{"model": "m", "item": 7}', "e", "j", annotation=outcomes.Annotation("c"))))
|
||||
self.assertEqual(reversal_lines(archive), ["- All 1 finished chunk: none."])
|
||||
for worker in ('{"model": "m"}', '{"model": "m", "item": true}', '["m"]', "not json"):
|
||||
with self.subTest(worker=worker):
|
||||
chunk_file(archive, "reversed", ("2026-11-04T10:00:00Z", outcomes.finished(worker, "e", "j", record=outcomes.Record("c", reversed="No."))))
|
||||
chunk_file(archive, "reversed", ("2026-11-04T10:00:00Z", outcomes.finished(worker, "e", "j", annotation=outcomes.Annotation("c", reversed="No."))))
|
||||
self.assertEqual(reversal_lines(archive), ["- 2026-11-04T10:00:00Z: 1 reversed", "- The other 1 finished chunk: none."])
|
||||
(archive / "reversed" / "a.outcomes.md").unlink()
|
||||
(archive / "reversed").rmdir()
|
||||
@@ -525,11 +528,11 @@ class Reversals(unittest.TestCase):
|
||||
def test_says_which_archive_it_could_not_read(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
archive = pathlib.Path(root)
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished("w", "e", "j", record=outcomes.Record("c"))))
|
||||
chunk_file(archive, "kept", ("2026-11-03T10:00:00Z", outcomes.finished("w", "e", "j", annotation=outcomes.Annotation("c"))))
|
||||
(archive / "kept" / "b.outcomes.md").write_bytes(b"\xff")
|
||||
found = reversal_lines(archive)
|
||||
self.assertIsInstance(found, Failure)
|
||||
self.assertTrue(found.reason.startswith(f"could not read the archive {archive}: "), found.reason)
|
||||
self.assertTrue(found.reason.startswith(f"could not read the archive file {archive / 'kept' / 'b.outcomes.md'}: "), found.reason)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -18,13 +18,21 @@ class FinishedChunk:
|
||||
def finished_chunks(project_archive: pathlib.Path) -> list[FinishedChunk] | Failure:
|
||||
found = []
|
||||
try:
|
||||
for folder in sorted(path for path in project_archive.glob("*") if path.is_dir()):
|
||||
copies = sorted((stamps[-1], outcomes) for outcomes in chunk_outcomes(folder) if (stamps := FINISHED_AT.findall(outcomes.read_text(encoding="utf-8"))))
|
||||
if copies:
|
||||
finished, outcomes = copies[-1]
|
||||
found.append(FinishedChunk(folder.name, finished, transcript_of(outcomes), sorted(folder.glob("session/*/subagents/*.jsonl"))))
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
folders = sorted(path for path in project_archive.glob("*") if path.is_dir())
|
||||
except OSError as failure:
|
||||
return Failure(f"could not read the archive {project_archive}: {failure}")
|
||||
for folder in folders:
|
||||
copies = []
|
||||
for outcomes in chunk_outcomes(folder):
|
||||
try:
|
||||
stamps = FINISHED_AT.findall(outcomes.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
return Failure(f"could not read the archive file {outcomes}: {failure}")
|
||||
if stamps:
|
||||
copies.append((stamps[-1], outcomes))
|
||||
if copies:
|
||||
finished, outcomes = max(copies)
|
||||
found.append(FinishedChunk(folder.name, finished, transcript_of(outcomes), sorted(folder.glob("session/*/subagents/*.jsonl"))))
|
||||
return sorted(found, key=lambda chunk: (chunk.finished, chunk.folder))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user