Give record, verdict and judge one meaning each

This commit is contained in:
2026-10-03 17:53:25 +02:00
parent 9f75fe8c51
commit 5f7d34b87d
12 changed files with 39 additions and 39 deletions
+2 -2
View File
@@ -37,6 +37,6 @@
!skills/larv-review/*.md
!workflow-bench/
!workflow-bench/bench.py
!workflow-bench/judge-prompt.md
!workflow-bench/judge.py
!workflow-bench/compare-prompt.md
!workflow-bench/compare.py
!workflow-bench/run-prompt.md
+1 -1
View File
@@ -12,7 +12,7 @@ Decisions live in `docs/decisions.md`:
- loop-claude: each project's model lives in `~/.claude/loop-claude-models.json`, which only loop-claude writes
- review: a pass gets at most three rounds per chunk; instead of a fourth, the run asks the human
- review: a fix for a 🟢 or ❓ re-enters review like any fix
- review: a pass skips only on a record older than the chunk
- review: a pass skips only on a standing decision, one the repo held before the chunk began
- loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop
- loop-claude: a `claude` that cannot start is a failed worker run, retried
- review: an ask no user answers leaves the PR unmerged
+1 -1
View File
@@ -27,6 +27,6 @@
- `loop-claude` retries a failed worker within its cycle.
- `loop-claude` counts an interrupt reply starting with `<` as typed.
- `loop-claude`'s judge sees a reply the human types while claude works, and an interrupt counts one as typed.
- `loop-claude` archives the raw `.jsonl` transcript at every interrupt and as every cycle ends, Ctrl-C and failures included, in one folder per chunk. Each distinct transcript is kept once, named by its SHA-256, with `<sha256>.verdict.md` beside it holding the verdict and why the loop stopped. The subagents' transcripts and the session's other files go in the chunk folder's `session/` subfolder. `loop-claude` exits 1 when it cannot archive.
- `loop-claude` archives the raw `.jsonl` transcript at every interrupt and as every cycle ends, Ctrl-C and failures included, in one folder per chunk. Each distinct transcript is kept once, named by its SHA-256, with `<sha256>.outcomes.md` beside it holding the verdict and why the loop stopped. The subagents' transcripts and the session's other files go in the chunk folder's `session/` subfolder. `loop-claude` exits 1 when it cannot archive.
- `loop-claude` exits 1 when it cannot delete a finished chunk's session, naming what to delete before rerunning; the next chunk would otherwise resume it.
- `loop-claude` keeps the mode of `~/.claude.json` and of its models file when it rewrites them, and creates the models file 0600.
+8 -8
View File
@@ -23,11 +23,11 @@ def archive(project_key: str, path: str, task: str, outcome: str, paths: Paths)
if sibling != path and (unkept := mirror(pathlib.Path(sibling), folder / "session" / os.path.basename(sibling))):
return unkept
copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl"
verdict = copy.with_suffix(".verdict.md")
outcomes = copy.with_suffix(".outcomes.md")
if copy.exists():
return add_outcome(verdict, outcome)
# The copy lands last, so a copy on disk always has its verdict and its session's files beside it.
for dest, data in ((verdict, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)):
return add_outcome(outcomes, outcome)
# The copy lands last, so a copy on disk always has its outcomes and its session's files beside it.
for dest, data in ((outcomes, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)):
if unwritten := replace_file(dest, data):
return unarchived(dest, unwritten.reason)
print(f"transcript: {copy}", file=sys.stderr)
@@ -38,13 +38,13 @@ def unarchived(where: object, why: str | None) -> Failure:
return Failure(f"could not archive the transcript: {where}: {why}")
def add_outcome(verdict: pathlib.Path, outcome: str) -> Failure | None:
def add_outcome(outcomes: pathlib.Path, outcome: str) -> Failure | None:
try:
outcomes = verdict.read_bytes()
kept = outcomes.read_bytes()
except OSError as failure:
return unarchived(failure.filename, failure.strerror)
if unwritten := replace_file(verdict, outcomes + f"\n{outcome}\n".encode()):
return unarchived(verdict, unwritten.reason)
if unwritten := replace_file(outcomes, kept + f"\n{outcome}\n".encode()):
return unarchived(outcomes, unwritten.reason)
return None
+1 -1
View File
@@ -21,7 +21,7 @@ relative to ~/code, or to / outside it. The session is deleted when its chunk fi
interrupt, and as every cycle ends, Ctrl-C and a failure included, the session's transcript is
copied to $LOOP_CLAUDE_TRANSCRIPTS/<key>/<chunk>/<sha256>.jsonl (default
~/.claude/loop-claude-transcripts), named by its hash. <chunk> is the hash of the transcript's first
line with a timestamp, the same for every cycle of a chunk. Beside the copy, <sha256>.verdict.md
line with a timestamp, the same for every cycle of a chunk. Beside the copy, <sha256>.outcomes.md
holds the verdict, and why the loop stopped. Claude Code keeps a folder per session beside its
transcript, holding the subagents' transcripts among other files; that folder, and every other file
named after the session except the transcript, are copied into <chunk>/session/. A file there is
+1 -1
View File
@@ -27,7 +27,7 @@ PRIVATE = [
"tasks/a/b.json",
"workflow-bench/a-1-1.md",
"workflow-bench/a-1-1/README.md",
"workflow-bench/judge/a/A1/fixes.diff",
"workflow-bench/compare/a/A1/fixes.diff",
"workflow-bench/logs/a.jsonl",
]
+1 -1
View File
@@ -169,7 +169,7 @@ class Cycles(unittest.TestCase):
return run.Cycle(effects=fake.effects(), models=model_choice.Models("judge-model", "worker-model"), paths=self.paths, premises="## Goals", project_key="proj", records=records, root=str(self.home), session=self.session, task="Do it.")
def archived_verdicts(self):
return sorted(path.read_text() for path in (self.paths.transcript_archive / "proj").glob("*/*.verdict.md"))
return sorted(path.read_text() for path in (self.paths.transcript_archive / "proj").glob("*/*.outcomes.md"))
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
+6 -6
View File
@@ -250,17 +250,17 @@ class Checkout:
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
def archived(self):
"""Each archived transcript's verdict, after checking the copy is named by its content hash."""
verdicts = {}
"""Each archived transcript's outcomes, after checking the copy is named by its content hash."""
outcomes = {}
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
verdict = copy.with_suffix(".verdict.md")
for archived in (copy, verdict):
kept = copy.with_suffix(".outcomes.md")
for archived in (copy, kept):
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
verdicts[copy.read_text()] = verdict.read_text()
return verdicts
outcomes[copy.read_text()] = kept.read_text()
return outcomes
def argument(call, flag):
+3 -3
View File
@@ -48,13 +48,13 @@ The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.
2026-09-30, lilleman. Serves goal 4: no human reads an unattended chunk before it merges, so every change it makes, a nit's wording included, is reviewed. In the 2026-09-30 bench this cost four of nine rounds in one run and drove two passes to the round cap. Valid while a pass is the only reader of a fix before merge.
## review: a pass skips only on a record older than the chunk
## review: a pass skips only on a standing decision, one the repo held before the chunk began
2026-09-30, lilleman. Serves goal 4: a record the chunk writes is the chunk's own unreviewed decision, and an unattended run has no human to catch it. Valid while a pass is the only reader of a chunk before merge.
2026-09-30, lilleman. Serves goal 4: a decision the chunk writes is the chunk's own and unreviewed, and an unattended run has no human to catch it. Valid while a pass is the only reader of a chunk before merge.
## loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop
2026-09-28, lilleman. Serves goals 2 and 4: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with the verdict beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. Valid while transcripts stay small enough to copy whole.
2026-09-28, lilleman. Serves goals 2 and 4: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with its outcomes beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. Valid while transcripts stay small enough to copy whole.
## loop-claude: a `claude` that cannot start is a failed worker run, retried
+5 -5
View File
@@ -41,12 +41,12 @@ A comment is prose, not code. Passes run in this order: line-level polish on cod
Give a reviewer without Bash the branch diff and `git ls-files`'s output in its prompt: some harnesses grant it Read alone.
**Record, then ask** — for the architecture pass and the full scan:
**Standing decision, then ask** — for the architecture pass and the full scan:
- **Skip.** A record is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite the record.
- **Departure.** A change that departs from the record is unreviewed, however thorough the record is.
- **Skip.** A standing decision is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite it.
- **Departure.** A change that departs from the standing decision is unreviewed, however thorough the decision is.
- **Ask.** Fired and not skipped → ask the user whether to run it, with the evidence and your recommendation; no user reachable → run it. For the full scan, give the evidence and your recommendation, then launch it: the plugin asks the user to confirm its cost, and that prompt is the ask.
- **Audit.** A record or the user's no skips only the architect, never the principles audit or the scoring run: no record older than the chunk checked the code it writes.
- **Audit.** A standing decision or the user's no skips only the architect, never the principles audit or the scoring run: no decision older than the chunk checked the code it writes.
**Round cap.**
@@ -70,7 +70,7 @@ Give a reviewer without Bash the branch diff and `git ls-files`'s output in its
- **Goal or decision.** A fix that would change a goal, or a decision the repo held before the chunk began, is asked; no user reachable → filed, marked `question`, the goal or decision left as is.
- **🔴.** A 🔴 is fixed, or shown factually wrong, and what disproved it recorded per `~/.claude/CLAUDE.md` → "Decisions: one home, each linked to a goal or a technical principle": in a comment at the code the finding concerns, or in `docs/decisions.md` when it concerns more than one place. Never declined.
- **🟡.** A 🟡 is fixed, shown factually wrong like a 🔴, or declined. Decline one only where the reason is sound, consistent with the goals and the technical principles, and confidence is very high; the decline and its reason are recorded the same way. A reason neither speaks to is a gap in the goals: fix the finding, or ask per `~/.claude/CLAUDE.md` → "Working loop", "Ask, don't guess"; no user reachable → fix. Never drop one silently, and never decline one for convenience or the size of the fix.
- **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a record settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`.
- **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a standing decision settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`.
- **🔀.** A 🔀 runs the named reviewer's pass once the current pass is done.
- **Other scales.** Other reviewers map onto this scale: `/security-review`'s HIGH is 🔴, MEDIUM 🟡, LOW 🟢; what a forge's auto-reviewer marks blocking is 🔴. A finding of theirs repeating one declined or filed does not stand.
@@ -1,4 +1,4 @@
You judge a bench of two code-review processes, A and B, on one real chunk of work. Each process reviewed the same chunk three times, independently: runs A1–A3 and B1–B3. Rule whether B reviews at least as well as A, and whether A reviews at least as well as B.
You compare two code-review processes, A and B, on one real chunk of work. Each process reviewed the same chunk three times, independently: runs A1–A3 and B1–B3. Rule whether B reviews at least as well as A, and whether A reviews at least as well as B.
Everything is under `{dir}`; read nothing outside it:
@@ -7,7 +7,7 @@ Everything is under `{dir}`; read nothing outside it:
- `<run>/report.md`: the run's own account of its passes, findings and outcomes. Its claims are not evidence.
- `<run>/fixes.diff`: what the run actually changed. This is the evidence.
Judge each run on:
Rate each run on:
1. **Truth coverage**: for each truth issue, caught and fixed ✔, partly ◐, or missed ✘, checked against `fixes.diff`. A filed or declined issue with a sound reason counts ◐.
2. **Other real defects**: a finding the truth lacks that is a real defect in the chunk, fixed or filed.
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Blind-judge each case's six runs: BENCH_SOURCES=<as for bench.py> judge.py <case> ..."""
"""Blind-compare each case's six runs: BENCH_SOURCES=<as for bench.py> compare.py <case> ..."""
import concurrent.futures
import json
import pathlib
@@ -33,7 +33,7 @@ def blind(text):
def prepare(case):
folder = BENCH / "judge" / case
folder = BENCH / "compare" / case
if folder.exists():
shutil.rmtree(folder)
folder.mkdir(parents=True)
@@ -53,15 +53,15 @@ def prepare(case):
(run / "report.md").write_text(blind(report.read_text()))
(run / "fixes.diff").write_text(bench.git(clone, "diff", head, "chunk") + "\n")
mapping[f"{label}{n}"] = str(report)
(BENCH / "judge" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
(BENCH / "compare" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
return folder
def judge(case):
def compare(case):
folder = prepare(case)
ruling = BENCH / "judge" / f"{case}.ruling.md"
prompt = (BENCH / "judge-prompt.md").read_text().format(dir=folder, ruling=ruling)
with (BENCH / "logs" / f"judge-{case}.jsonl").open("w") as out:
ruling = BENCH / "compare" / f"{case}.ruling.md"
prompt = (BENCH / "compare-prompt.md").read_text().format(dir=folder, ruling=ruling)
with (BENCH / "logs" / f"compare-{case}.jsonl").open("w") as out:
subprocess.run(
["claude", "-p", prompt, "--model", "claude-opus-5-5", "--allowedTools", "Read,Grep,Glob,Write", "--output-format", "stream-json", "--verbose"],
cwd=folder, stdout=out, stderr=subprocess.STDOUT)
@@ -70,5 +70,5 @@ def judge(case):
if __name__ == "__main__":
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(judge, sys.argv[1:]):
for result in pool.map(compare, sys.argv[1:]):
print(result, flush=True)