Give record, verdict and judge one meaning each
This commit is contained in:
+2
-2
@@ -37,6 +37,6 @@
|
||||
!skills/larv-review/*.md
|
||||
!workflow-bench/
|
||||
!workflow-bench/bench.py
|
||||
!workflow-bench/judge-prompt.md
|
||||
!workflow-bench/judge.py
|
||||
!workflow-bench/compare-prompt.md
|
||||
!workflow-bench/compare.py
|
||||
!workflow-bench/run-prompt.md
|
||||
|
||||
@@ -12,7 +12,7 @@ Decisions live in `docs/decisions.md`:
|
||||
- loop-claude: each project's model lives in `~/.claude/loop-claude-models.json`, which only loop-claude writes
|
||||
- review: a pass gets at most three rounds per chunk; instead of a fourth, the run asks the human
|
||||
- review: a fix for a 🟢 or ❓ re-enters review like any fix
|
||||
- review: a pass skips only on a record older than the chunk
|
||||
- review: a pass skips only on a standing decision, one the repo held before the chunk began
|
||||
- loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop
|
||||
- loop-claude: a `claude` that cannot start is a failed worker run, retried
|
||||
- review: an ask no user answers leaves the PR unmerged
|
||||
|
||||
+1
-1
@@ -27,6 +27,6 @@
|
||||
- `loop-claude` retries a failed worker within its cycle.
|
||||
- `loop-claude` counts an interrupt reply starting with `<` as typed.
|
||||
- `loop-claude`'s judge sees a reply the human types while claude works, and an interrupt counts one as typed.
|
||||
- `loop-claude` archives the raw `.jsonl` transcript at every interrupt and as every cycle ends, Ctrl-C and failures included, in one folder per chunk. Each distinct transcript is kept once, named by its SHA-256, with `<sha256>.verdict.md` beside it holding the verdict and why the loop stopped. The subagents' transcripts and the session's other files go in the chunk folder's `session/` subfolder. `loop-claude` exits 1 when it cannot archive.
|
||||
- `loop-claude` archives the raw `.jsonl` transcript at every interrupt and as every cycle ends, Ctrl-C and failures included, in one folder per chunk. Each distinct transcript is kept once, named by its SHA-256, with `<sha256>.outcomes.md` beside it holding the verdict and why the loop stopped. The subagents' transcripts and the session's other files go in the chunk folder's `session/` subfolder. `loop-claude` exits 1 when it cannot archive.
|
||||
- `loop-claude` exits 1 when it cannot delete a finished chunk's session, naming what to delete before rerunning; the next chunk would otherwise resume it.
|
||||
- `loop-claude` keeps the mode of `~/.claude.json` and of its models file when it rewrites them, and creates the models file 0600.
|
||||
|
||||
@@ -23,11 +23,11 @@ def archive(project_key: str, path: str, task: str, outcome: str, paths: Paths)
|
||||
if sibling != path and (unkept := mirror(pathlib.Path(sibling), folder / "session" / os.path.basename(sibling))):
|
||||
return unkept
|
||||
copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl"
|
||||
verdict = copy.with_suffix(".verdict.md")
|
||||
outcomes = copy.with_suffix(".outcomes.md")
|
||||
if copy.exists():
|
||||
return add_outcome(verdict, outcome)
|
||||
# The copy lands last, so a copy on disk always has its verdict and its session's files beside it.
|
||||
for dest, data in ((verdict, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)):
|
||||
return add_outcome(outcomes, outcome)
|
||||
# The copy lands last, so a copy on disk always has its outcomes and its session's files beside it.
|
||||
for dest, data in ((outcomes, f"Task: {task}\n\n{outcome}\n".encode()), (copy, raw)):
|
||||
if unwritten := replace_file(dest, data):
|
||||
return unarchived(dest, unwritten.reason)
|
||||
print(f"transcript: {copy}", file=sys.stderr)
|
||||
@@ -38,13 +38,13 @@ def unarchived(where: object, why: str | None) -> Failure:
|
||||
return Failure(f"could not archive the transcript: {where}: {why}")
|
||||
|
||||
|
||||
def add_outcome(verdict: pathlib.Path, outcome: str) -> Failure | None:
|
||||
def add_outcome(outcomes: pathlib.Path, outcome: str) -> Failure | None:
|
||||
try:
|
||||
outcomes = verdict.read_bytes()
|
||||
kept = outcomes.read_bytes()
|
||||
except OSError as failure:
|
||||
return unarchived(failure.filename, failure.strerror)
|
||||
if unwritten := replace_file(verdict, outcomes + f"\n{outcome}\n".encode()):
|
||||
return unarchived(verdict, unwritten.reason)
|
||||
if unwritten := replace_file(outcomes, kept + f"\n{outcome}\n".encode()):
|
||||
return unarchived(outcomes, unwritten.reason)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ relative to ~/code, or to / outside it. The session is deleted when its chunk fi
|
||||
interrupt, and as every cycle ends, Ctrl-C and a failure included, the session's transcript is
|
||||
copied to $LOOP_CLAUDE_TRANSCRIPTS/<key>/<chunk>/<sha256>.jsonl (default
|
||||
~/.claude/loop-claude-transcripts), named by its hash. <chunk> is the hash of the transcript's first
|
||||
line with a timestamp, the same for every cycle of a chunk. Beside the copy, <sha256>.verdict.md
|
||||
line with a timestamp, the same for every cycle of a chunk. Beside the copy, <sha256>.outcomes.md
|
||||
holds the verdict, and why the loop stopped. Claude Code keeps a folder per session beside its
|
||||
transcript, holding the subagents' transcripts among other files; that folder, and every other file
|
||||
named after the session except the transcript, are copied into <chunk>/session/. A file there is
|
||||
|
||||
@@ -27,7 +27,7 @@ PRIVATE = [
|
||||
"tasks/a/b.json",
|
||||
"workflow-bench/a-1-1.md",
|
||||
"workflow-bench/a-1-1/README.md",
|
||||
"workflow-bench/judge/a/A1/fixes.diff",
|
||||
"workflow-bench/compare/a/A1/fixes.diff",
|
||||
"workflow-bench/logs/a.jsonl",
|
||||
]
|
||||
|
||||
|
||||
@@ -169,7 +169,7 @@ class Cycles(unittest.TestCase):
|
||||
return run.Cycle(effects=fake.effects(), models=model_choice.Models("judge-model", "worker-model"), paths=self.paths, premises="## Goals", project_key="proj", records=records, root=str(self.home), session=self.session, task="Do it.")
|
||||
|
||||
def archived_verdicts(self):
|
||||
return sorted(path.read_text() for path in (self.paths.transcript_archive / "proj").glob("*/*.verdict.md"))
|
||||
return sorted(path.read_text() for path in (self.paths.transcript_archive / "proj").glob("*/*.outcomes.md"))
|
||||
|
||||
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
||||
|
||||
@@ -250,17 +250,17 @@ class Checkout:
|
||||
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
|
||||
|
||||
def archived(self):
|
||||
"""Each archived transcript's verdict, after checking the copy is named by its content hash."""
|
||||
verdicts = {}
|
||||
"""Each archived transcript's outcomes, after checking the copy is named by its content hash."""
|
||||
outcomes = {}
|
||||
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
|
||||
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
|
||||
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
|
||||
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
|
||||
verdict = copy.with_suffix(".verdict.md")
|
||||
for archived in (copy, verdict):
|
||||
kept = copy.with_suffix(".outcomes.md")
|
||||
for archived in (copy, kept):
|
||||
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
|
||||
verdicts[copy.read_text()] = verdict.read_text()
|
||||
return verdicts
|
||||
outcomes[copy.read_text()] = kept.read_text()
|
||||
return outcomes
|
||||
|
||||
|
||||
def argument(call, flag):
|
||||
|
||||
+3
-3
@@ -48,13 +48,13 @@ The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.
|
||||
|
||||
2026-09-30, lilleman. Serves goal 4: no human reads an unattended chunk before it merges, so every change it makes, a nit's wording included, is reviewed. In the 2026-09-30 bench this cost four of nine rounds in one run and drove two passes to the round cap. Valid while a pass is the only reader of a fix before merge.
|
||||
|
||||
## review: a pass skips only on a record older than the chunk
|
||||
## review: a pass skips only on a standing decision, one the repo held before the chunk began
|
||||
|
||||
2026-09-30, lilleman. Serves goal 4: a record the chunk writes is the chunk's own unreviewed decision, and an unattended run has no human to catch it. Valid while a pass is the only reader of a chunk before merge.
|
||||
2026-09-30, lilleman. Serves goal 4: a decision the chunk writes is the chunk's own and unreviewed, and an unattended run has no human to catch it. Valid while a pass is the only reader of a chunk before merge.
|
||||
|
||||
## loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop
|
||||
|
||||
2026-09-28, lilleman. Serves goals 2 and 4: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with the verdict beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. Valid while transcripts stay small enough to copy whole.
|
||||
2026-09-28, lilleman. Serves goals 2 and 4: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with its outcomes beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. Valid while transcripts stay small enough to copy whole.
|
||||
|
||||
## loop-claude: a `claude` that cannot start is a failed worker run, retried
|
||||
|
||||
|
||||
@@ -41,12 +41,12 @@ A comment is prose, not code. Passes run in this order: line-level polish on cod
|
||||
|
||||
Give a reviewer without Bash the branch diff and `git ls-files`'s output in its prompt: some harnesses grant it Read alone.
|
||||
|
||||
**Record, then ask** — for the architecture pass and the full scan:
|
||||
**Standing decision, then ask** — for the architecture pass and the full scan:
|
||||
|
||||
- **Skip.** A record is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite the record.
|
||||
- **Departure.** A change that departs from the record is unreviewed, however thorough the record is.
|
||||
- **Skip.** A standing decision is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite it.
|
||||
- **Departure.** A change that departs from the standing decision is unreviewed, however thorough the decision is.
|
||||
- **Ask.** Fired and not skipped → ask the user whether to run it, with the evidence and your recommendation; no user reachable → run it. For the full scan, give the evidence and your recommendation, then launch it: the plugin asks the user to confirm its cost, and that prompt is the ask.
|
||||
- **Audit.** A record or the user's no skips only the architect, never the principles audit or the scoring run: no record older than the chunk checked the code it writes.
|
||||
- **Audit.** A standing decision or the user's no skips only the architect, never the principles audit or the scoring run: no decision older than the chunk checked the code it writes.
|
||||
|
||||
**Round cap.**
|
||||
|
||||
@@ -70,7 +70,7 @@ Give a reviewer without Bash the branch diff and `git ls-files`'s output in its
|
||||
- **Goal or decision.** A fix that would change a goal, or a decision the repo held before the chunk began, is asked; no user reachable → filed, marked `question`, the goal or decision left as is.
|
||||
- **🔴.** A 🔴 is fixed, or shown factually wrong, and what disproved it recorded per `~/.claude/CLAUDE.md` → "Decisions: one home, each linked to a goal or a technical principle": in a comment at the code the finding concerns, or in `docs/decisions.md` when it concerns more than one place. Never declined.
|
||||
- **🟡.** A 🟡 is fixed, shown factually wrong like a 🔴, or declined. Decline one only where the reason is sound, consistent with the goals and the technical principles, and confidence is very high; the decline and its reason are recorded the same way. A reason neither speaks to is a gap in the goals: fix the finding, or ask per `~/.claude/CLAUDE.md` → "Working loop", "Ask, don't guess"; no user reachable → fix. Never drop one silently, and never decline one for convenience or the size of the fix.
|
||||
- **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a record settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`.
|
||||
- **🟢 and ❓.** A 🟢 or ❓ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a standing decision settles it, and quote what in the summary. A ❓ only the user can answer is asked; no user reachable → filed, marked `question`.
|
||||
- **🔀.** A 🔀 runs the named reviewer's pass once the current pass is done.
|
||||
- **Other scales.** Other reviewers map onto this scale: `/security-review`'s HIGH is 🔴, MEDIUM 🟡, LOW 🟢; what a forge's auto-reviewer marks blocking is 🔴. A finding of theirs repeating one declined or filed does not stand.
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
You judge a bench of two code-review processes, A and B, on one real chunk of work. Each process reviewed the same chunk three times, independently: runs A1–A3 and B1–B3. Rule whether B reviews at least as well as A, and whether A reviews at least as well as B.
|
||||
You compare two code-review processes, A and B, on one real chunk of work. Each process reviewed the same chunk three times, independently: runs A1–A3 and B1–B3. Rule whether B reviews at least as well as A, and whether A reviews at least as well as B.
|
||||
|
||||
Everything is under `{dir}`; read nothing outside it:
|
||||
|
||||
@@ -7,7 +7,7 @@ Everything is under `{dir}`; read nothing outside it:
|
||||
- `<run>/report.md`: the run's own account of its passes, findings and outcomes. Its claims are not evidence.
|
||||
- `<run>/fixes.diff`: what the run actually changed. This is the evidence.
|
||||
|
||||
Judge each run on:
|
||||
Rate each run on:
|
||||
|
||||
1. **Truth coverage**: for each truth issue, caught and fixed ✔, partly ◐, or missed ✘, checked against `fixes.diff`. A filed or declined issue with a sound reason counts ◐.
|
||||
2. **Other real defects**: a finding the truth lacks that is a real defect in the chunk, fixed or filed.
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Blind-judge each case's six runs: BENCH_SOURCES=<as for bench.py> judge.py <case> ..."""
|
||||
"""Blind-compare each case's six runs: BENCH_SOURCES=<as for bench.py> compare.py <case> ..."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import pathlib
|
||||
@@ -33,7 +33,7 @@ def blind(text):
|
||||
|
||||
|
||||
def prepare(case):
|
||||
folder = BENCH / "judge" / case
|
||||
folder = BENCH / "compare" / case
|
||||
if folder.exists():
|
||||
shutil.rmtree(folder)
|
||||
folder.mkdir(parents=True)
|
||||
@@ -53,15 +53,15 @@ def prepare(case):
|
||||
(run / "report.md").write_text(blind(report.read_text()))
|
||||
(run / "fixes.diff").write_text(bench.git(clone, "diff", head, "chunk") + "\n")
|
||||
mapping[f"{label}{n}"] = str(report)
|
||||
(BENCH / "judge" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
|
||||
(BENCH / "compare" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
|
||||
return folder
|
||||
|
||||
|
||||
def judge(case):
|
||||
def compare(case):
|
||||
folder = prepare(case)
|
||||
ruling = BENCH / "judge" / f"{case}.ruling.md"
|
||||
prompt = (BENCH / "judge-prompt.md").read_text().format(dir=folder, ruling=ruling)
|
||||
with (BENCH / "logs" / f"judge-{case}.jsonl").open("w") as out:
|
||||
ruling = BENCH / "compare" / f"{case}.ruling.md"
|
||||
prompt = (BENCH / "compare-prompt.md").read_text().format(dir=folder, ruling=ruling)
|
||||
with (BENCH / "logs" / f"compare-{case}.jsonl").open("w") as out:
|
||||
subprocess.run(
|
||||
["claude", "-p", prompt, "--model", "claude-opus-5-5", "--allowedTools", "Read,Grep,Glob,Write", "--output-format", "stream-json", "--verbose"],
|
||||
cwd=folder, stdout=out, stderr=subprocess.STDOUT)
|
||||
@@ -70,5 +70,5 @@ def judge(case):
|
||||
|
||||
if __name__ == "__main__":
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
for result in pool.map(judge, sys.argv[1:]):
|
||||
for result in pool.map(compare, sys.argv[1:]):
|
||||
print(result, flush=True)
|
||||
Reference in New Issue
Block a user