Read a Touches list from a Jira description, as the codec spells it, and reword four decisions' expiry and two reading-cost goal citations
CI / tests (pull_request) Failing after 3m8s
CI / tests (pull_request) Failing after 3m8s
This commit is contained in:
@@ -10,7 +10,7 @@ The spelling, which the decision `todo.md: an item of the next release predicts
|
||||
|
||||
A path alone is the whole file, and (small) marks a part the item changes only a little. file_parts says how a part is named.
|
||||
|
||||
A path or a name that holds a backtick is spelt as a Markdown code span that holds one: in a longer run of backticks than any it holds, with a space inside each end, as in `` ## The `foo` tool ``. So every path and name on one line has a spelling."""
|
||||
A path or a name that holds a backtick is spelt as a Markdown code span that holds one: in a longer run of backticks than any it holds, with a space inside each end, as in `` ## The `foo` tool ``. So every path and name on one line has a spelling. A description kept in Jira reaches here through adf_codec, which spells such a name without the spaces where it neither opens nor ends in a backtick, as ``a`b.md``; that is read as the same name."""
|
||||
import dataclasses
|
||||
import re
|
||||
|
||||
@@ -82,10 +82,11 @@ def code_span(line: str, at: int) -> tuple[str, int] | None:
|
||||
return None
|
||||
text = line[opened.end():opened.end() + closed.start()]
|
||||
if len(run) > 1:
|
||||
# A longer run holds a space inside each end, which keeps a backtick at either end of the text apart from the run.
|
||||
if len(text) < 3 or not text.startswith(" ") or not text.endswith(" "):
|
||||
if not text.strip():
|
||||
return None
|
||||
text = text[1:-1]
|
||||
# As a Markdown code span reads: a space inside each end keeps a backtick at either end of the text apart from the run, and goes. adf_codec pads only that case, so no space is also the text.
|
||||
if len(text) >= 3 and text.startswith(" ") and text.endswith(" "):
|
||||
text = text[1:-1]
|
||||
return (text, opened.end() + closed.end()) if text else None
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -8,6 +9,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from adf_codec import convert
|
||||
from touch_predictions.beside import Beside, chunks_beside
|
||||
from touch_predictions import diff_parts, file_parts, main, record
|
||||
from failure import Failure
|
||||
@@ -31,6 +33,30 @@ Touches:
|
||||
More prose, with `bin/b.py` named.
|
||||
"""
|
||||
|
||||
needs_docker = unittest.skipIf(shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True, timeout=20).returncode != 0, "adf_codec runs in a container, which needs a docker daemon")
|
||||
|
||||
|
||||
def text(words: str, code: bool = False) -> dict:
|
||||
return {"type": "text", "text": words, **({"marks": [{"type": "code"}]} if code else {})}
|
||||
|
||||
|
||||
def bullet_of(*spans: dict) -> dict:
|
||||
return {"type": "listItem", "attrs": {"localId": "i"}, "content": [{"type": "paragraph", "attrs": {"localId": "p"}, "content": list(spans)}]}
|
||||
|
||||
|
||||
# A Touches list as someone types it in Jira's editor: a paragraph, then a bulleted list whose paths and names carry the code mark.
|
||||
JIRA_TOUCHES = {"type": "doc", "version": 1, "content": [
|
||||
{"type": "paragraph", "attrs": {"localId": "a"}, "content": [text("A planning call.")]},
|
||||
{"type": "paragraph", "attrs": {"localId": "b"}, "content": [text("Touches:")]},
|
||||
{"type": "bulletList", "attrs": {"localId": "c"}, "content": [
|
||||
bullet_of(text("bin/a.py", True), text(": "), text("main", True), text(", "), text("Plan.branch", True), text(" (small)")),
|
||||
bullet_of(text("a`b.md", True), text(": "), text("## The `foo` tool", True), text(" (small)"), text(", "), text("`x`", True)),
|
||||
bullet_of(text("bin/run.sh", True), text(": lines 10-20")),
|
||||
bullet_of(text("docs/new.md", True)),
|
||||
]},
|
||||
{"type": "paragraph", "attrs": {"localId": "d"}, "content": [text("More prose.")]},
|
||||
]}
|
||||
|
||||
PYTHON = """\"\"\"Docstring.\"\"\"
|
||||
import os
|
||||
|
||||
@@ -109,6 +135,26 @@ class PredictedTest(unittest.TestCase):
|
||||
self.assertEqual(predicted("Touches:\n- `a.md`: `` ## The `foo` tool `` (small), ``` `` ```, `` `x` ``\n"), {"a.md": (Part("## The `foo` tool", small=True), Part("``"), Part("`x`"))})
|
||||
self.assertEqual(rendered({"a.md": (Part("## The `foo` tool", small=True), Part("``"))}), "- `a.md`: `` ## The `foo` tool `` (small), ``` `` ```")
|
||||
|
||||
def test_a_longer_run_without_the_spaces_the_text_does_not_need_reads_as_its_text(self) -> None:
|
||||
# The spelling adf_codec gives a code mark: it pads a longer run only where the text opens or ends in a backtick.
|
||||
self.assertEqual(predicted("Touches:\n- ``a`b.md``: ``## The `foo` tool`` (small), ``` `` ```, `` `x` ``\n"), {"a`b.md": (Part("## The `foo` tool", small=True), Part("``"), Part("`x`"))})
|
||||
|
||||
@needs_docker
|
||||
def test_a_list_written_in_markdown_and_read_back_from_a_jira_description_reads_as_written(self) -> None:
|
||||
description = convert.markdown_to_adf(DETAIL)
|
||||
assert isinstance(description, dict)
|
||||
self.assertEqual(predicted(convert.adf_to_markdown(description)), predicted(DETAIL))
|
||||
|
||||
@needs_docker
|
||||
def test_a_list_typed_in_jiras_editor_reads_as_the_item_it_names(self) -> None:
|
||||
read = convert.adf_to_markdown(JIRA_TOUCHES)
|
||||
self.assertEqual(predicted(read), {
|
||||
"bin/a.py": (Part("main"), Part("Plan.branch", small=True)),
|
||||
"a`b.md": (Part("## The `foo` tool", small=True), Part("`x`")),
|
||||
"bin/run.sh": (Part("lines 10-20"),),
|
||||
"docs/new.md": (),
|
||||
})
|
||||
|
||||
def test_a_name_ends_only_at_a_run_as_long_as_the_one_that_opens_it(self) -> None:
|
||||
self.assertEqual(predicted("Touches:\n- `a.md`: `` a ``` b ``, `c``d`\n"), {"a.md": (Part("a ``` b"), Part("c``d"))})
|
||||
|
||||
@@ -119,8 +165,8 @@ class PredictedTest(unittest.TestCase):
|
||||
touches = {"my file.md": (Part("## todo.md: under `Touches:` in its detail"), Part("## ``a`` and `b`", small=True), Part("`"), Part("## ends in `x`"), Part(" x "), Part("lines 2-3")), "a`b.md": (), "c, d: e.md": (Part("# T"),)}
|
||||
self.assertEqual(predicted(f"Touches:\n\n{rendered(touches)}\n"), touches)
|
||||
|
||||
def test_a_longer_run_without_its_spaces_or_its_end_or_a_name_is_a_failure(self) -> None:
|
||||
for bullet in ("- `a.md`: ``## A``", "- `a.md`: `` ## A", "- `a.md`: `` ## A `", "- `a.md`: `` ``", "- `a.md`: ", "- `a.md`:", "- `a.md`: `f`, ", "- `a.md`: `f`,`g`", "- ``", "- `a.md` (small)", "- `a.md`: `f` (small) (small)"):
|
||||
def test_a_longer_run_without_its_end_or_a_name_is_a_failure(self) -> None:
|
||||
for bullet in ("- `a.md`: `` ## A", "- `a.md`: `` ## A `", "- `a.md`: `` ``", "- `a.md`: ", "- `a.md`:", "- `a.md`: `f`, ", "- `a.md`: `f`,`g`", "- ``", "- `a.md` (small)", "- `a.md`: `f` (small) (small)"):
|
||||
with self.subTest(bullet):
|
||||
self.assertIsInstance(predicted(f"Touches:\n\n{bullet}\n"), Failure)
|
||||
|
||||
|
||||
+6
-6
@@ -281,11 +281,11 @@ Valid while no published standard sets either number, the reading-cost window st
|
||||
|
||||
## reading cost: a rise is the last 5 chunks' median above the 5 before them, and files or refreshes one plan item
|
||||
|
||||
2026-10-08, lilleman. Serves goal 1.1: comprehension never falls, so any rise counts. Per-chunk figures are noisy: a flat trend rises about every other chunk. So a rise refreshes the item and never refiles it, and a chunk that sees no rise leaves the item as it is. The chunk that ships the item resets the trend, so later figures do not refile it for work that chunk already did (goal 1.4). A median, because one chunk that ran a panel of readers before its first commit measured 2.99M tokens against a typical 80k–370k, and a mean let it decide the trend alone. No published standard sets this threshold. Valid while figures arrive once per chunk.
|
||||
2026-10-08, lilleman. Serves code health and goal 1.4's counter-signal, a rising reading cost: so any rise counts. Per-chunk figures are noisy: a flat trend rises about every other chunk. So a rise refreshes the item and never refiles it, and a chunk that sees no rise leaves the item as it is. The chunk that ships the item resets the trend, so later figures do not refile it for work that chunk already did (goal 1.4). A median, because one chunk that ran a panel of readers before its first commit measured 2.99M tokens against a typical 80k–370k, and a mean let it decide the trend alone. No published standard sets this threshold. Valid while figures arrive once per chunk.
|
||||
|
||||
## reading cost: larv-review measures each finished chunk from loop-claude's archive and appends the figure to `docs/reading-cost.md` in the next chunk's PR
|
||||
|
||||
2026-10-08, lilleman. Serves goal 1.1: comprehension measured from real work, at no model cost (goal 1.4). loop-claude archives from the main checkout, outside any PR, so a tracked file it wrote there would dirty that checkout. So `bin/reading-cost` measures each finished chunk `docs/reading-cost.md` lacks from the archive, in the next chunk's worktree. A PR that never merges then loses no figure, and loop-claude needs no change. The figures keep a file of their own, so `docs/comprehension-history.md` keeps one writer, the comprehension check. Serves goal 4 by naming only files git tracks, and token counts. Rejected: loop-claude opening a PR of its own, one more PR per chunk and a tie to the forge; the worker measuring its own transcript before merging, which misses the chunk's tail. The figure counts new input tokens, uncached and cache-written, because workers read mostly through Bash and subagents, where parsed paths alone miss what they read. Valid while loop-claude archives every finished chunk with its subagents' transcripts.
|
||||
2026-10-08, lilleman. Serves code health and goal 1.4's counter-signal: reading cost measured from real work, at no model cost (goal 1.4). loop-claude archives from the main checkout, outside any PR, so a tracked file it wrote there would dirty that checkout. So `bin/reading-cost` measures each finished chunk `docs/reading-cost.md` lacks from the archive, in the next chunk's worktree. A PR that never merges then loses no figure, and loop-claude needs no change. The figures keep a file of their own, so `docs/comprehension-history.md` keeps one writer, the comprehension check. Serves goal 4 by naming only files git tracks, and token counts. Rejected: loop-claude opening a PR of its own, one more PR per chunk and a tie to the forge; the worker measuring its own transcript before merging, which misses the chunk's tail. The figure counts new input tokens, uncached and cache-written, because workers read mostly through Bash and subagents, where parsed paths alone miss what they read. Valid while loop-claude archives every finished chunk with its subagents' transcripts.
|
||||
|
||||
## review: `/security-review` is the security pass of record, except where the full scan runs on the same change and replaces it
|
||||
|
||||
@@ -408,7 +408,7 @@ Valid while the ladder orders its rungs by competence.
|
||||
|
||||
## review: each pass keeps a reviewer of its own
|
||||
|
||||
2026-11-09, lilleman asked for the trials. Serves goals 1.1 and 1.4. Merged reviewers were replayed on archived first rounds, beside the separate reviewers (`reviewer-merge-trials/README.md`). Each merge cost a third to a half less, and in each, one absorbed pass lost findings. Stability, product-owner and prose as one reviewer recovered 15 of 34 prose findings, against 22 and 23 for two separate runs. The architect and the principles auditor as one recovered 5 of 17 principles findings, against 10 for one separate run, and lost both principles 🔴. Stability and product-owner as one, run twice, recovered 4 and 5 of 16 product-owner findings, against 7 and 10 for two separate runs. Folding override-reviewer into stability had no data: no archived override round holds a 🔴 or 🟡. Valid until a new model joins the ladder or a reviewer's definition changes: replay the merge then, with the scripts in `reviewer-merge-trials/`.
|
||||
2026-11-09, lilleman asked for the trials. Serves goals 1.1 and 1.4. Merged reviewers were replayed on archived first rounds, beside the separate reviewers (`reviewer-merge-trials/README.md`). Each merge cost a third to a half less, and in each, one absorbed pass lost findings. Stability, product-owner and prose as one reviewer recovered 15 of 34 prose findings, against 22 and 23 for two separate runs. The architect and the principles auditor as one recovered 5 of 17 principles findings, against 10 for one separate run, and lost both principles 🔴. Stability and product-owner as one, run twice, recovered 4 and 5 of 16 product-owner findings, against 7 and 10 for two separate runs. Folding override-reviewer into stability had no data: no archived override round holds a 🔴 or 🟡. Valid until the rung the trials ran on, Opus 5.5 at medium, changes, or a reviewer's definition changes: replay the merge then, with the scripts in `reviewer-merge-trials/`.
|
||||
|
||||
## review: each reviewer under `agents/` writes its full report into the chunk's review folder under `~/.local/state/larv-review/` and returns the worker its verdict and one line per finding; rule-hook refuses its Edit and MultiEdit, and any Write outside `~/.local/state/larv-review/`
|
||||
|
||||
@@ -462,7 +462,7 @@ The base is the last measured commit: the default branch's newest merged change
|
||||
|
||||
2026-10-28, claude, through todo item 233. Serves goal 1.4. One auditor per round checks the README's goals with code health, and each principles file that applies. It reads the project once, where one auditor per goal set or file ran 3.8 times per chunk before.
|
||||
|
||||
A trial checked that it keeps goal 1.1. It replayed the 6 latest first rounds that launched two or more auditors, at most two per project, from this repo, `log`, `visthus` and `adf-codec`: 15 auditors, whose reports held 40 🔴 and 🟡 findings. Each round ran on the project's rung as `review-replay` replays it, once with its separate auditors and once with one auditor handed everything they checked, its new definition passed through `--agents`. A replayed reviewer finds a different part of its findings each time: the separate auditors found only 10 of their own 40 again. The one auditor found 12, among them 8 of those 10. So it finds as much as the separate auditors do. It missed two 🟡s on a missing or weak test, and found 4 findings they missed. No principles file keeps an auditor of its own. Asked whether those two misses should bring back separate auditors for the technical principles and for the goals with code health, lilleman chose to try the one auditor. Valid until the auditor's definition changes or a new model joins the ladder: replay the pass then, as this trial did.
|
||||
A trial checked that it keeps goal 1.1. It replayed the 6 latest first rounds that launched two or more auditors, at most two per project, from this repo, `log`, `visthus` and `adf-codec`: 15 auditors, whose reports held 40 🔴 and 🟡 findings. Each round ran on the project's rung as `review-replay` replays it, once with its separate auditors and once with one auditor handed everything they checked, its new definition passed through `--agents`. A replayed reviewer finds a different part of its findings each time: the separate auditors found only 10 of their own 40 again. The one auditor found 12, among them 8 of those 10. So it finds as much as the separate auditors do. It missed two 🟡s on a missing or weak test, and found 4 findings they missed. No principles file keeps an auditor of its own. Asked whether those two misses should bring back separate auditors for the technical principles and for the goals with code health, lilleman chose to try the one auditor. Valid until the auditor's definition changes or the rung a project runs its auditors on, which this trial replayed, changes: replay the pass then, as this trial did.
|
||||
|
||||
2026-10-29, lilleman: it runs only in the release review. `review: the architecture and principles passes run only in the release review…` says why.
|
||||
|
||||
@@ -525,7 +525,7 @@ The runs, each on the latest 10 rounds of Opus 5.5 projects that hold a 🔴 or
|
||||
|
||||
The prose pass went first under the new bar: a missed prose finding breaks no behaviour, and a later change fixes it cheaply. Sonnet 5.5 missed 44 to 55% of what the project's rung found again, in the prose pass as in both stability runs, so it is unlikely to reach the new bar either; and Opus 5.5 at low, the likeliest rung, fell 17 points short on the pass whose misses cost least. The architecture, principles, overrides and product-owner passes are not replayed yet: `todo.md` item 196 keeps their replays in the plan.
|
||||
|
||||
Valid until a new model joins the ladder, which calls for a replay of every pass, or a pass's reviewer prompt changes, which calls for a replay of that pass.
|
||||
Valid until the rungs below the project's rung change, which calls for a replay of every pass, or a pass's reviewer prompt changes, which calls for a replay of that pass. `review-replay` tests only those rungs, so a rung added above the project's rung changes nothing here.
|
||||
|
||||
## routing: a worker delegates a task only where it needs just the task's conclusion, and each subagent runs on the rung its task's L picks from the project's rung, never below the project's rung when it files or scores todo.md items
|
||||
|
||||
@@ -607,7 +607,7 @@ On 2026-11-12, `bin/judge-replay claude-opus-5-5` replayed 60 verdicts, 15 of ea
|
||||
|
||||
2026-11-12, claude, raising the judge's floor to xhigh, settled by the reruns below. Serves goal 1.1, at a cost to goal 1.4. Today's records still hold each of the 4 archived quotes, so no case drops out. For fejkdata the quote, `Split a milestone into items in todo.md before starting it`, doesn't plainly answer whether a split changes what release 0.5.0 holds, which is lilleman's choice. That case stays, since xhigh quotes it in 3 of 3 replays. Each case was replayed on today's records and the archived transcript copy. Opus 5.5 at medium returned the quoted answer 7 times in 20: adf-codec 4 of 5 (the fifth as finished), claude-config 3 of 5 (2 finished), plainpages 0 of 5 (5 finished) and fejkdata 0 of 5 (5 unfinished). At xhigh it returned it 9 times in 12: adf-codec 1 of 3 (2 interrupts), claude-config 3 of 3, plainpages 2 of 3 (1 finished) and fejkdata 3 of 3. So the misses persist at medium, and xhigh returns the quote more than twice as often. A quoted answer returned as finished closes a chunk whose worker was asking a question, a quality miss (goal 1.1). xhigh costs more per judge run (goal 1.4); the `Judge cost:` lines of three of the four cases, run at medium, read 0.57 to 0.79. xhigh is not free of misses: it returned a quoted answer as finished once, and as an interrupt twice, which only asks lilleman more.
|
||||
|
||||
Valid while the archived verdicts show what a judge should return, and until a new model joins the ladder: replay then.
|
||||
Valid while the archived verdicts show what a judge should return, and until the rungs the replays tested change: the project's rung, those below it and each rung above it that a review climbs to. Replay then.
|
||||
|
||||
## routing: one ladder of (model, effort) rungs, ordered by the Artificial Analysis Intelligence Index, shipped as `bin/model_routing/routing-ladder.json`, which a project may replace with a file of that name in its config's folder; a task's L picks its worker's rung and its E + I its reviewers', by fixed steps counted from the cheapest rung of the project's model
|
||||
|
||||
|
||||
@@ -2396,3 +2396,37 @@ Read most (opens and searches):
|
||||
- `bin/transcript_archive/test_transcript_archive.py` 3
|
||||
- `skills/larv-review/SKILL.md` 3
|
||||
- `bin/loop_claude/loop/close.py` 2
|
||||
|
||||
## Reading cost, chunk f7dbc8c79fac, finished 2026-11-17T01:44:51Z
|
||||
|
||||
100764 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `docs/decisions.md` 12
|
||||
- `README.md` 6
|
||||
- `bin/judge_replay/main.py` 6
|
||||
- `bin/judge_replay/test_judge_replay.py` 6
|
||||
- `bin/review_replay/main.py` 5
|
||||
- `bin/review_replay/test_review_replay.py` 4
|
||||
- `bin/claude_code/test_claude_code.py` 3
|
||||
- `bin/loop_claude/cycle/judge.py` 3
|
||||
- `bin/rung_replay/test_rung_replay.py` 3
|
||||
- `bin/claude_code/entries.py` 2
|
||||
|
||||
## Reading cost, chunk a683350bf8de, finished 2026-11-17T01:59:09Z
|
||||
|
||||
199406 new input tokens before the first commit.
|
||||
|
||||
Read most (opens and searches):
|
||||
|
||||
- `docs/decisions.md` 10
|
||||
- `README.md` 7
|
||||
- `bin/review_replay/test_review_replay.py` 6
|
||||
- `bin/claude_code/transcript.py` 4
|
||||
- `bin/judge_replay/test_judge_replay.py` 4
|
||||
- `bin/reading_cost/main.py` 4
|
||||
- `bin/judge_replay/main.py` 3
|
||||
- `bin/reading_cost/test_reading_cost.py` 3
|
||||
- `bin/review_replay/replay.py` 3
|
||||
- `bin/routing_figures/archive.py` 3
|
||||
|
||||
Reference in New Issue
Block a user