Merge remote-tracking branch 'origin/main' into item-222
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
Decisions live in `docs/decisions.md`:
|
||||
|
||||
- bin/: an outcome is an entry of a `.outcomes.md` file in loop-claude's archive, a record is a document tools or people keep entries in, and a place is what a comprehension seat names
|
||||
- bin/: the tests rerun themselves on a `uv`-fetched Python at README Install's floor
|
||||
- bin/: the tools and their tests run on the host's Python, with no container
|
||||
- chunks: a chunk takes every unblocked item of its release that fires the same review passes, whatever its purpose, and an item above R 5 ships alone
|
||||
|
||||
@@ -112,11 +112,11 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
|
||||
|
||||
Test on the host, with `uv` on `PATH` to fetch mypy and the Python version Install names (`docs/decisions.md` → `bin/`): `python3 -m unittest discover -s bin -p 'test_*.py'`. Test the bench's git wrapper with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
|
||||
|
||||
To mutation-test a changed module, copy `bin/` to a scratch directory (`cp -r bin "$scratch"`): mutmut writes files beside its config. In the copy, name the module and its tests in a `setup.cfg`, as below for `score_outcomes/outcomes.py`. Leave out each test class that starts a tool's launcher through `subprocess`: a mutated module fails to load in a process mutmut did not start.
|
||||
To mutation-test a changed module, copy `bin/` to a scratch directory (`cp -r bin "$scratch"`): mutmut writes files beside its config. In the copy, name the module and its tests in a `setup.cfg`, as below for `score_outcomes/results.py`. Leave out each test class that starts a tool's launcher through `subprocess`: a mutated module fails to load in a process mutmut did not start.
|
||||
|
||||
```ini
|
||||
[mutmut]
|
||||
source_paths=score_outcomes/outcomes.py
|
||||
source_paths=score_outcomes/results.py
|
||||
also_copy=.
|
||||
pytest_add_cli_args_test_selection=score_outcomes/
|
||||
pytest_add_cli_args=-k
|
||||
|
||||
@@ -93,12 +93,12 @@ class Parse(unittest.TestCase):
|
||||
|
||||
class ClaudeReply(unittest.TestCase):
|
||||
@staticmethod
|
||||
def claude(*outcomes):
|
||||
def claude(*results):
|
||||
def run(args, cwd, prompt):
|
||||
outcome = outcomes[0]
|
||||
if isinstance(outcome, OSError):
|
||||
raise outcome
|
||||
return subprocess.CompletedProcess(args, *outcome)
|
||||
result = results[0]
|
||||
if isinstance(result, OSError):
|
||||
raise result
|
||||
return subprocess.CompletedProcess(args, *result)
|
||||
return run
|
||||
|
||||
def test_runs_claude_headless_with_the_args_in_cwd_on_the_prompt(self):
|
||||
|
||||
@@ -31,7 +31,7 @@ def trust(config: pathlib.Path, roots: list[str]) -> None:
|
||||
def mark_trusted(config: object, roots: list[str]) -> tuple[dict, list[str]] | Failure:
|
||||
if not isinstance(config, dict) or not isinstance(projects := config.get("projects", {}), dict):
|
||||
return Failure('not an object whose "projects" is an object')
|
||||
if (wrong := next((path for path, record in projects.items() if not isinstance(record, dict)), None)) is not None:
|
||||
if (wrong := next((path for path, settings in projects.items() if not isinstance(settings, dict)), None)) is not None:
|
||||
return Failure(f"projects[{json.dumps(wrong)}] is not an object")
|
||||
untrusted = [path for path in roots if not projects.get(path, {}).get("hasTrustDialogAccepted")]
|
||||
marked = {path: {**projects.get(path, {}), "hasTrustDialogAccepted": True} for path in untrusted}
|
||||
|
||||
@@ -16,12 +16,12 @@ Kind = str
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Case:
|
||||
"""One archived verdict: on_record, judged at its UTC stamp."""
|
||||
"""One archived verdict: the kind the archive gives it, judged at its UTC stamp."""
|
||||
root: str
|
||||
task: str
|
||||
transcript: pathlib.Path
|
||||
at: str
|
||||
on_record: Kind
|
||||
archived_kind: Kind
|
||||
|
||||
|
||||
def case_from(root: str, outcomes: str, transcript: pathlib.Path) -> Case | None:
|
||||
@@ -40,8 +40,8 @@ def sample(cases: list[Case], per_kind: int) -> list[Case]:
|
||||
seen: collections.Counter[Kind] = collections.Counter()
|
||||
kept = []
|
||||
for found in sorted(cases, key=lambda found: (found.at, found.root, str(found.transcript)), reverse=True):
|
||||
seen[found.on_record] += 1
|
||||
if seen[found.on_record] <= per_kind:
|
||||
seen[found.archived_kind] += 1
|
||||
if seen[found.archived_kind] <= per_kind:
|
||||
kept.append(found)
|
||||
return kept[::-1]
|
||||
|
||||
@@ -63,12 +63,12 @@ def verdict_kind(verdict: Verdict | Failure) -> Kind:
|
||||
|
||||
|
||||
def reproduced(case: Case, kind: Kind) -> list[Kind]:
|
||||
"""The kind on record, where the reply returns it: a rung is held only to a verdict the reference judge reproduces."""
|
||||
return [kind] if kind == case.on_record else []
|
||||
"""The archived kind, where the reply returns it: a rung is held only to a verdict the reference judge reproduces."""
|
||||
return [kind] if kind == case.archived_kind else []
|
||||
|
||||
|
||||
def report(model: str, reference: Rung, cases: list[Case], replay: Replay[Case, Kind, Kind]) -> str:
|
||||
kinds = collections.Counter(found.on_record for found in cases)
|
||||
kinds = collections.Counter(found.archived_kind for found in cases)
|
||||
unreproduced = [(found, kind) for found, kind in zip(cases, replay.on_reference) if not reproduced(found, kind)]
|
||||
lines = [
|
||||
f"Replayed {len(cases)} verdicts of {model} projects: {', '.join(f'{count} {kind}' for kind, count in sorted(kinds.items()))}.",
|
||||
@@ -86,4 +86,4 @@ def report(model: str, reference: Rung, cases: list[Case], replay: Replay[Case,
|
||||
|
||||
|
||||
def missed_lines(missed: list[tuple[Case, Kind]]) -> list[str]:
|
||||
return [f" {found.root} {found.at}: on record {found.on_record}, replayed {kind}" for found, kind in missed]
|
||||
return [f" {found.root} {found.at}: archived {found.archived_kind}, replayed {kind}" for found, kind in missed]
|
||||
|
||||
@@ -25,8 +25,8 @@ MIDDLE = Rung("claude-sonnet-5-5", "high")
|
||||
REFERENCE = Rung("claude-opus-5-5", "xhigh")
|
||||
|
||||
|
||||
def case(at, on_record):
|
||||
return Case("/code/p", "Do the next chunk.", TRANSCRIPT, at, on_record)
|
||||
def case(at, archived_kind):
|
||||
return Case("/code/p", "Do the next chunk.", TRANSCRIPT, at, archived_kind)
|
||||
|
||||
|
||||
class CaseFrom(unittest.TestCase):
|
||||
@@ -37,7 +37,7 @@ class CaseFrom(unittest.TestCase):
|
||||
def test_reads_each_kind_of_verdict(self):
|
||||
for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED):
|
||||
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT)
|
||||
self.assertEqual(found.on_record, verdict)
|
||||
self.assertEqual(found.archived_kind, verdict)
|
||||
|
||||
def test_skips_a_copy_with_no_verdict(self):
|
||||
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
|
||||
@@ -60,7 +60,7 @@ class VerdictKind(unittest.TestCase):
|
||||
|
||||
|
||||
class Reproduced(unittest.TestCase):
|
||||
def test_holds_a_rung_to_a_verdict_only_where_the_reference_returned_the_kind_on_record(self):
|
||||
def test_holds_a_rung_to_a_verdict_only_where_the_reference_returned_the_archived_kind(self):
|
||||
self.assertEqual(reproduced(case("2026-10-14T10:00:00Z", FINISHED), FINISHED), [FINISHED])
|
||||
self.assertEqual(reproduced(case("2026-10-14T10:00:00Z", FINISHED), INTERRUPTED), [])
|
||||
|
||||
@@ -72,10 +72,10 @@ class Report(unittest.TestCase):
|
||||
self.assertEqual(report("claude-opus-5-5", REFERENCE, cases, replay).splitlines(), [
|
||||
"Replayed 2 verdicts of claude-opus-5-5 projects: 1 Verdict: finished, 1 Verdict: interrupt.",
|
||||
"The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:",
|
||||
" /code/p 2026-10-14T10:00:00Z: on record Verdict: finished, replayed Verdict: unfinished",
|
||||
" /code/p 2026-10-14T10:00:00Z: archived Verdict: finished, replayed Verdict: unfinished",
|
||||
"claude-sonnet-5-5 at high effort missed 0.",
|
||||
"claude-haiku-5-5 at xhigh effort missed 1:",
|
||||
" /code/p 2026-10-14T10:00:01Z: on record Verdict: interrupt, replayed Verdict: finished",
|
||||
" /code/p 2026-10-14T10:00:01Z: archived Verdict: interrupt, replayed Verdict: finished",
|
||||
"Lowest rung that, like each rung above it, returns every verdict the reference judge reproduces: claude-sonnet-5-5 at high effort",
|
||||
])
|
||||
missed_all = report("claude-opus-5-5", REFERENCE, cases, Replay([FINISHED, INTERRUPTED], [Tried(MIDDLE, [Miss(cases[1], INTERRUPTED, FINISHED)])], None)).splitlines()
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
from failure import Failure
|
||||
from git_repo import file_at, full_history, git
|
||||
from comprehension_history import HISTORY, HistoryChange
|
||||
|
||||
|
||||
def history_changes(default: str) -> list[HistoryChange] | Failure:
|
||||
"""Each merged change on the default branch that touched the history, newest first; a merge commit counts as one change against its first parent."""
|
||||
if partial := full_history():
|
||||
return partial
|
||||
commits = git("log", "--first-parent", "--format=%H", default, "--", HISTORY)
|
||||
if isinstance(commits, Failure):
|
||||
return commits
|
||||
changes = []
|
||||
for commit in commits.split():
|
||||
before = file_at(f"{commit}^1", HISTORY)
|
||||
if isinstance(before, Failure):
|
||||
return before
|
||||
after = file_at(commit, HISTORY)
|
||||
if isinstance(after, Failure):
|
||||
return after
|
||||
changes.append(HistoryChange(commit, before, after))
|
||||
return changes
|
||||
@@ -0,0 +1,30 @@
|
||||
from failure import Failure
|
||||
from git_repo import file_at, full_history, git
|
||||
from comprehension_history import HISTORY, HistoryChange
|
||||
|
||||
|
||||
def history_changes(default: str) -> list[HistoryChange] | Failure:
|
||||
"""Each merged change on the default branch that touched the history, newest first."""
|
||||
if partial := full_history():
|
||||
return partial
|
||||
commits = git("log", "--first-parent", "--format=%H", default, "--", HISTORY)
|
||||
if isinstance(commits, Failure):
|
||||
return commits
|
||||
changes = []
|
||||
for commit in commits.split():
|
||||
change = history_change(commit)
|
||||
if isinstance(change, Failure):
|
||||
return change
|
||||
changes.append(change)
|
||||
return changes
|
||||
|
||||
|
||||
def history_change(commit: str) -> HistoryChange | Failure:
|
||||
"""The history before and after commit. A merge commit counts as one change, against its first parent."""
|
||||
before = file_at(f"{commit}^1", HISTORY)
|
||||
if isinstance(before, Failure):
|
||||
return before
|
||||
after = file_at(commit, HISTORY)
|
||||
if isinstance(after, Failure):
|
||||
return after
|
||||
return HistoryChange(commit, before, after)
|
||||
@@ -77,7 +77,7 @@ def scored_seats(given: SeatsFile) -> ScoringRun | Failure:
|
||||
|
||||
|
||||
def read_seats(path: pathlib.Path, paired: bool) -> tuple[Heading, SeatsFile] | Failure:
|
||||
"""The seats file and the heading its record goes under. Fails when the file names a commit the checkout lacks, or when the history already holds an entry on what it reads; it runs before anything is printed."""
|
||||
"""The seats file and the heading its entry goes under. Fails when the file names a commit the checkout lacks, or when the history already holds an entry on what it reads; it runs before anything is printed."""
|
||||
try:
|
||||
given = seats_file(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
@@ -86,7 +86,7 @@ and the inherited architect vote alike overall in a paired ruling more often tha
|
||||
same way in a pair. It prints both counts. It opens `Unchecked:` until the history holds {pairs} such
|
||||
pairs and one paired ruling, then `Assumption holds:` or `Assumption fails:`, and exits 0 each time.
|
||||
|
||||
Each record is appended to the history, created when missing, under a heading of the current UTC
|
||||
Each entry is appended to the history, created when missing, under a heading of the current UTC
|
||||
time and the seats file's first line. Each commit that line names must be in the checkout. It exits 1
|
||||
when the history already holds an entry on what that line reads.
|
||||
|
||||
@@ -105,12 +105,12 @@ from round_titles import CAP, round_counts
|
||||
from git_repo import default_branch, git, top
|
||||
from failure import Failure
|
||||
from comprehension_history import FLOOR
|
||||
from larv_review.git import history_changes
|
||||
from larv_review.history_changes import history_changes
|
||||
from larv_review.ruling_base import measured_commit
|
||||
from larv_review.agreement import MIN_PAIRS
|
||||
from review_passes import PASSES
|
||||
from larv_review.plan import plan_reasons
|
||||
from larv_review.record import record_ruling, record_scoring, show_agreement, show_places
|
||||
from larv_review.history_commands import record_ruling, record_scoring, show_agreement, show_places
|
||||
from review_folder import REPORTS, review_folder
|
||||
|
||||
USAGE = "usage: larv-review review-folder | ruling-base | round-count | needs-scoring-run [<item ID>...] | record-ruling <votes file> | record-scoring <scores file> [<item ID>...] | places | agreement"
|
||||
|
||||
@@ -77,7 +77,7 @@ def ruling_step_up(cycle: Cycle, verdict: Verdict) -> StepUp | None:
|
||||
|
||||
def stopped(cycle: Cycle, stop: Stop, why: str) -> Stop:
|
||||
path = transcript_file(cycle.session)
|
||||
failure = archive(cycle.project_archive, path, cycle.task, outcomes.stopped(cycle.route.record(), why), cycle.effects.now()) if path else None
|
||||
failure = archive(cycle.project_archive, path, cycle.task, outcomes.stopped(cycle.route.worker_line(), why), cycle.effects.now()) if path else None
|
||||
# A failed archive exits 1 whatever the stop; when the stop was this archive failing, its reason is said once.
|
||||
return Stop(1, f"{failure.reason}\n{stop.reason}".rstrip()) if failure and failure.reason != stop.reason else stop
|
||||
|
||||
@@ -135,7 +135,7 @@ def keep_answer(cycle: Cycle, answered: QuotedAnswer, step_up: StepUp | None) ->
|
||||
if not path:
|
||||
return None
|
||||
# As in ask: the answer names the rung the worker ran on until then, and a step-up archived after it names the rung it runs on from then.
|
||||
quoted = outcomes.answered(answered.question, prompts.QUOTE_SOURCES, answered.answer, answered.evidence, answered.escalation, str(judge_of(cycle)), ran_on(cycle, step_up).record())
|
||||
quoted = outcomes.answered(answered.question, prompts.QUOTE_SOURCES, answered.answer, answered.evidence, answered.escalation, str(judge_of(cycle)), ran_on(cycle, step_up).worker_line())
|
||||
if failure := archive_stepped(cycle, path, quoted, step_up):
|
||||
return Stop(1, failure.reason)
|
||||
return None
|
||||
@@ -160,7 +160,7 @@ def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer, st
|
||||
question = question_for(ruling)
|
||||
# Archived now, so the transcript the question was asked on is kept with it; the human's answer then grows the transcript.
|
||||
# The interrupt names the rung the worker ran on before it; a step-up archived after it names the stepped rung, so the figures charge each stretch to its own rung.
|
||||
interrupted = outcomes.interrupted(question, ruling.evidence, str(judge_of(cycle)), ruling.escalation, ran_on(cycle, step_up).record())
|
||||
interrupted = outcomes.interrupted(question, ruling.evidence, str(judge_of(cycle)), ruling.escalation, ran_on(cycle, step_up).worker_line())
|
||||
if failure := archive_stepped(cycle, transcript_path, interrupted, step_up):
|
||||
return Stop(1, failure.reason)
|
||||
# Only the human may cut a release, so a release ready to cut opens claude's session at once.
|
||||
@@ -178,7 +178,7 @@ def ran_on(cycle: Cycle, step_up: StepUp | None) -> Route:
|
||||
|
||||
def archive_stepped(cycle: Cycle, transcript_path: str, outcome: str, step_up: StepUp | None) -> Failure | None:
|
||||
"""Archives the outcome of a ruling, then the step-up it made, if it made one. The first failure is returned."""
|
||||
for kept in (outcome, outcomes.stepped_up(step_up.after.record())) if step_up else (outcome,):
|
||||
for kept in (outcome, outcomes.stepped_up(step_up.after.worker_line())) if step_up else (outcome,):
|
||||
if failure := archive(cycle.project_archive, transcript_path, cycle.task, kept, cycle.effects.now()):
|
||||
return failure
|
||||
return None
|
||||
@@ -191,7 +191,7 @@ def question_for(ruling: Interrupt | QuotedAnswer) -> str:
|
||||
|
||||
|
||||
def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) -> Archived | Stop:
|
||||
outcome = outcomes.finished(cycle.route.record(), verdict.evidence, str(judge_of(cycle))) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.record(), verdict.evidence, str(judge_of(cycle)), verdict.escalation)
|
||||
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle))) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation)
|
||||
if failure := archive(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects.now()):
|
||||
return Stop(1, failure.reason)
|
||||
if isinstance(verdict, Finished) and (failure := delete_session(cycle.session)):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""The escalation mark makes one escalation trigger step the worker up one rung, never two (docs/decisions.md → `routing: a worker whose run ends asking for a higher rung`).
|
||||
|
||||
A worker whose run ends asking loop-claude for a higher rung cites the trigger that fired, and the judge quotes the passage where it fired as the verdict's escalation. The mark, Route.escalated_at, is how many transcript entries the session held when an escalation last stepped the worker up, 0 before any. The session's transcript only grows, so the mark holds across cycles, and the archive's Worker: records keep it for a rerun.
|
||||
A worker whose run ends asking loop-claude for a higher rung cites the trigger that fired, and the judge quotes the passage where it fired as the verdict's escalation. The mark, Route.escalated_at, is how many transcript entries the session held when an escalation last stepped the worker up, 0 before any. The session's transcript only grows, so the mark holds across cycles, and the archive's Worker: lines keep it for a rerun.
|
||||
|
||||
- Stamp. stamped() writes on the judge's verdict how many entries the judge read.
|
||||
- Step. routing.after_unfinished() and routing.after_ruling() step the worker up on an escalation and move the mark to moved_to(verdict). A ruling's StepUp travels through the cycle's states, so the archive and the worker's next prompt name it.
|
||||
|
||||
@@ -62,7 +62,7 @@ the judge finds in the transcript an escalation trigger that fired and the worke
|
||||
rung. The rules are in bin/loop_claude/routing.py and bin/model_routing/ladder.py, the choice of
|
||||
each review's rung in bin/model_routing/review_choice.py, and the prompts' wording in
|
||||
bin/loop_claude/cycle/prompts.py. Each verdict, interrupt, quoted answer, stop and
|
||||
step-up in the archive ends with a Worker: line, the route's JSON record (Route.record in
|
||||
step-up in the archive ends with a Worker: line, the route as JSON (Route.worker_line in
|
||||
bin/model_routing/route.py); a rerun resumes an open chunk on the latest route named there. A
|
||||
todo.md or an archive that cannot be read or parsed is said on stderr, and the chunk is routed
|
||||
without it.
|
||||
@@ -217,11 +217,11 @@ def stepped_route(project: Rung | None, ended_on: Route, verdict: Unfinished) ->
|
||||
|
||||
|
||||
def archive_step(archive_folder: pathlib.Path, session: Session, task: str, before: Route, after: Route | None, effects: Effects) -> Stop | None:
|
||||
"""Archives the route after an unfinished verdict as a step-up, where its record differs from the route before it: a higher rung, or a higher E + I taken in. The verdict itself names the route before, which its cycle ran on."""
|
||||
if after is None or after.record() == before.record():
|
||||
"""Archives the route after an unfinished verdict as a step-up, where its worker line differs from the one before it: a higher rung, or a higher E + I taken in. The verdict itself names the route before, which its cycle ran on."""
|
||||
if after is None or after.worker_line() == before.worker_line():
|
||||
return None
|
||||
path = transcript_file(session)
|
||||
failure = archive(archive_folder, path, task, outcomes.stepped_up(after.record()), effects.now()) if path else None
|
||||
failure = archive(archive_folder, path, task, outcomes.stepped_up(after.worker_line()), effects.now()) if path else None
|
||||
return Stop(1, failure.reason) if failure else None
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ from loop_claude.loop import main, reload, routing_check
|
||||
from machine_load import memory
|
||||
from transcript_archive import outcomes, write
|
||||
from transcript_archive.location import project_archive
|
||||
from model_routing.route import Record, Route, route_from
|
||||
from model_routing.route import Route, route_from, WorkerLine
|
||||
from loop_claude.stop import Stop
|
||||
from utc_stamp import utc
|
||||
from loop_claude.cycle.judge import verdict_from
|
||||
@@ -918,24 +918,24 @@ class Routing(unittest.TestCase):
|
||||
without_i = TODO.replace("| E | I | R |", "| E | R |").replace("|---|---|---|---|---|---|---|---|---|---|---|", "|---|---|---|---|---|---|---|---|---|---|").replace("| 4 | 3 | 5 | 10 |", "| 4 | 3 | 10 |").replace("| 4 | 2 | 3 | 7 |", "| 4 | 2 | 7 |").replace("| 1 | 1 | 1 | 1 |", "| 1 | 1 | 1 |")
|
||||
self.assertIsNone(routing.routed_item(without_i, NO_CUT))
|
||||
|
||||
def test_archives_the_route_as_a_json_record_it_reads_back_field_by_field(self):
|
||||
def test_archives_the_route_as_a_json_worker_line_it_reads_back_field_by_field(self):
|
||||
route = Route(self.HIGH, 3, 4, 5)
|
||||
self.assertEqual(route.record(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5}')
|
||||
self.assertEqual(route_from(route.record()), route)
|
||||
self.assertEqual(Route(Rung("worker-model")).record(), '{"model": "worker-model"}')
|
||||
self.assertEqual(route.worker_line(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5}')
|
||||
self.assertEqual(route_from(route.worker_line()), route)
|
||||
self.assertEqual(Route(Rung("worker-model")).worker_line(), '{"model": "worker-model"}')
|
||||
self.assertEqual(route_from('{"model": "worker-model"}'), Route(Rung("worker-model")))
|
||||
marked = Route(self.HIGH, 3, 4, 5, escape_and_impact_taken_in=7, escalated_at=12)
|
||||
self.assertEqual(marked.record(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5, "escape_and_impact_taken_in": 7, "escalated_at": 12}')
|
||||
self.assertEqual(route_from(marked.record()), marked)
|
||||
self.assertEqual(route_from(Route(self.HIGH, escalated_at=3).record()), Route(self.HIGH, escalated_at=3))
|
||||
self.assertEqual(marked.worker_line(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5, "escape_and_impact_taken_in": 7, "escalated_at": 12}')
|
||||
self.assertEqual(route_from(marked.worker_line()), marked)
|
||||
self.assertEqual(route_from(Route(self.HIGH, escalated_at=3).worker_line()), Route(self.HIGH, escalated_at=3))
|
||||
|
||||
def test_leaves_the_count_of_unfinished_verdicts_out_of_the_record_so_a_rerun_restarts_it(self):
|
||||
self.assertEqual(route_from(Route(self.HIGH, 3, 4, 5, unfinished=1).record()), Route(self.HIGH, 3, 4, 5))
|
||||
def test_leaves_the_count_of_unfinished_verdicts_out_of_the_worker_line_so_a_rerun_restarts_it(self):
|
||||
self.assertEqual(route_from(Route(self.HIGH, 3, 4, 5, unfinished=1).worker_line()), Route(self.HIGH, 3, 4, 5))
|
||||
|
||||
def test_reads_no_route_from_a_record_of_unknown_or_mistyped_fields_or_levels_off_their_scales(self):
|
||||
for record in ('{"effort": "high"}', '{"model": "claude-opus-5-5", "effort": "high", "unfinished": 1}', '{"model": "claude-opus-5-5", "item": "3"}', '{"model": "claude-opus-5-5", "effort": 3}', '{"model": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escalated_at": -3}', '{"model": "claude-opus-5-5", "item": true, "likelihood": 4, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 5, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escape_and_impact_taken_in": 9}', '["claude-opus-5-5"]', '{"model": '):
|
||||
with self.subTest(record=record):
|
||||
self.assertIsNone(route_from(record))
|
||||
def test_reads_no_route_from_a_worker_line_of_unknown_or_mistyped_fields_or_levels_off_their_scales(self):
|
||||
for line in ('{"effort": "high"}', '{"model": "claude-opus-5-5", "effort": "high", "unfinished": 1}', '{"model": "claude-opus-5-5", "item": "3"}', '{"model": "claude-opus-5-5", "effort": 3}', '{"model": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escalated_at": -3}', '{"model": "claude-opus-5-5", "item": true, "likelihood": 4, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 5, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escape_and_impact_taken_in": 9}', '["claude-opus-5-5"]', '{"model": '):
|
||||
with self.subTest(line=line):
|
||||
self.assertIsNone(route_from(line))
|
||||
|
||||
def test_reads_the_route_from_a_line_loop_claude_wrote_before_it_archived_json(self):
|
||||
self.assertEqual(route_from("claude-opus-5-5 at high effort, routed by item 3's L of 4 and E + I of 5"), Route(self.HIGH, 3, 4, 5))
|
||||
@@ -985,12 +985,12 @@ class RouteSources(unittest.TestCase):
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
self.assertIsNone(main.archived_route(archive, opened))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(route.record(), "working", "j", ""), NOW))
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(route.worker_line(), "working", "j", ""), NOW))
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
self.assertEqual(main.archived_route(archive, opened), route)
|
||||
self.assertEqual(stderr.getvalue(), "")
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.stepped_up(Record('{"model": "claude-opus-5-5", "rung": 3}')), NOW + 1))
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.stepped_up(WorkerLine('{"model": "claude-opus-5-5", "rung": 3}')), NOW + 1))
|
||||
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
||||
self.assertIsNone(main.archived_route(archive, opened))
|
||||
self.assertEqual(stderr.getvalue(), 'routing the chunk afresh: the archive\'s latest Worker: line names no route loop-claude can read: {"model": "claude-opus-5-5", "rung": 3}\n')
|
||||
@@ -1036,7 +1036,7 @@ class Steps(unittest.TestCase):
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
before = Route(self.OPUS, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.record(), "working", "j", ""), NOW))
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
||||
after = main.after_verdict(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", 4), before))
|
||||
self.assertIsNone(main.archive_step(archive, opened, "Do it.", before, after.route, FakeEffects().effects()))
|
||||
self.assertEqual(main.archived_route(archive, opened), Route(Rung("claude-opus-5-5", "high"), 3, 3, 4))
|
||||
@@ -1058,7 +1058,7 @@ class Steps(unittest.TestCase):
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
before = Route(self.OPUS, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.record(), "working", "j", ""), NOW))
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
||||
after = main.after_verdict(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", highest_escape_and_impact_taken_in=6), before))
|
||||
self.assertIsNone(main.archive_step(archive, opened, "Do it.", before, after.route, FakeEffects().effects()))
|
||||
self.assertEqual(main.archived_route(archive, opened), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6))
|
||||
|
||||
@@ -36,7 +36,7 @@ class Rung:
|
||||
return Rung(model, effort)
|
||||
return Rung(family.model, effort) if effort else family
|
||||
|
||||
# The one spelling of a rung. loop-claude hands it to rule-hook in environment variables, and archive lines written before route.Route.record hold it, so parse must keep reading it.
|
||||
# The one spelling of a rung. loop-claude hands it to rule-hook in environment variables, and archive lines written before route.Route.worker_line hold it, so parse must keep reading it.
|
||||
def __str__(self) -> str:
|
||||
return f"{self.model} at {self.effort} effort" if self.effort else self.model
|
||||
|
||||
|
||||
+20
-20
@@ -1,4 +1,4 @@
|
||||
"""A chunk's route: its worker's rung and the plan item that picked it. The archive keeps the route as a record beside each verdict."""
|
||||
"""A chunk's route: its worker's rung and the plan item that picked it. The archive keeps the route as JSON on the Worker: line that ends each outcome."""
|
||||
import dataclasses
|
||||
import json
|
||||
import re
|
||||
@@ -9,9 +9,9 @@ from model_routing.ladder import REVIEW_STEPS, Rung
|
||||
|
||||
# How a worker opens the prompt of a subagent it escalates a task to, `Escalation trigger 2: …`: docs/decisions.md → `routing: a worker hands a task one rung up`.
|
||||
ESCALATION_MARKER = "Escalation trigger"
|
||||
# What Route.record writes. The outcome writers take only this as the worker, so mypy refuses a str(route) there.
|
||||
Record = typing.NewType("Record", str)
|
||||
# A worker line loop-claude wrote before it archived the route as a record. Nothing writes this format any more, so its wording never changes. route_from reads it for the chunks the routing figures still count, and for a chunk a rerun resumes across the change.
|
||||
# What Route.worker_line writes. The outcome writers take only this as the worker, so mypy refuses a str(route) there.
|
||||
WorkerLine = typing.NewType("WorkerLine", str)
|
||||
# A worker line loop-claude wrote before it archived the route as JSON. Nothing writes this format any more, so its wording never changes. route_from reads it for the chunks the routing figures still count, and for a chunk a rerun resumes across the change.
|
||||
LEGACY_WORKER = re.compile(r"(?P<rung>.+?)(?:, routed by item (?P<item>[0-9]+)'s L of (?P<likelihood>[0-9]+) and E \+ I of (?P<escape_and_impact>[0-9]+))?(?:, taking in an E \+ I of (?P<escape_and_impact_taken_in>[0-9]+))?(?:, escalated at transcript entry (?P<escalated_at>[0-9]+))?")
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ class Route:
|
||||
|
||||
unfinished counts the unfinished verdicts towards the next one-rung step; a reload hands it over with the rung.
|
||||
"""
|
||||
# Each field below but rung and unfinished is a key of the archive's records, so renaming one makes the records already archived unreadable, and their chunks route afresh.
|
||||
# Each field below but rung and unfinished is a key of the archive's worker lines, so renaming one makes the lines already archived unreadable, and their chunks route afresh.
|
||||
# The fields a loop-claude from before L and E + I replaced R + C hands on reload; the reloaded script drops such a route, and the chunk routes afresh.
|
||||
RETIRED: typing.ClassVar[frozenset[str]] = frozenset({"risk_and_complexity"})
|
||||
rung: Rung
|
||||
@@ -71,38 +71,38 @@ class Route:
|
||||
"""The E + I that picks the reviewers' and the judge's rung: the routed item's or the highest taken in, whichever is higher; 0 for neither."""
|
||||
return max(self.escape_and_impact, self.escape_and_impact_taken_in)
|
||||
|
||||
def record(self) -> Record:
|
||||
"""What the archive records as each verdict's worker: a JSON object of the rung's model and effort and the RECORD_FIELDS, leaving out each that is 0 or empty; route_from reads it back."""
|
||||
fields = {"model": self.rung.model, "effort": self.rung.effort, **{name: getattr(self, name) for name in RECORD_FIELDS}}
|
||||
return Record(json.dumps({name: value for name, value in fields.items() if value}))
|
||||
def worker_line(self) -> WorkerLine:
|
||||
"""What the archive records as each verdict's worker: a JSON object of the rung's model and effort and the WORKER_LINE_FIELDS, leaving out each that is 0 or empty; route_from reads it back."""
|
||||
fields = {"model": self.rung.model, "effort": self.rung.effort, **{name: getattr(self, name) for name in WORKER_LINE_FIELDS}}
|
||||
return WorkerLine(json.dumps({name: value for name, value in fields.items() if value}))
|
||||
|
||||
|
||||
# The Route fields a record holds beside the rung's model and effort. A human reads the record in the archive, so it is flatter than the JSON a reload hands over (loop_claude/loop/reload.py) and leaves out zeros. It leaves out unfinished, so a rerun restarts that count.
|
||||
RECORD_FIELDS = tuple(field.name for field in dataclasses.fields(Route) if field.name not in ("rung", "unfinished"))
|
||||
# The Route fields a worker line holds beside the rung's model and effort. A human reads the line in the archive, so it is flatter than the JSON a reload hands over (loop_claude/loop/reload.py) and leaves out zeros. It leaves out unfinished, so a rerun restarts that count.
|
||||
WORKER_LINE_FIELDS = tuple(field.name for field in dataclasses.fields(Route) if field.name not in ("rung", "unfinished"))
|
||||
|
||||
|
||||
def route_from(worker: str) -> Route | None:
|
||||
"""The route an archived worker line names, whether a record or a line written before records. None where it names no model, holds fields Route.record never writes, or holds levels off their scales; and for an older line, where it names a model without an effort."""
|
||||
fields = record_fields(worker) if worker.startswith("{") else legacy_fields(worker)
|
||||
"""The route an archived worker line names, whether JSON or a line written before JSON. None where it names no model, holds fields Route.worker_line never writes, or holds levels off their scales; and for an older line, where it names a model without an effort."""
|
||||
fields = json_fields(worker) if worker.startswith("{") else legacy_fields(worker)
|
||||
if fields is None or not Route.fits(fields):
|
||||
return None
|
||||
return Route(Rung(fields["model"], fields.get("effort", "")), **{name: fields.get(name, 0) for name in RECORD_FIELDS})
|
||||
return Route(Rung(fields["model"], fields.get("effort", "")), **{name: fields.get(name, 0) for name in WORKER_LINE_FIELDS})
|
||||
|
||||
|
||||
def record_fields(record: str) -> dict[str, typing.Any] | None:
|
||||
"""The fields of a record Route.record wrote; None for any other text."""
|
||||
def json_fields(line: str) -> dict[str, typing.Any] | None:
|
||||
"""The fields of a line Route.worker_line wrote; None for any other text."""
|
||||
try:
|
||||
fields = json.loads(record)
|
||||
fields = json.loads(line)
|
||||
except ValueError:
|
||||
return None
|
||||
if not (isinstance(fields, dict) and isinstance(fields.get("model"), str) and isinstance(fields.get("effort", ""), str) and fields.keys() <= {"model", "effort", *RECORD_FIELDS}):
|
||||
if not (isinstance(fields, dict) and isinstance(fields.get("model"), str) and isinstance(fields.get("effort", ""), str) and fields.keys() <= {"model", "effort", *WORKER_LINE_FIELDS}):
|
||||
return None
|
||||
# type(), not isinstance(): JSON true is no integer.
|
||||
return fields if all(type(fields.get(name, 0)) is int and fields.get(name, 0) >= 0 for name in RECORD_FIELDS) else None
|
||||
return fields if all(type(fields.get(name, 0)) is int and fields.get(name, 0) >= 0 for name in WORKER_LINE_FIELDS) else None
|
||||
|
||||
|
||||
def legacy_fields(line: str) -> dict[str, typing.Any] | None:
|
||||
"""The fields of a LEGACY_WORKER line, as record_fields gives them; None for any other text."""
|
||||
"""The fields of a LEGACY_WORKER line, as json_fields gives them; None for any other text."""
|
||||
found = LEGACY_WORKER.fullmatch(line)
|
||||
rung = Rung.parse(found["rung"]) if found else None
|
||||
if not (found and rung):
|
||||
|
||||
@@ -9,8 +9,8 @@ from markdown_text import sections
|
||||
from utc_stamp import STAMPED
|
||||
|
||||
FIGURE = re.compile(rf"^Reading cost, chunk (\w+), finished ({STAMPED})$")
|
||||
FIGURE_PLACE = re.compile(r"^- `(.+)` (\d+)$")
|
||||
PLACES_KEPT = 10
|
||||
FIGURE_FILE = re.compile(r"^- `(.+)` (\d+)$")
|
||||
FILES_KEPT = 10
|
||||
RECORD = "docs/reading-cost.md"
|
||||
RESET = "Reading cost reset, "
|
||||
RESET_AT = re.compile(rf"^{RESET}({STAMPED})$")
|
||||
@@ -28,8 +28,8 @@ def chunk_id(chunk: FinishedChunk) -> str:
|
||||
|
||||
|
||||
def figure(chunk: FinishedChunk, reading: Reading) -> str:
|
||||
places = "".join(f"- `{path}` {times}\n" for path, times in ranked(reading.places))
|
||||
return f"## Reading cost, chunk {chunk_id(chunk)}, finished {chunk.finished}\n\n{reading.tokens} new input tokens before the first commit.\n" + (f"\nRead most (opens and searches):\n\n{places}" if places else "")
|
||||
files = "".join(f"- `{path}` {times}\n" for path, times in ranked(reading.files))
|
||||
return f"## Reading cost, chunk {chunk_id(chunk)}, finished {chunk.finished}\n\n{reading.tokens} new input tokens before the first commit.\n" + (f"\nRead most (opens and searches):\n\n{files}" if files else "")
|
||||
|
||||
|
||||
# The figures after the last reset, in the order their chunks finished: each machine appends its own archive's chunks.
|
||||
@@ -44,14 +44,14 @@ def readings_of(record: str) -> list[Reading] | Failure:
|
||||
tokens = [int(match.group(1)) for line in lines if (match := TOKENS.match(line))]
|
||||
if len(tokens) != 1:
|
||||
return Failure(f"the figure under \"## {heading}\" holds no single tokens line; restore it from git history")
|
||||
dated.append((named.group(2), heading, Reading(tokens[0], collections.Counter({match.group(1): int(match.group(2)) for line in lines if (match := FIGURE_PLACE.match(line))}))))
|
||||
dated.append((named.group(2), heading, Reading(tokens[0], collections.Counter({match.group(1): int(match.group(2)) for line in lines if (match := FIGURE_FILE.match(line))}))))
|
||||
elif heading.startswith("Reading cost"):
|
||||
return Failure(f"\"## {heading}\" names no chunk and UTC time, or no reset time; restore it from git history")
|
||||
return [reading for finished, _, reading in sorted(dated, key=lambda item: item[:2]) if finished > max(resets)]
|
||||
|
||||
|
||||
def ranked(places: collections.Counter[str]) -> list[tuple[str, int]]:
|
||||
return sorted(places.items(), key=lambda item: (-item[1], item[0]))[:PLACES_KEPT]
|
||||
def ranked(files: collections.Counter[str]) -> list[tuple[str, int]]:
|
||||
return sorted(files.items(), key=lambda item: (-item[1], item[0]))[:FILES_KEPT]
|
||||
|
||||
|
||||
def merged(record: str, figures: list[str]) -> str:
|
||||
@@ -65,7 +65,7 @@ def trend(readings: list[Reading]) -> str:
|
||||
medians = f"the last {WINDOW} chunks' median is {last} new input tokens before their first commit, the {WINDOW} before them {earlier}."
|
||||
if last <= earlier:
|
||||
return f"Not rising: {medians}"
|
||||
read_most = "".join(f"\n- `{path}` {times}" for path, times in ranked(sum((reading.places for reading in readings[-WINDOW:]), collections.Counter())))
|
||||
read_most = "".join(f"\n- `{path}` {times}" for path, times in ranked(sum((reading.files for reading in readings[-WINDOW:]), collections.Counter())))
|
||||
return f"Rising: {medians}\n\nRead most over the last {WINDOW} (opens and searches):\n{read_most}"
|
||||
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ from shell_command import resolved, simple_commands
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Reading:
|
||||
"""What the worker and its subagents took in before the worker's first commit."""
|
||||
"""What the worker and its subagents took in before the worker's first commit: tokens counts the new input tokens, and files counts how often each tracked file was opened or searched, by its path in the checkout."""
|
||||
tokens: int
|
||||
places: collections.Counter[str]
|
||||
files: collections.Counter[str]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -32,8 +32,8 @@ def measure(threads: list[bytes], checkout: Checkout, home: str) -> Reading | No
|
||||
before = [[entry for entry in thread if entry.get("type") == "assistant" and str(entry.get("timestamp", "")) < commit] for thread in (worker, *subagents)]
|
||||
reads = [reads_of(part, str(entry.get("cwd") or "/"), home) for thread in before for entry in thread for part in assistant_parts(entry)]
|
||||
roots = [checkout.main_root, *(added for found in reads for added in found.added)]
|
||||
places = collections.Counter(place for found in reads for path in found.read if (place := relative(path, roots)) in checkout.tracked)
|
||||
return Reading(sum(map(tokens_in, before)), places)
|
||||
files = collections.Counter(file for found in reads for path in found.read if (file := relative(path, roots)) in checkout.tracked)
|
||||
return Reading(sum(map(tokens_in, before)), files)
|
||||
|
||||
|
||||
def entries_of(raw: bytes) -> list[dict]:
|
||||
|
||||
@@ -16,11 +16,11 @@ folder, holding only the reports written after the chunk began and before the ro
|
||||
finds nothing where its prompt still names the review folder, or names a file there, other than a
|
||||
round's report, that the folder did not hold from the chunk's start until the round's.
|
||||
|
||||
It first replays every round on the project's rung, the reference reviewer. Findings on record
|
||||
It first replays every round on the project's rung, the reference reviewer. Archived findings
|
||||
that the reference reviewer does not raise again may rest on code or rules that have changed
|
||||
since; no rung is held to them. It then replays the rounds where the reference raised any, on each
|
||||
rung below, highest first, down to the ladder's lowest, and stops at the first rung that misses
|
||||
one. For each replay, claude on the project's rung matches the report to the findings on record,
|
||||
one. For each replay, claude on the project's rung matches the report to the archived findings,
|
||||
and counts a finding only where the report raises it as 🔴 or 🟡. The tool prints each finding a
|
||||
rung missed and the lowest rung that, like each rung above it, missed none. That rung is the
|
||||
evidence for running the pass's reviewer on it; the decision above records each run.
|
||||
@@ -150,7 +150,7 @@ def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Roun
|
||||
def code_of(found: Round, effects: Effects) -> Code | Failure:
|
||||
"""Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then.
|
||||
|
||||
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess costs no rung: the reference reviewer, replayed on that code, finds fewer findings on record, and no rung is held to those.
|
||||
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess costs no rung: the reference reviewer, replayed on that code, finds fewer of the archived findings, and no rung is held to those.
|
||||
"""
|
||||
if not os.path.isdir(found.root):
|
||||
return Failure("the project has no checkout")
|
||||
|
||||
@@ -125,8 +125,8 @@ def add(tokens: dict[str, collections.Counter[str]], project: str, kind: str, ru
|
||||
tokens["subagent type"][kind] += spent
|
||||
tokens["rung"][rung] += spent
|
||||
# Split whole, the first tools taking the remainder, so the tool table adds up to the others.
|
||||
for place, tool in enumerate(tools):
|
||||
tokens["tool"][tool] += spent // len(tools) + (place < spent % len(tools))
|
||||
for index, tool in enumerate(tools):
|
||||
tokens["tool"][tool] += spent // len(tools) + (index < spent % len(tools))
|
||||
|
||||
|
||||
def rung_at(cycles: list[tuple[str, str]], timestamp: str) -> Rung | str:
|
||||
|
||||
@@ -19,7 +19,7 @@ from model_routing.route import Route
|
||||
|
||||
LOW = Rung("claude-opus-5-5", "medium")
|
||||
HIGH = Rung("claude-opus-5-5", "high")
|
||||
LOW_ROUTE = Route(LOW, 7, 3, 4).record()
|
||||
LOW_ROUTE = Route(LOW, 7, 3, 4).worker_line()
|
||||
|
||||
|
||||
def message(id_, at, *parts, model="claude-opus-5-5", tokens=(1, 10, 100)):
|
||||
@@ -50,7 +50,7 @@ class Measured(unittest.TestCase):
|
||||
message("m2", "2026-10-14T10:05:00.000Z", {"type": "text", "text": "done"}),
|
||||
message("m3", "2026-10-14T11:00:00.000Z", {"type": "text", "text": "on"}),
|
||||
]
|
||||
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T10:30:00Z", outcomes.stepped_up(Route(HIGH).record())) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).record(), "merged", "j"))
|
||||
kept = stamped("2026-10-14T10:30:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T10:30:00Z", outcomes.stepped_up(Route(HIGH).worker_line())) + stamped("2026-10-14T12:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "merged", "j"))
|
||||
found = measured(Chunk("log", [kept], worker, []), "/h")
|
||||
self.assertEqual(found.tokens["project"], {"log": 333})
|
||||
self.assertEqual(found.tokens["subagent type"], {"worker": 333})
|
||||
@@ -93,14 +93,14 @@ class Measured(unittest.TestCase):
|
||||
self.assertEqual(found.tokens["rung"], {str(LOW): 111})
|
||||
|
||||
def test_starts_no_route_for_a_chunk_no_item_routed(self):
|
||||
kept = stamped("2026-10-14T10:00:00Z", outcomes.finished(Route(LOW).record(), "m", "j"))
|
||||
kept = stamped("2026-10-14T10:00:00Z", outcomes.finished(Route(LOW).worker_line(), "m", "j"))
|
||||
found = measured(Chunk("log", [kept], [message("m1", "2026-10-14T09:00:00Z")], []), "/h")
|
||||
self.assertIsNone(found.start)
|
||||
self.assertEqual(found.tokens["rung"], {str(LOW): 111})
|
||||
|
||||
|
||||
def test_puts_a_subagent_on_the_model_it_ran_at_the_effort_of_the_cycle_that_launched_it(self):
|
||||
kept = stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T11:00:00Z", outcomes.finished(Route(HIGH).record(), "m", "j"))
|
||||
kept = stamped("2026-10-14T10:00:00Z", outcomes.unfinished(LOW_ROUTE, "e", "j", "")) + stamped("2026-10-14T11:00:00Z", outcomes.finished(Route(HIGH).worker_line(), "m", "j"))
|
||||
worker = [
|
||||
message("m1", "2026-10-14T10:00:00.900Z"),
|
||||
message("m2", "2026-10-14T10:30:00Z", call("Agent", "a1", subagent_type="Explore", model="sonnet")),
|
||||
@@ -212,7 +212,7 @@ class Archive(unittest.TestCase):
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.unfinished(LOW_ROUTE, "e", "j", ""), 1790000000))
|
||||
transcript.write_text(json.dumps(first) + "\n" + json.dumps(message("m2", "2099-01-01T00:00:00Z")) + "\n")
|
||||
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.finished(Route(HIGH).record(), "m", "j"), 4070908800))
|
||||
self.assertIsNone(write.archive(kept / "log", str(transcript), "t", outcomes.finished(Route(HIGH).worker_line(), "m", "j"), 4070908800))
|
||||
(found,) = archive.measure_archive(kept, "/h", "").chunks
|
||||
self.assertEqual((found.start, found.unfinished), (LOW, 1))
|
||||
self.assertEqual(found.tokens["rung"], {str(LOW): 111, str(HIGH): 111})
|
||||
|
||||
@@ -37,7 +37,7 @@ import time
|
||||
import typing
|
||||
|
||||
from rule_hook.call import applied, Call, FILE_TOOLS, parsed_call, text
|
||||
from rule_hook.rules import Breach, command_breaches, file_breaches, Outcome, rules_help, verdict
|
||||
from rule_hook.rules import Breach, check, Checked, command_breaches, file_breaches, rules_help
|
||||
from git_command import commits
|
||||
from failure import Failure
|
||||
from rule_hook.override_log import Log, log_path, logged, print_overrides, write_override
|
||||
@@ -67,7 +67,7 @@ def main(argv: list[str], stdin: str) -> int:
|
||||
if found is None:
|
||||
return 0
|
||||
written, what, breaches = found
|
||||
return answer(verdict(breaches, written, text(call.given, "file_path").endswith(".md"), call.tool == "Bash"), what, time.time())
|
||||
return answer(check(breaches, written, text(call.given, "file_path").endswith(".md"), call.tool == "Bash"), what, time.time())
|
||||
|
||||
|
||||
def breaches_of(call: Call, home: str, environ: typing.Mapping[str, str]) -> tuple[str, str, list[Breach]] | None:
|
||||
@@ -85,12 +85,12 @@ def breaches_of(call: Call, home: str, environ: typing.Mapping[str, str]) -> tup
|
||||
return added, path, report_breaches(call, path, home) + file_breaches(path, before, added, after)
|
||||
|
||||
|
||||
def answer(judged: Outcome, what: str, now: float) -> int:
|
||||
for message in judged.messages:
|
||||
def answer(checked: Checked, what: str, now: float) -> int:
|
||||
for message in checked.messages:
|
||||
print(message, file=sys.stderr)
|
||||
if judged.blocked:
|
||||
if checked.blocked:
|
||||
return 2
|
||||
logs = [(breach, reason, log_path(breach.directory)) for breach, reason in judged.overridden]
|
||||
logs = [(breach, reason, log_path(breach.directory)) for breach, reason in checked.overridden]
|
||||
# Resolve every log before writing any, so a call that ends blocked has logged nothing.
|
||||
unloggable = [f"{breach.rule}: {log.reason}" for breach, _, log in logs if isinstance(log, Failure)]
|
||||
if not unloggable:
|
||||
|
||||
@@ -92,7 +92,8 @@ class Breach:
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Outcome:
|
||||
class Checked:
|
||||
"""What rule-hook found in one tool call: the messages to show, whether it blocks the call, and the overrides to log."""
|
||||
messages: list[str]
|
||||
blocked: bool
|
||||
# Each breach an override let through, with its reason; logged only when the call goes through.
|
||||
@@ -156,7 +157,7 @@ def overrides(written: str) -> tuple[dict[str, str], list[str]]:
|
||||
return granted, [f"rule-override {rule}: {reason}" for rule, reason in given if rule not in RULES or not reason]
|
||||
|
||||
|
||||
def verdict(breaches: list[Breach], written: str, markdown: bool, bash: bool) -> Outcome:
|
||||
def check(breaches: list[Breach], written: str, markdown: bool, bash: bool) -> Checked:
|
||||
granted, malformed = overrides(written)
|
||||
blocking = [breach for breach in breaches if breach.rule not in granted]
|
||||
messages = [f"`{marker}` names no rule below, or gives no reason." for marker in malformed] if blocking else []
|
||||
@@ -168,7 +169,7 @@ def verdict(breaches: list[Breach], written: str, markdown: bool, bash: bool) ->
|
||||
continue
|
||||
marker = f"<!-- rule-override {rule}: <reason> -->" if markdown else f"# rule-override {rule}: <reason>"
|
||||
messages.append(f"Blocked by rule {rule}: {details}. The rule: {RULES[rule].holds}. Where it does not fit this case, add `{marker}` to {where}; larv-review rules on the reason.")
|
||||
return Outcome(messages, bool(blocking), [(breach, granted[breach.rule]) for breach in breaches if breach.rule in granted])
|
||||
return Checked(messages, bool(blocking), [(breach, granted[breach.rule]) for breach in breaches if breach.rule in granted])
|
||||
|
||||
|
||||
def command_breaches(command: str, simples: list[Simple], home: str) -> list[Breach]:
|
||||
|
||||
@@ -9,7 +9,7 @@ import unittest.mock
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from rule_hook.rules import blocks, Breach, command_breaches, file_breaches, overrides, verdict
|
||||
from rule_hook.rules import blocks, Breach, check, command_breaches, file_breaches, overrides
|
||||
from rule_hook.main import breaches_of
|
||||
from rule_hook.call import Call
|
||||
from git_command import Commit, commit_message, commits
|
||||
@@ -125,7 +125,7 @@ class CommandBreaches(unittest.TestCase):
|
||||
|
||||
def test_an_override_quoting_the_human_s_yes_lets_a_round_past_the_cap_through(self):
|
||||
command = "git commit -m 'Answer stability round 4: 0 🔴, 1 🟡; fix' # rule-override round-cap: the human answered 'yes, one more'"
|
||||
self.assertFalse(verdict(command_breaches(command, simple_commands(command, "/r", "/h"), "/h"), command, False, True).blocked)
|
||||
self.assertFalse(check(command_breaches(command, simple_commands(command, "/r", "/h"), "/h"), command, False, True).blocked)
|
||||
|
||||
def test_an_attribution_breaks_only_in_a_git_gh_or_tea_command(self):
|
||||
self.assertEqual(checked(f"tea pr create --description '{TRAILER}'"), ["no-ai-attribution"])
|
||||
@@ -182,13 +182,13 @@ class Overrides(unittest.TestCase):
|
||||
|
||||
def test_a_blocked_call_overrides_nothing_yet(self):
|
||||
breaches = [Breach("image-pin", "x", "/r"), Breach("action-pin", "y", "/r")]
|
||||
judged = verdict(breaches, "# rule-override image-pin: a", False, False)
|
||||
self.assertTrue(judged.blocked)
|
||||
passed = verdict(breaches[:1], "# rule-override image-pin: a", False, False)
|
||||
blocked_call = check(breaches, "# rule-override image-pin: a", False, False)
|
||||
self.assertTrue(blocked_call.blocked)
|
||||
passed = check(breaches[:1], "# rule-override image-pin: a", False, False)
|
||||
self.assertEqual((passed.blocked, passed.overridden), (False, [(breaches[0], "a")]))
|
||||
|
||||
def test_a_markdown_file_is_told_the_html_comment(self):
|
||||
self.assertIn("`<!-- rule-override no-ai-attribution: <reason> -->`", verdict([Breach("no-ai-attribution", "x", "/r")], "", True, False).messages[0])
|
||||
self.assertIn("`<!-- rule-override no-ai-attribution: <reason> -->`", check([Breach("no-ai-attribution", "x", "/r")], "", True, False).messages[0])
|
||||
|
||||
|
||||
class OverrideLog(unittest.TestCase):
|
||||
|
||||
@@ -13,7 +13,7 @@ from model_routing.ladder import Rung
|
||||
AT_ONCE = 4
|
||||
|
||||
Case = typing.TypeVar("Case")
|
||||
# What of a case's record a reply reproduces, such as a verdict's kind or a finding's index.
|
||||
# What of an archived case a reply reproduces, such as a verdict's kind or a finding's index.
|
||||
Item = typing.TypeVar("Item")
|
||||
Reply = typing.TypeVar("Reply")
|
||||
T = typing.TypeVar("T")
|
||||
|
||||
@@ -2,7 +2,7 @@ import re
|
||||
|
||||
from todo_table import cells, LEVELS, parse_ids
|
||||
from failure import Failure
|
||||
from score_outcomes.outcomes import Outcome, render
|
||||
from score_outcomes.results import render, Result
|
||||
|
||||
ANCHORS_PATH = "docs/score-anchors.md"
|
||||
# A level, such as `L 1`; every scale tops out below 10.
|
||||
@@ -29,13 +29,13 @@ def parse_anchors(text: str) -> Anchors | Failure:
|
||||
return found
|
||||
|
||||
|
||||
def anchor_lines(found: Anchors | Failure, outcomes: list[Outcome]) -> list[str]:
|
||||
def anchor_lines(found: Anchors | Failure, results: list[Result]) -> list[str]:
|
||||
"""Each level, then each of its anchors with what its chunk measured. Where the file cannot be read, one line saying why; where it names no level, one line saying how to choose anchors."""
|
||||
if isinstance(found, Failure):
|
||||
return [f"- None: {found.reason}"]
|
||||
if not found:
|
||||
return [f"- None: the project has no {ANCHORS_PATH}, or it names no level; ~/.claude/rules/todo-items.md says how to choose anchors."]
|
||||
by_id = {outcome.item.id: outcome for outcome in outcomes}
|
||||
by_id = {result.item.id: result for result in results}
|
||||
lines = []
|
||||
for level, ids in found:
|
||||
lines.append(f"{level}:")
|
||||
|
||||
+20
-20
@@ -57,7 +57,7 @@ import re
|
||||
import sys
|
||||
|
||||
from score_outcomes.anchors import anchor_lines, Anchors, ANCHORS_PATH, parse_anchors
|
||||
from score_outcomes.outcomes import Chunk, Landing, nearest, Outcome, outcomes_of, render, report, shipped
|
||||
from score_outcomes.results import Chunk, Landing, nearest, render, report, Result, results_of, shipped
|
||||
from git_repo import default_branch, file_at, full_history, git, top
|
||||
from failure import Failure
|
||||
from todo_table import LEVELS, risk, TOP_SCORE
|
||||
@@ -91,24 +91,24 @@ def main(argv: list[str]) -> int:
|
||||
default, note = found
|
||||
if note:
|
||||
print(f"score-outcomes: {note}", file=sys.stderr)
|
||||
outcomes = read_outcomes(default)
|
||||
if isinstance(outcomes, Failure):
|
||||
return failed(outcomes)
|
||||
results = read_results(default)
|
||||
if isinstance(results, Failure):
|
||||
return failed(results)
|
||||
anchored = read_anchors()
|
||||
print(report_text(outcomes, anchored, dict(zip(GIVEN, map(int, argv))) if argv else None))
|
||||
print(report_text(results, anchored, dict(zip(GIVEN, map(int, argv))) if argv else None))
|
||||
return failed(anchored) if isinstance(anchored, Failure) else 0
|
||||
|
||||
|
||||
def report_text(outcomes: list[Outcome], anchored: Anchors | Failure, given: dict[str, int] | None) -> str:
|
||||
def report_text(results: list[Result], anchored: Anchors | Failure, given: dict[str, int] | None) -> str:
|
||||
"""The report: the finished items nearest the given scores, where any are given, then the anchors, then the medians."""
|
||||
sections = []
|
||||
if given is not None:
|
||||
r = risk(given["L"], given["E"], given["I"])
|
||||
near = [render(outcome) for outcome in nearest(outcomes, given | {"R": r})] or ["- None: no chunk on the default branch shipped an item from todo.md."]
|
||||
near = [render(result) for result in nearest(results, given | {"R": r})] or ["- None: no chunk on the default branch shipped an item from todo.md."]
|
||||
sections.append([f"Finished items nearest L {given['L']}, E {given['E']}, I {given['I']}, so R {r}, and S {given['S']}, closest first:", "", *near])
|
||||
sections.append([f"Anchors in {ANCHORS_PATH}, each level's shipped items:", "", *anchor_lines(anchored, outcomes)])
|
||||
sections.append(["Median outcome at each guessed score, among chunks that shipped one item, never-replayed ones left out:", "", *report(outcomes)])
|
||||
if any(outcome.landed != "merge" for outcome in outcomes):
|
||||
sections.append([f"Anchors in {ANCHORS_PATH}, each level's shipped items:", "", *anchor_lines(anchored, results)])
|
||||
sections.append(["Median outcome at each guessed score, among chunks that shipped one item, never-replayed ones left out:", "", *report(results)])
|
||||
if any(result.landed != "merge" for result in results):
|
||||
sections.append(UNMERGED)
|
||||
return "\n\n".join("\n".join(section) for section in sections)
|
||||
|
||||
@@ -131,19 +131,19 @@ def read_anchors() -> Anchors | Failure:
|
||||
return Failure(f"cannot read {ANCHORS_PATH}: {failure}")
|
||||
|
||||
|
||||
def read_outcomes(default: str) -> list[Outcome] | Failure:
|
||||
def read_results(default: str) -> list[Result] | Failure:
|
||||
"""What each chunk on the default branch's first-parent line measured."""
|
||||
if partial := full_history():
|
||||
return partial
|
||||
log = git("log", "--first-parent", "--format=%x1e%h %ct %at %p", "--name-only", default)
|
||||
if isinstance(log, Failure):
|
||||
return log
|
||||
# Each record is a commit's line, then the files it changed against its first parent.
|
||||
records = [record.splitlines() for record in log.split("\x1e")[1:]]
|
||||
headers = [record[0] for record in records]
|
||||
# Each commit's part of the log is its line, then the files it changed against its first parent.
|
||||
commits = [part.splitlines() for part in log.split("\x1e")[1:]]
|
||||
headers = [commit[0] for commit in commits]
|
||||
hashes = [header.split()[0] for header in headers]
|
||||
plan = {short for short, record in zip(hashes, records) if "todo.md" in record}
|
||||
outcomes: list[Outcome] = []
|
||||
plan = {short for short, commit in zip(hashes, commits) if "todo.md" in commit}
|
||||
results: list[Result] = []
|
||||
for first, last, committed, landed in chunk_log(headers):
|
||||
# Only a chunk that changed todo.md can ship an item, so the others skip measure's git calls.
|
||||
if not any(short in plan for short in hashes[hashes.index(last):hashes.index(first) + 1]):
|
||||
@@ -153,8 +153,8 @@ def read_outcomes(default: str) -> list[Outcome] | Failure:
|
||||
measured = measure(f"{first}^1", last, label, utc(committed), titled, landed)
|
||||
if isinstance(measured, Failure):
|
||||
return measured
|
||||
outcomes += measured
|
||||
return outcomes
|
||||
results += measured
|
||||
return results
|
||||
|
||||
|
||||
def chunk_log(lines: list[str]) -> list[Chunk]:
|
||||
@@ -175,7 +175,7 @@ def chunk_log(lines: list[str]) -> list[Chunk]:
|
||||
return chunks
|
||||
|
||||
|
||||
def measure(before: str, tip: str, label: str, merged: str, titled: str, landed: Landing) -> list[Outcome] | Failure:
|
||||
def measure(before: str, tip: str, label: str, merged: str, titled: str, landed: Landing) -> list[Result] | Failure:
|
||||
"""What one chunk measured for each item it shipped: its change from the commit before it to its tip, and the titles of the commits in the range titled."""
|
||||
planned = file_at(before, "todo.md")
|
||||
if isinstance(planned, Failure):
|
||||
@@ -192,4 +192,4 @@ def measure(before: str, tip: str, label: str, merged: str, titled: str, landed:
|
||||
titles = git("log", "--format=%s", titled)
|
||||
if isinstance(titles, Failure):
|
||||
return titles
|
||||
return outcomes_of(items, label, merged, numstat, titles.splitlines(), landed)
|
||||
return results_of(items, label, merged, numstat, titles.splitlines(), landed)
|
||||
|
||||
@@ -19,8 +19,9 @@ Landing = typing.Literal["merge", "rebase", "push"]
|
||||
Chunk = tuple[str, str, int, Landing]
|
||||
|
||||
|
||||
# The tool's name and report call a Result an outcome, a word the code keeps for an entry of a `.outcomes.md` file: docs/decisions.md → `bin/: an outcome is an entry`.
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Outcome:
|
||||
class Result:
|
||||
"""What the chunk that shipped an item measured. findings holds the 🔴 and 🟡 counts, None where a round title lacks them or the branch has no round. landed says how the chunk reached the default branch: by a merge commit, by a rebase that replayed its commits, or by commits never replayed."""
|
||||
item: Item
|
||||
merge: str
|
||||
@@ -33,10 +34,10 @@ class Outcome:
|
||||
landed: Landing
|
||||
|
||||
|
||||
# An outcome's name and how to read it from an Outcome, None where the chunk did not record it.
|
||||
Measure = tuple[str, typing.Callable[[Outcome], int | None]]
|
||||
# The scores whose median outcome the report gives at each guessed value.
|
||||
MEDIANS: dict[str, Measure] = {"R": ("🔴 findings", lambda outcome: outcome.findings[0] if outcome.findings else None), "S": ("lines changed", lambda outcome: outcome.lines)}
|
||||
# A measure's name and how to read it from a Result, None where the chunk did not record it.
|
||||
Measure = tuple[str, typing.Callable[[Result], int | None]]
|
||||
# The scores whose median result the report gives at each guessed value.
|
||||
MEDIANS: dict[str, Measure] = {"R": ("🔴 findings", lambda result: result.findings[0] if result.findings else None), "S": ("lines changed", lambda result: result.lines)}
|
||||
# The levels R is built from; for each, the report gives how well it orders chunks by review rounds and by 🔴 findings.
|
||||
ORDERED = ("L", "E", "I")
|
||||
# The scores render shows, in this order; C only on an item scored before L, E and I replaced it.
|
||||
@@ -52,7 +53,7 @@ def shipped(before: str, after: str) -> list[Item]:
|
||||
return sorted((item for item in planned if item.id not in kept), key=lambda item: item.id)
|
||||
|
||||
|
||||
def outcomes_of(shipped: list[Item], merge: str, merged: str, numstat: str, titles: list[str], landed: Landing) -> list[Outcome]:
|
||||
def results_of(shipped: list[Item], merge: str, merged: str, numstat: str, titles: list[str], landed: Landing) -> list[Result]:
|
||||
"""What one chunk measured for each item it shipped, from `git diff --numstat` against the commit before it and its commit titles; none where it changed nothing but the plan and the records."""
|
||||
changed = [row.split("\t", 2) for row in numstat.splitlines()]
|
||||
work = [row for row in changed if len(row) == 3 and row[2] not in NOT_WORK]
|
||||
@@ -62,24 +63,24 @@ def outcomes_of(shipped: list[Item], merge: str, merged: str, numstat: str, titl
|
||||
lines = sum(int(count) for row in work for count in row[:2] if count.isdigit())
|
||||
past_first = sum(max(0, count - 1) for count in round_counts(titles).values())
|
||||
ids = [item.id for item in shipped]
|
||||
return [Outcome(item, merge, merged, tuple(id_ for id_ in ids if id_ != item.id), lines, len(work), past_first, round_findings(titles), landed) for item in shipped]
|
||||
return [Result(item, merge, merged, tuple(id_ for id_ in ids if id_ != item.id), lines, len(work), past_first, round_findings(titles), landed) for item in shipped]
|
||||
|
||||
|
||||
def nearest(outcomes: list[Outcome], given: dict[str, int], count: int = 5) -> list[Outcome]:
|
||||
"""The outcomes whose guesses sit closest to the given scores: the least mean squared difference over the scores both hold, the newest first among equals."""
|
||||
def distance(outcome: Outcome) -> float:
|
||||
shared = [name for name in given if name in outcome.item.scores]
|
||||
return sum((given[name] - outcome.item.scores[name]) ** 2 for name in shared) / len(shared)
|
||||
newest_first = sorted(outcomes, key=lambda outcome: outcome.merged, reverse=True)
|
||||
def nearest(results: list[Result], given: dict[str, int], count: int = 5) -> list[Result]:
|
||||
"""The results whose guesses sit closest to the given scores: the least mean squared difference over the scores both hold, the newest first among equals."""
|
||||
def distance(result: Result) -> float:
|
||||
shared = [name for name in given if name in result.item.scores]
|
||||
return sum((given[name] - result.item.scores[name]) ** 2 for name in shared) / len(shared)
|
||||
newest_first = sorted(results, key=lambda result: result.merged, reverse=True)
|
||||
return sorted(newest_first, key=distance)[:count]
|
||||
|
||||
|
||||
def report(outcomes: list[Outcome]) -> list[str]:
|
||||
"""Among chunks that shipped one item, since a chunk measures all its items at once, and that were merged or replayed, since a never-replayed one may hold only part of a pull request: for each score in MEDIANS, the median outcome at each value it was guessed at; for each level in ORDERED, how well it orders chunks by review rounds and by 🔴 findings."""
|
||||
single = [outcome for outcome in outcomes if not outcome.shipped_with and outcome.landed != "push"]
|
||||
def report(results: list[Result]) -> list[str]:
|
||||
"""Among chunks that shipped one item, since a chunk measures all its items at once, and that were merged or replayed, since a never-replayed one may hold only part of a pull request: for each score in MEDIANS, the median result at each value it was guessed at; for each level in ORDERED, how well it orders chunks by review rounds and by 🔴 findings."""
|
||||
single = [result for result in results if not result.shipped_with and result.landed != "push"]
|
||||
lines = []
|
||||
for name, (what, measure) in MEDIANS.items():
|
||||
held = [(outcome.item.scores[name], value) for outcome in single if name in outcome.item.scores and (value := measure(outcome)) is not None]
|
||||
held = [(result.item.scores[name], value) for result in single if name in result.item.scores and (value := measure(result)) is not None]
|
||||
if len(held) < ENOUGH:
|
||||
lines.append(f"- {name}, {what}: {too_few(len(held), name, 'its outcome')}")
|
||||
continue
|
||||
@@ -87,7 +88,7 @@ def report(outcomes: list[Outcome]) -> list[str]:
|
||||
lines.append(f"- {name}, {what}: " + ", ".join(f"{guess} → {statistics.median(found):g} ({plural(len(found), 'chunk')})" for guess, found in medians))
|
||||
for name in ORDERED:
|
||||
# Each pass takes a first round whatever the change, so only the rounds after it measure how hard the change was to get right.
|
||||
rows = [(outcome.item.scores[name], outcome.rounds_past_first, outcome.findings[0]) for outcome in single if name in outcome.item.scores and outcome.findings is not None]
|
||||
rows = [(result.item.scores[name], result.rounds_past_first, result.findings[0]) for result in single if name in result.item.scores and result.findings is not None]
|
||||
opening = f"- {name}, by review rounds past each pass's first and 🔴 findings: "
|
||||
if len(rows) < ENOUGH:
|
||||
lines.append(opening + too_few(len(rows), name, "both outcomes"))
|
||||
@@ -104,11 +105,11 @@ def too_few(held: int, name: str, what: str) -> str:
|
||||
return f"{held} {chunks} {name} and {what}; the report needs {ENOUGH}."
|
||||
|
||||
|
||||
def rank_correlation(guesses: list[int], outcomes: list[int]) -> float | None:
|
||||
def rank_correlation(guesses: list[int], results: list[int]) -> float | None:
|
||||
"""Spearman's rank correlation, tied values sharing their mean rank; None where either side holds one value, which no order can rank."""
|
||||
if len(set(guesses)) < 2 or len(set(outcomes)) < 2:
|
||||
if len(set(guesses)) < 2 or len(set(results)) < 2:
|
||||
return None
|
||||
return statistics.correlation(ranks(guesses), ranks(outcomes))
|
||||
return statistics.correlation(ranks(guesses), ranks(results))
|
||||
|
||||
|
||||
def ranks(values: list[int]) -> list[float]:
|
||||
@@ -117,15 +118,15 @@ def ranks(values: list[int]) -> list[float]:
|
||||
return [ordered.index(value) + (ordered.count(value) + 1) / 2 for value in values]
|
||||
|
||||
|
||||
def render(outcome: Outcome) -> str:
|
||||
scores = ", ".join(f"{name} {outcome.item.scores[name]}" for name in SHOWN if name in outcome.item.scores)
|
||||
def render(result: Result) -> str:
|
||||
scores = ", ".join(f"{name} {result.item.scores[name]}" for name in SHOWN if name in result.item.scores)
|
||||
# Rebase drops a round's commit that changed no file, so history shows such a chunk's rounds and findings short.
|
||||
at_least = "" if outcome.landed == "merge" else "at least "
|
||||
merged = {"merge": f"merged {outcome.merged} in {outcome.merge}", "rebase": f"merged {outcome.merged} by rebase in {outcome.merge}", "push": f"committed {outcome.merged} in {outcome.merge}, never replayed, so perhaps part of a pull request"}[outcome.landed]
|
||||
findings = f"{at_least}{outcome.findings[0]} 🔴, {outcome.findings[1]} 🟡" if outcome.findings else "findings unknown"
|
||||
others = outcome.shipped_with
|
||||
at_least = "" if result.landed == "merge" else "at least "
|
||||
merged = {"merge": f"merged {result.merged} in {result.merge}", "rebase": f"merged {result.merged} by rebase in {result.merge}", "push": f"committed {result.merged} in {result.merge}, never replayed, so perhaps part of a pull request"}[result.landed]
|
||||
findings = f"{at_least}{result.findings[0]} 🔴, {result.findings[1]} 🟡" if result.findings else "findings unknown"
|
||||
others = result.shipped_with
|
||||
together = f", with item{'s' if len(others) > 1 else ''} {', '.join(map(str, others))}" if others else ""
|
||||
return f"- Item {outcome.item.id}, {scores}: {plural(outcome.lines, 'line')} in {plural(outcome.files, 'file')}, {at_least}{plural(outcome.rounds_past_first, 'round')} past each pass's first, {findings}; {merged}{together}. {outcome.item.tagline}"
|
||||
return f"- Item {result.item.id}, {scores}: {plural(result.lines, 'line')} in {plural(result.files, 'file')}, {at_least}{plural(result.rounds_past_first, 'round')} past each pass's first, {findings}; {merged}{together}. {result.item.tagline}"
|
||||
|
||||
|
||||
def plural(count: int, noun: str) -> str:
|
||||
@@ -12,7 +12,7 @@ from score_outcomes.main import chunk_log, report_text
|
||||
from failure import Failure
|
||||
from test_env import isolated_env
|
||||
from todo_table import Item
|
||||
from score_outcomes.outcomes import Landing, nearest, Outcome, outcomes_of, rank_correlation, render, report, shipped
|
||||
from score_outcomes.results import Landing, nearest, rank_correlation, render, report, Result, results_of, shipped
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "score-outcomes"
|
||||
HEADER = "| ID | Release | Exempt | Item | R | S | C | A | G | Goals | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n"
|
||||
@@ -29,8 +29,8 @@ def item(id_: int, scores: dict[str, int]) -> Item:
|
||||
return Item(id_, (0, 1, 0), "", f"**Item {id_}.**", scores, (("1", None),), ())
|
||||
|
||||
|
||||
def outcome(id_: int, r: int, s: int, c: int | None, lines: int = 10, rounds: int = 1, findings: tuple[int, int] | None = (0, 0), merged: str = "2026-10-13T10:00:00Z", shipped_with: tuple[int, ...] = (), levels: dict[str, int] | None = None, landed: Landing = "merge") -> Outcome:
|
||||
return Outcome(item(id_, {"R": r, "S": s} | ({"C": c} if c else {}) | (levels or {})), "aaaaaaa", merged, shipped_with, lines, 1, rounds, findings, landed)
|
||||
def result(id_: int, r: int, s: int, c: int | None, lines: int = 10, rounds: int = 1, findings: tuple[int, int] | None = (0, 0), merged: str = "2026-10-13T10:00:00Z", shipped_with: tuple[int, ...] = (), levels: dict[str, int] | None = None, landed: Landing = "merge") -> Result:
|
||||
return Result(item(id_, {"R": r, "S": s} | ({"C": c} if c else {}) | (levels or {})), "aaaaaaa", merged, shipped_with, lines, 1, rounds, findings, landed)
|
||||
|
||||
|
||||
class Shipped(unittest.TestCase):
|
||||
@@ -72,47 +72,47 @@ class ChunkLog(unittest.TestCase):
|
||||
self.assertEqual(chunk_log(["c2 300 300 c1", "c1 200 200 r"]), [("c2", "c2", 300, "push"), ("c1", "c1", 200, "push")])
|
||||
|
||||
|
||||
class OutcomesOf(unittest.TestCase):
|
||||
class ResultsOf(unittest.TestCase):
|
||||
def test_measures_the_change_less_the_plan_and_the_tool_records_and_counts_binary_files(self):
|
||||
numstat = "3\t1\ta.py\n-\t-\tlogo.png\n0\t1\t2026\n5\t0\ttodo.md\n9\t0\tdocs/reading-cost.md\n4\t0\tdocs/comprehension-history.md\n"
|
||||
titles = ["Answer stability round 1: 1 🔴, 2 🟡; fix", "Answer stability round 2: 0 🔴, 1 🟡; fix", "Answer prose round 1: 0 🔴, 3 🟡", "Do it"]
|
||||
found = outcomes_of([item(4, {"R": 2}), item(5, {"R": 3})], "abc1234", "2026-10-13T10:00:00Z", numstat, titles, "merge")
|
||||
self.assertEqual(found[0], Outcome(item(4, {"R": 2}), "abc1234", "2026-10-13T10:00:00Z", (5,), 5, 3, 1, (1, 6), "merge"))
|
||||
found = results_of([item(4, {"R": 2}), item(5, {"R": 3})], "abc1234", "2026-10-13T10:00:00Z", numstat, titles, "merge")
|
||||
self.assertEqual(found[0], Result(item(4, {"R": 2}), "abc1234", "2026-10-13T10:00:00Z", (5,), 5, 3, 1, (1, 6), "merge"))
|
||||
self.assertEqual(found[1].shipped_with, (4,))
|
||||
|
||||
def test_measures_no_chunk_that_changed_only_the_plan(self):
|
||||
self.assertEqual(outcomes_of([item(4, {"R": 2})], "abc1234", "2026-10-13T10:00:00Z", "0\t5\ttodo.md\n", [], "merge"), [])
|
||||
self.assertEqual(results_of([item(4, {"R": 2})], "abc1234", "2026-10-13T10:00:00Z", "0\t5\ttodo.md\n", [], "merge"), [])
|
||||
|
||||
|
||||
class Nearest(unittest.TestCase):
|
||||
def test_lists_the_closest_guesses_first_by_the_mean_over_the_scores_both_hold(self):
|
||||
found = [outcome(1, 9, 9, 9), outcome(2, 3, 5, None), outcome(3, 3, 4, 6), outcome(4, 3, 4, 9)]
|
||||
found = [result(1, 9, 9, 9), result(2, 3, 5, None), result(3, 3, 4, 6), result(4, 3, 4, 9)]
|
||||
self.assertEqual([each.item.id for each in nearest(found, {"R": 3, "S": 4, "C": 5}, 3)], [3, 2, 4])
|
||||
|
||||
def test_lists_five_by_default(self):
|
||||
self.assertEqual(len(nearest([outcome(id_, 3, 4, 5) for id_ in range(6)], {"R": 3, "S": 4, "C": 5})), 5)
|
||||
self.assertEqual(len(nearest([result(id_, 3, 4, 5) for id_ in range(6)], {"R": 3, "S": 4, "C": 5})), 5)
|
||||
|
||||
def test_compares_an_item_scored_before_l_e_and_i_by_its_r_and_s(self):
|
||||
given = {"L": 2, "E": 2, "I": 3, "R": 5, "S": 4}
|
||||
found = [outcome(1, 8, 4, None, levels={"L": 4, "E": 3, "I": 3}), outcome(2, 5, 4, 9, merged="2026-10-12T10:00:00Z"), outcome(3, 5, 4, None, levels={"L": 2, "E": 2, "I": 3})]
|
||||
found = [result(1, 8, 4, None, levels={"L": 4, "E": 3, "I": 3}), result(2, 5, 4, 9, merged="2026-10-12T10:00:00Z"), result(3, 5, 4, None, levels={"L": 2, "E": 2, "I": 3})]
|
||||
self.assertEqual([each.item.id for each in nearest(found, given)], [3, 2, 1])
|
||||
|
||||
def test_lists_the_newest_first_among_equals(self):
|
||||
found = [outcome(1, 3, 4, 5, merged="2026-10-12T10:00:00Z"), outcome(2, 3, 4, 5, merged="2026-10-13T09:00:00Z")]
|
||||
found = [result(1, 3, 4, 5, merged="2026-10-12T10:00:00Z"), result(2, 3, 4, 5, merged="2026-10-13T09:00:00Z")]
|
||||
self.assertEqual([each.item.id for each in nearest(found, {"R": 3, "S": 4, "C": 5})], [2, 1])
|
||||
|
||||
|
||||
class Report(unittest.TestCase):
|
||||
def test_gives_each_guessed_values_median_outcome_once_ten_single_item_chunks_hold_it(self):
|
||||
found = [outcome(id_, 2, 1 + id_ % 2, 4, lines=10 * id_, rounds=id_, findings=(1, 3)) for id_ in range(1, 11)]
|
||||
lines = report(found + [outcome(11, 2, 2, None, lines=1000, findings=None, shipped_with=(12,))])
|
||||
def test_gives_each_guessed_values_median_result_once_ten_single_item_chunks_hold_it(self):
|
||||
found = [result(id_, 2, 1 + id_ % 2, 4, lines=10 * id_, rounds=id_, findings=(1, 3)) for id_ in range(1, 11)]
|
||||
lines = report(found + [result(11, 2, 2, None, lines=1000, findings=None, shipped_with=(12,))])
|
||||
self.assertIn("- R, 🔴 findings: 2 → 1 (10 chunks)", lines)
|
||||
self.assertIn("- S, lines changed: 1 → 60 (5 chunks), 2 → 50 (5 chunks)", lines)
|
||||
self.assertFalse([line for line in lines if line.startswith("- C")])
|
||||
|
||||
def test_says_how_many_chunks_it_has_under_ten(self):
|
||||
self.assertIn("- S, lines changed: 1 such chunk records S and its outcome; the report needs 10.", report([outcome(1, 2, 2, 4), outcome(2, 2, 2, None, shipped_with=(1,))]))
|
||||
too_few = report([outcome(1, 2, 2, None, levels={"L": 2, "E": 1, "I": 1}), outcome(2, 2, 2, None, levels={"L": 2}, findings=None)])
|
||||
self.assertIn("- S, lines changed: 1 such chunk records S and its outcome; the report needs 10.", report([result(1, 2, 2, 4), result(2, 2, 2, None, shipped_with=(1,))]))
|
||||
too_few = report([result(1, 2, 2, None, levels={"L": 2, "E": 1, "I": 1}), result(2, 2, 2, None, levels={"L": 2}, findings=None)])
|
||||
for name in "LEI":
|
||||
with self.subTest(name=name):
|
||||
self.assertIn(f"- {name}, by review rounds past each pass's first and 🔴 findings: 1 such chunk records {name} and both outcomes; the report needs 10.", too_few)
|
||||
@@ -122,12 +122,12 @@ class Report(unittest.TestCase):
|
||||
self.assertAlmostEqual(rank_correlation([1, 1, 2, 3], [1, 2, 3, 4]), 0.949, places=3)
|
||||
self.assertAlmostEqual(rank_correlation([1, 2, 3, 4], [1, 1, 2, 3]), 0.949, places=3)
|
||||
|
||||
def test_ranks_an_outcome_that_holds_two_values(self):
|
||||
def test_ranks_a_result_that_holds_two_values(self):
|
||||
self.assertAlmostEqual(rank_correlation([1, 2, 3, 4, 5], [1, 1, 2, 2, 2]), 0.866, places=3)
|
||||
|
||||
def test_gives_how_well_l_e_and_i_each_order_chunks_by_review_rounds_and_red_findings(self):
|
||||
levels = [1, 1, 2, 2, 2, 3, 3, 3, 4, 4]
|
||||
found = [outcome(id_, 2, 2, None, rounds=level, findings=(5 - level, 0), levels={"L": level, "E": 2, "I": level}) for id_, level in enumerate(levels, 1)]
|
||||
found = [result(id_, 2, 2, None, rounds=level, findings=(5 - level, 0), levels={"L": level, "E": 2, "I": level}) for id_, level in enumerate(levels, 1)]
|
||||
lines = report(found)
|
||||
self.assertIn("- L, by review rounds past each pass's first and 🔴 findings: rank correlation +1.00 and -1.00 (10 chunks)", lines)
|
||||
self.assertIn("- I, by review rounds past each pass's first and 🔴 findings: rank correlation +1.00 and -1.00 (10 chunks)", lines)
|
||||
@@ -136,19 +136,19 @@ class Report(unittest.TestCase):
|
||||
|
||||
class Render(unittest.TestCase):
|
||||
def test_shows_an_items_l_e_and_i_before_its_r_and_s(self):
|
||||
self.assertTrue(render(outcome(4, 3, 3, None, levels={"L": 2, "E": 2, "I": 1})).startswith("- Item 4, L 2, E 2, I 1, R 3, S 3:"))
|
||||
self.assertTrue(render(result(4, 3, 3, None, levels={"L": 2, "E": 2, "I": 1})).startswith("- Item 4, L 2, E 2, I 1, R 3, S 3:"))
|
||||
|
||||
def test_names_the_items_shipped_together_and_unknown_findings(self):
|
||||
self.assertEqual(render(outcome(4, 2, 3, None, lines=1, rounds=2, findings=None, shipped_with=(5, 6))), "- Item 4, R 2, S 3: 1 line in 1 file, 2 rounds past each pass's first, findings unknown; merged 2026-10-13T10:00:00Z in aaaaaaa, with items 5, 6. **Item 4.**")
|
||||
self.assertEqual(render(outcome(4, 2, 3, 4, findings=(1, 2))), "- Item 4, R 2, S 3, C 4: 10 lines in 1 file, 1 round past each pass's first, 1 🔴, 2 🟡; merged 2026-10-13T10:00:00Z in aaaaaaa. **Item 4.**")
|
||||
self.assertIn("1 round past each pass's first, 0 🔴, 0 🟡; merged 2026-10-13T10:00:00Z in aaaaaaa, with item 5. ", render(outcome(4, 2, 3, 4, rounds=1, shipped_with=(5,))))
|
||||
self.assertEqual(render(result(4, 2, 3, None, lines=1, rounds=2, findings=None, shipped_with=(5, 6))), "- Item 4, R 2, S 3: 1 line in 1 file, 2 rounds past each pass's first, findings unknown; merged 2026-10-13T10:00:00Z in aaaaaaa, with items 5, 6. **Item 4.**")
|
||||
self.assertEqual(render(result(4, 2, 3, 4, findings=(1, 2))), "- Item 4, R 2, S 3, C 4: 10 lines in 1 file, 1 round past each pass's first, 1 🔴, 2 🟡; merged 2026-10-13T10:00:00Z in aaaaaaa. **Item 4.**")
|
||||
self.assertIn("1 round past each pass's first, 0 🔴, 0 🟡; merged 2026-10-13T10:00:00Z in aaaaaaa, with item 5. ", render(result(4, 2, 3, 4, rounds=1, shipped_with=(5,))))
|
||||
|
||||
def test_gives_a_rebase_merged_chunks_rounds_and_findings_as_lower_bounds(self):
|
||||
self.assertEqual(render(outcome(4, 2, 3, None, findings=(1, 2), landed="rebase")), "- Item 4, R 2, S 3: 10 lines in 1 file, at least 1 round past each pass's first, at least 1 🔴, 2 🟡; merged 2026-10-13T10:00:00Z by rebase in aaaaaaa. **Item 4.**")
|
||||
self.assertIn("at least 1 round past each pass's first, findings unknown; merged 2026-10-13T10:00:00Z by rebase in aaaaaaa", render(outcome(4, 2, 3, None, findings=None, landed="rebase")))
|
||||
self.assertEqual(render(result(4, 2, 3, None, findings=(1, 2), landed="rebase")), "- Item 4, R 2, S 3: 10 lines in 1 file, at least 1 round past each pass's first, at least 1 🔴, 2 🟡; merged 2026-10-13T10:00:00Z by rebase in aaaaaaa. **Item 4.**")
|
||||
self.assertIn("at least 1 round past each pass's first, findings unknown; merged 2026-10-13T10:00:00Z by rebase in aaaaaaa", render(result(4, 2, 3, None, findings=None, landed="rebase")))
|
||||
|
||||
def test_says_a_chunk_never_replayed_may_be_part_of_a_pull_request(self):
|
||||
self.assertEqual(render(outcome(4, 2, 3, None, findings=(1, 2), landed="push")), "- Item 4, R 2, S 3: 10 lines in 1 file, at least 1 round past each pass's first, at least 1 🔴, 2 🟡; committed 2026-10-13T10:00:00Z in aaaaaaa, never replayed, so perhaps part of a pull request. **Item 4.**")
|
||||
self.assertEqual(render(result(4, 2, 3, None, findings=(1, 2), landed="push")), "- Item 4, R 2, S 3: 10 lines in 1 file, at least 1 round past each pass's first, at least 1 🔴, 2 🟡; committed 2026-10-13T10:00:00Z in aaaaaaa, never replayed, so perhaps part of a pull request. **Item 4.**")
|
||||
|
||||
|
||||
ANCHORS_TEXT = "# Score anchors\n\nWhy these.\n\n| Level | Items |\n|---|---|\n| L 1 | 4, 9 |\n| I 3 | 5 |\n"
|
||||
@@ -173,7 +173,7 @@ class Anchors(unittest.TestCase):
|
||||
self.assertEqual(parse_anchors(ANCHORS_TEXT + "| L 1 | 7 |\n"), Failure("docs/score-anchors.md names L 1 twice"))
|
||||
|
||||
def test_lists_each_anchor_with_what_its_chunk_measured_and_says_which_it_cannot_find(self):
|
||||
lines = anchor_lines([("L 1", (4, 9))], [outcome(4, 2, 3, None, findings=(1, 2))])
|
||||
lines = anchor_lines([("L 1", (4, 9))], [result(4, 2, 3, None, findings=(1, 2))])
|
||||
self.assertEqual(lines, ["L 1:", "- Item 4, R 2, S 3: 10 lines in 1 file, 1 round past each pass's first, 1 🔴, 2 🟡; merged 2026-10-13T10:00:00Z in aaaaaaa. **Item 4.**", "- Item 9: no chunk shipped it."])
|
||||
|
||||
def test_points_to_the_rules_where_the_project_names_no_anchors(self):
|
||||
@@ -186,7 +186,7 @@ class Anchors(unittest.TestCase):
|
||||
|
||||
class ReportText(unittest.TestCase):
|
||||
def test_lists_the_nearest_by_r_then_the_anchors_then_the_medians(self):
|
||||
found = [outcome(1, 5, 3, None, merged="2026-10-13T10:00:00Z"), outcome(2, 1, 3, None, merged="2026-10-12T10:00:00Z")]
|
||||
found = [result(1, 5, 3, None, merged="2026-10-13T10:00:00Z"), result(2, 1, 3, None, merged="2026-10-12T10:00:00Z")]
|
||||
too_few = "such chunks record"
|
||||
self.assertEqual(report_text(found, [("L 1", (2,))], {"L": 1, "E": 1, "I": 1, "S": 3}), f"""Finished items nearest L 1, E 1, I 1, so R 1, and S 3, closest first:
|
||||
|
||||
@@ -207,21 +207,21 @@ Median outcome at each guessed score, among chunks that shipped one item, never-
|
||||
- I, by review rounds past each pass's first and 🔴 findings: 0 {too_few} I and both outcomes; the report needs 10.""")
|
||||
|
||||
def test_ends_with_how_it_reads_chunks_merged_by_rebase_only_where_it_measured_one(self):
|
||||
text = report_text([outcome(1, 5, 3, None, landed="rebase")], [], None)
|
||||
text = report_text([result(1, 5, 3, None, landed="rebase")], [], None)
|
||||
self.assertIn("\n\nChunks that left no merge commit:\n\n- Each is the run of commits", text.split("Median outcome")[1])
|
||||
self.assertIn("a pull request that shipped nothing adds its lines, files and rounds", text)
|
||||
self.assertIn("at least what history shows, and the medians count the rebased ones as shown.", text)
|
||||
self.assertIn("Chunks that left no merge commit", report_text([outcome(1, 5, 3, None, landed="push")], [], None))
|
||||
self.assertNotIn("Chunks that left no merge commit", report_text([outcome(1, 5, 3, None)], [], None))
|
||||
self.assertIn("Chunks that left no merge commit", report_text([result(1, 5, 3, None, landed="push")], [], None))
|
||||
self.assertNotIn("Chunks that left no merge commit", report_text([result(1, 5, 3, None)], [], None))
|
||||
|
||||
def test_leaves_a_never_replayed_chunk_out_of_the_s_medians_and_keeps_a_rebased_one(self):
|
||||
found = [outcome(id_, 2, 2, None, lines=10 * id_, landed="rebase" if id_ == 1 else "merge") for id_ in range(1, 11)]
|
||||
self.assertIn("- S, lines changed: 2 → 55 (10 chunks)", report_text(found + [outcome(11, 2, 2, None, lines=100000, landed="push")], [], None))
|
||||
found = [result(id_, 2, 2, None, lines=10 * id_, landed="rebase" if id_ == 1 else "merge") for id_ in range(1, 11)]
|
||||
self.assertIn("- S, lines changed: 2 → 55 (10 chunks)", report_text(found + [result(11, 2, 2, None, lines=100000, landed="push")], [], None))
|
||||
|
||||
def test_leaves_a_never_replayed_chunk_out_of_the_r_medians_and_the_rank_correlations(self):
|
||||
levels = [1, 1, 2, 2, 2, 3, 3, 3, 4, 4]
|
||||
found = [outcome(id_, 2, 2, None, rounds=level, findings=(5 - level, 0), levels={"L": level, "E": 2, "I": level}) for id_, level in enumerate(levels, 1)]
|
||||
pushed = outcome(11, 2, 2, None, rounds=9, findings=(9, 0), levels={"L": 1, "E": 2, "I": 1}, landed="push")
|
||||
found = [result(id_, 2, 2, None, rounds=level, findings=(5 - level, 0), levels={"L": level, "E": 2, "I": level}) for id_, level in enumerate(levels, 1)]
|
||||
pushed = result(11, 2, 2, None, rounds=9, findings=(9, 0), levels={"L": 1, "E": 2, "I": 1}, landed="push")
|
||||
text = report_text(found + [pushed], [], None)
|
||||
self.assertIn("- R, 🔴 findings: 2 → 2.5 (10 chunks)", text)
|
||||
self.assertIn("- L, by review rounds past each pass's first and 🔴 findings: rank correlation +1.00 and -1.00 (10 chunks)", text)
|
||||
@@ -229,7 +229,7 @@ Median outcome at each guessed score, among chunks that shipped one item, never-
|
||||
self.assertIn("- E, by review rounds past each pass's first and 🔴 findings: rank correlation none and none, since every chunk held one value (10 chunks)", text)
|
||||
|
||||
def test_keeps_a_never_replayed_chunk_in_the_nearest_and_anchors_lists(self):
|
||||
pushed = outcome(9, 2, 3, None, landed="push")
|
||||
pushed = result(9, 2, 3, None, landed="push")
|
||||
text = report_text([pushed], [("L 1", (9,))], {"L": 1, "E": 1, "I": 1, "S": 3})
|
||||
nearest_section, anchors_section = text.split("Anchors in ")
|
||||
self.assertIn("- Item 9, R 2, S 3:", nearest_section)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import re
|
||||
|
||||
from markdown_text import quoted
|
||||
from model_routing.route import Record
|
||||
from utc_stamp import STAMPED
|
||||
from model_routing.route import WorkerLine
|
||||
|
||||
TASK = "Task: "
|
||||
FINISHED = "Verdict: finished"
|
||||
@@ -29,23 +29,23 @@ def escalation_block(escalation: str) -> str:
|
||||
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
|
||||
|
||||
|
||||
def finished(worker: Record, evidence: str, judge: str) -> str:
|
||||
def finished(worker: WorkerLine, evidence: str, judge: str) -> str:
|
||||
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{JUDGE}{judge}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def unfinished(worker: Record, evidence: str, judge: str, escalation: str) -> str:
|
||||
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str) -> str:
|
||||
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
|
||||
escalated = escalation_block(escalation)
|
||||
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{JUDGE}{judge}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: Record) -> str:
|
||||
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the record of the route the worker ran on until the interrupt."""
|
||||
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine) -> str:
|
||||
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
|
||||
escalated = escalation_block(escalation)
|
||||
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{JUDGE}{judge}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def answered(question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: Record) -> str:
|
||||
def answered(question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine) -> str:
|
||||
"""escalation and worker are as on interrupted."""
|
||||
escalated = escalation_block(escalation)
|
||||
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{JUDGE}{judge}\n{WORKER}{worker}"
|
||||
@@ -56,12 +56,12 @@ def rejected(why: str, reply: str, judge: str) -> str:
|
||||
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{JUDGE}{judge}"
|
||||
|
||||
|
||||
def stepped_up(worker: Record) -> str:
|
||||
def stepped_up(worker: WorkerLine) -> str:
|
||||
"""The rung the next cycle steps up to, archived as it steps, so a rerun after any exit resumes on it."""
|
||||
return f"Stepped up\n\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def stopped(worker: Record, why: str) -> str:
|
||||
def stopped(worker: WorkerLine, why: str) -> str:
|
||||
return f"Stopped: {why}\n\n{WORKER}{worker}"
|
||||
|
||||
|
||||
|
||||
@@ -2599,3 +2599,30 @@ Inherited architect, decided by:
|
||||
- Shape: `bin/loop_claude/verdicts.py:50` `Escalating`: the base's escalation_mark function returned where the mark would move; now the module carries the concept's name, moved_to says what it returns, and the type has one name
|
||||
- Self-sufficiency: `bin/loop_claude/cycle/judge.py:70` `Reply.escalation`: the comment points at code that explains the rule rather than at docs/decisions.md
|
||||
- Navigation: `bin/round_titles.py:6` `CAP`: names the decision, rule-hook's round-cap and larv-review round-count, so a fourth-round page leads to the code that enforces the cap
|
||||
|
||||
## 2026-10-23T09:16:16Z, PR #367 at 5a56fff040dd437f8d5f94007fbfe371c874d1c4, against dcbdc970a67d07ab53d82a792c14f7b240a4b200
|
||||
|
||||
Ruling: better
|
||||
|
||||
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||
|---|---|---|---|---|---|
|
||||
| Mid A | better | same | better | same | better |
|
||||
| Inherited architect | better | same | better | better | better |
|
||||
|
||||
Mid A, decided by:
|
||||
|
||||
- Navigation, Shape: `bin/larv_review/history_changes.py:6` `history_changes`: named by its job where `git.py` named only that it calls git, and the merge-commit rule has its own unit in `history_change`
|
||||
- Navigation, Shape: `bin/larv_review/history_commands.py:100` `show_places`: the module's name now fits all six entry points, where `record.py` also held commands that record nothing
|
||||
- Shape: `bin/model_routing/route.py:13` `WorkerLine`: `Record`, `Route.record` and `RECORD_FIELDS` clashed with three other meanings of record; the new names match the `Worker:` line
|
||||
- Shape: `bin/score_outcomes/results.py:24` `Result`: no longer collides with the archive's outcomes, at the cost of a comment explaining why the tool still says outcome
|
||||
- Shape: `bin/judge_replay/replay.py:19` `archived_kind`: says it holds the archived verdict kind, where `on_record` read like a flag
|
||||
|
||||
Inherited architect, decided by:
|
||||
|
||||
- Shape, Self-sufficiency: `bin/model_routing/route.py:13` `WorkerLine`: says the value is the JSON on the `Worker:` line that ends each outcome, where `Record` overlapped the document sense and the verb
|
||||
- Navigation, Shape: `bin/larv_review/history_changes.py:6` `history_changes`: a ruling-base symptom now leads to a file named for its one job, and the first-parent rule sits in one small unit
|
||||
- Navigation: `bin/larv_review/history_commands.py:16` `record_ruling`: imported from a file named for the history commands it serves, not the overloaded record
|
||||
- Shape: `bin/rule_hook/rules.py:91` `Checked`: no longer reuses outcome and verdict, two words the project uses for archive entries and judge verdicts
|
||||
- Shape, Self-sufficiency: `bin/reading_cost/measure.py:13` `Reading`: `files` no longer borrows the seats' place, and the docstring says what both fields count
|
||||
- Shape: `bin/judge_replay/replay.py:24` `Case.archived_kind`: says it is the kind the archive gave the verdict
|
||||
- Self-sufficiency: `bin/score_outcomes/results.py:22` `Result`: a reader translates between result in the code and outcome in the tool's name and report, a small cost the comment explains
|
||||
|
||||
+12
-2
@@ -1,5 +1,15 @@
|
||||
# Decisions
|
||||
|
||||
## bin/: an outcome is an entry of a `.outcomes.md` file in loop-claude's archive, a record is a document tools or people keep entries in, and a place is what a comprehension seat names
|
||||
|
||||
2026-10-23, larv-review, through PR #367. Serves code health: a reader can trust that one word names one thing in every tool. Before, "record" and "outcome" each named different things in three modules, and "place" in two. Each word kept the meaning it already had where renaming costs most:
|
||||
|
||||
- An outcome: the archive's file names end in `.outcomes.md`, and the tools that read them find them by that suffix. score-outcomes' `Outcome` became `Result`, and rule-hook's became `Checked`. The command `score-outcomes` keeps its name, since renaming it would be a change of its own: files every project loads, such as `rules/todo-items.md`, name it, and so do decision titles. Its help and report keep the word its name uses.
|
||||
- A record: "judge's records" is a term in `~/.claude/CLAUDE.md` → "Vocabulary", and reading-cost's help calls its file a record. A single item in such a document is an entry; the route the archive keeps is a worker line (`Route.worker_line`). The verb "to record" keeps its plain meaning.
|
||||
- A place: the comprehension panel's term, also in the Vocabulary. reading-cost's `Reading.places`, how often each file was read, became `Reading.files`.
|
||||
|
||||
Valid while the archive's file names, the Vocabulary and the panel keep these words.
|
||||
|
||||
## bin/: the tests rerun themselves on a `uv`-fetched Python at README Install's floor
|
||||
|
||||
2026-09-26, larv-review. Serves goal 2. The suite reruns itself on a `uv`-fetched interpreter of the version Install names: `ast.parse(feature_version=)` on the host's Python accepts the nested f-string 186181a fixed, a compile check passes a call newer than the floor, like 3.12's `itertools.batched`, and a `python3.N` on `PATH` is installed by hand on every machine and again at each bump. Valid while `uv` can fetch the floor interpreter and no machine taking the checkout ships an older Python.
|
||||
@@ -22,7 +32,7 @@ The cap is 5, `bin/todo_table.py`'s `BUNDLE_MAX_R`, from the same chunks: by the
|
||||
|
||||
2026-09-20, lilleman. Serves goal 1. Across six projects scored with the four seats, Locality or Self-sufficiency was the lowest dimension at every seat and Navigation at none, so a layout fix would not lift a project under the floor. Valid while the scoring run keeps those four seats.
|
||||
|
||||
2026-10-15, lilleman, set by PR #283 on 2026-10-16: an item that lifts the lowest dimension moves each goal it cites by 9, since the lowest dimension caps the overall and lifting it is worth the most to the floor. Within a chunk its work comes first; in the plan, lilleman answering that PR's review, such items compete with the others like any item and don't rank first. PR #257's scoring run found Shape lowest, at 6.25, and `todo.md` items 124 and 125 were to lift it. Release 0.3.0's scoring runs found Locality lowest, at 5.75 and 5.88, so the 9 moved from those items to the ones lifting Locality.
|
||||
2026-10-15, lilleman, set by PR #283 on 2026-10-16: an item that lifts the lowest dimension moves each goal it cites by 9, since the lowest dimension caps the overall and lifting it is worth the most to the floor. Within a chunk its work comes first; in the plan, lilleman answering that PR's review, such items compete with the others like any item and don't rank first. PR #257's scoring run found Shape lowest, at 6.25, and two items were to lift it: `todo.md` item 124, and the one PR #367 shipped, which gave "record", "outcome" and "place" one meaning each in `bin/`. Release 0.3.0's scoring runs found Locality lowest, at 5.75 and 5.88, so the 9 moved from those items to the ones lifting Locality.
|
||||
|
||||
## loop-claude: a `claude` that cannot start is a failed worker run, retried
|
||||
|
||||
@@ -82,7 +92,7 @@ Valid while the comprehension panel measures how hard a project is to read.
|
||||
|
||||
2026-10-16, lilleman, confirmed when asked. Serves goal 1.2: the answer is written once, so the same question is not asked again.
|
||||
|
||||
`score-outcomes` shows a score's medians only where 10 finished chunks that each shipped one item hold an outcome for that score (`ENOUGH` in `bin/score_outcomes/outcomes.py`). The reading-cost trend waits for 10 chunks measured since its last reset: the last 5 against the 5 before them (`2 * WINDOW` in `bin/reading_cost/figures.py`). Both stay at 10.
|
||||
`score-outcomes` shows a score's medians only where 10 finished chunks that each shipped one item hold an outcome for that score (`ENOUGH` in `bin/score_outcomes/results.py`). The reading-cost trend waits for 10 chunks measured since its last reset: the last 5 against the 5 before them (`2 * WINDOW` in `bin/reading_cost/figures.py`). Both stay at 10.
|
||||
|
||||
Valid while no published standard sets either number, the reading-cost window stays 5, and until lilleman sets another.
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
|
||||
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 125 | 0.4.0 | | decision | **Give "record", "outcome" and "places" one meaning each across `bin/`, and rename `bin/larv_review/git.py` by what it holds.** | 3 | 1 | 3 | 5 | 5 | code health by 5, 1.4 by 3 | 10.2 |
|
||||
| 198 | 0.4.0 | | decision | **Count, per finished chunk, the findings that arose from misreading code that had already shipped, so code health's counter-signal shows.** | 3 | 2 | 2 | 5 | 5 | code health by 3, 1.1 by 2 | 6.2 |
|
||||
| 59 | 0.4.0 | | | **Bound a run of unfinished verdicts on one chunk, so a worker and judge that disagree cannot cycle unattended without end.** | 2 | 1 | 3 | 4 | 3 | 1.4 by 6 | 6.0 |
|
||||
| 167 | 0.4.0 | | | **Refuse a `/security-review` or `/claude-security` Skill call below the project's rung, as rule-hook refuses such an Agent call.** | 3 | 2 | 3 | 6 | 2 | 1.1 by 4 | 5.6 |
|
||||
@@ -37,7 +36,7 @@
|
||||
| 81 | 0.5.0 | | principle | **Have `workflow-bench/bench.py` return each run's failure as a value, so one failed clone does not hide the later runs' results.** | 1 | 2 | 1 | 2 | 2 | 1.1 by 4 | 5.6 |
|
||||
| 102 | 0.5.0 | | | **Give loop-claude's retry delay an environment override, so the black-box test stops replacing `time.sleep` through `sitecustomize`.** | 1 | 1 | 2 | 2 | 2 | 1.1 by 4 | 5.6 |
|
||||
| 173 | 0.5.0 | | | **Have rule-hook's `todo-rank-check` run a worktree's own `bin/rank_todo.py` when the worktree is of this repo, so a branch that changes `todo.md`'s shape commits without an override.** | 2 | 1 | 3 | 4 | 2 | 1.1 by 2, 1.4 by 2 | 5.5 |
|
||||
| 101 | 0.5.0 | | principle | **Have `larv_review/record.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.** | 2 | 1 | 2 | 3 | 3 | 1.1 by 5 | 5.3 |
|
||||
| 101 | 0.5.0 | | principle | **Have `larv_review/history_commands.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.** | 2 | 1 | 2 | 3 | 3 | 1.1 by 5 | 5.3 |
|
||||
| 110 | 0.5.0 | | | **Count, for each finished chunk, the later changes that fixed code it wrote, and show them beside its guessed E.** | 3 | 2 | 2 | 5 | 5 | 1.1 by 6, 1.4 by 6 | 4.9 |
|
||||
| 12 | 0.5.0 | | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 3 | 2 | 1 | 4 | 4 | 1.1 by 6 | 4.2 |
|
||||
| 135 | 0.5.0 | | | **Fingerprint only the modules loop-claude imports, so editing another tool's code no longer reloads a running loop.** | 2 | 2 | 3 | 5 | 3 | 1.1 by 4 | 4.2 |
|
||||
@@ -66,14 +65,10 @@
|
||||
| 197 | 0.5.0 | | | **Let the human edit what they type at loop-claude's terminal questions with the arrow keys, so Left moves the cursor in place of printing `^[[D`.** | 2 | 2 | 3 | 5 | 2 | 1.5 by 1 | 1.3 |
|
||||
| 52 | 0.5.0 | | principle | **Record why this checkout lives in Claude Code's own directory, or move it out and link it in.** | 4 | 3 | 4 | 9 | 6 | 1.1 by 4 | 0.9 |
|
||||
| 170 | 0.6.0 | | decision | **Re-tune `WORKER_STEPS` and `REVIEW_STEPS` once 50 chunks have shipped under them, from how well L, E and I order their review rounds and 🔴 findings.** | 2 | 2 | 3 | 5 | 2 | 1.1 by 3, 1.4 by 5 | 10.9 |
|
||||
| 218 | 0.6.0 | 170 | | **Drop `LEGACY_WORKER` from `bin/model_routing/route.py`, so `route_from` reads only the JSON record.** | 1 | 2 | 1 | 2 | 1 | code health by 2 | 14.4 |
|
||||
| 218 | 0.6.0 | 170 | | **Drop `LEGACY_WORKER` from `bin/model_routing/route.py`, so `route_from` reads only JSON worker lines.** | 1 | 2 | 1 | 2 | 1 | code health by 2 | 14.4 |
|
||||
|
||||
## Details
|
||||
|
||||
### 125. Give "record", "outcome" and "places" one meaning each across `bin/`, and rename `bin/larv_review/git.py` by what it holds.
|
||||
|
||||
The inherited architect of PR #257's scoring run named `bin/reading_cost/measure.py:16` `Reading` and `bin/larv_review/git.py:6` `history_changes`, and found each word used for several concepts. "Record": the history writer `larv_review/record.py`, the judge's `loop_claude/project/records.py`, and `reading_cost/figures.py`'s `RECORD`. "Outcome": `transcript_archive/outcomes.py`, `score_outcomes/outcomes.py` `Outcome` and `rule_hook/rules.py` `Outcome`. "Places": `comprehension_history.Place` and `Reading.places`. `larv_review/git.py` holds history logic, not a git runner. Make `Reading` and `history_changes` easier to read.
|
||||
|
||||
### 198. Count, per finished chunk, the findings that arose from misreading code that had already shipped, so code health's counter-signal shows.
|
||||
|
||||
Code health's counter-signal (`CLAUDE.md` → "Shared goal") has no count, since no tool records why a finding arose. Have each reviewer tag a finding that arose because the implementer misread code that had already shipped, and have `score-outcomes` count the tagged findings per finished chunk. Item 110's later fixes stand for the defects that keep reappearing.
|
||||
@@ -176,9 +171,9 @@ Found by the principles audit of PR #242. `bin/loop_claude/test_black_box.py:113
|
||||
|
||||
Raised by the overrides pass of PR #296. The hook imports the main checkout's `rank_todo.py`, so while a branch of this repo changes the table's shape, every commit touching `todo.md` needs a `todo-rank-check` override: 33 log lines across four branches, the score-outcomes, plan-136, plan-148 and item-146 ones. The hook then checks nothing, and the rule rests on the worker chaining the branch's own check. Run another repo's `bin/rank_todo.py` never: that would run its code before every commit.
|
||||
|
||||
### 101. Have `larv_review/record.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.
|
||||
### 101. Have `larv_review/history_commands.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.
|
||||
|
||||
Found by the principles audit of PR #242, against "Functional core, imperative shell" (`principles/technical-principles.md`). `bin/larv_review/record.py:67` reads `time.time()`, then checks git (`:70`, `commit_exists`) and the history file (`:75`) inside the function that parses the seats file and decides. A test cannot fix the heading's time, and every test of `read_seats` touches the real checkout.
|
||||
Found by the principles audit of PR #242, against "Functional core, imperative shell" (`principles/technical-principles.md`). `bin/larv_review/history_commands.py:87` reads `time.time()`, then checks git (`:90`, `commit_exists`) and the history file (`:92`) inside the function that parses the seats file and decides. A test cannot fix the heading's time, and every test of `read_seats` touches the real checkout.
|
||||
|
||||
### 110. Count, for each finished chunk, the later changes that fixed code it wrote, and show them beside its guessed E.
|
||||
|
||||
@@ -332,6 +327,6 @@ The principles pass of PR #301 found that only `routed_by` in `bin/loop_claude/c
|
||||
|
||||
`docs/decisions.md` → `routing: one ladder…` calls the steps a first guess. Start this item once 50 chunks have merged since PR #339 set today's steps, each shipping one item scored with L, E and I. By 2026-10-23, 15 had: those of items 117, 126, 122, 207, 130, 154, 186, 66, 121, 213, 137, 212, 214, 210 and 216. Then read `score-outcomes`' rank correlations for L, E and I and the routed-chunks table of `routing-figures.md`, and propose new steps to lilleman, whose call they are.
|
||||
|
||||
### 218. Drop `LEGACY_WORKER` from `bin/model_routing/route.py`, so `route_from` reads only the JSON record.
|
||||
### 218. Drop `LEGACY_WORKER` from `bin/model_routing/route.py`, so `route_from` reads only JSON worker lines.
|
||||
|
||||
PR #357 archived the route as a JSON record, and kept `LEGACY_WORKER` to read the `Worker:` lines an older loop-claude wrote as sentences. The routing figures still count chunks whose archive holds such lines: 51 when PR #357 shipped, every chunk finished after the reset of 2026-10-21T22:16:41Z. A chunk that changes the routing steps has a `Reset` appended to `routing-calls-raised.md`, after which the figures count only chunks that hold records. This item waits for item 170 whether or not 170 changes the steps, as lilleman decided on 2026-10-23. Before starting, check that `routing-calls-raised.md` holds a `Reset` stamped after PR #357's merge; where none does, ask lilleman whether to append one. Delete `LEGACY_WORKER`, `legacy_fields`, the tests that read sentences (in `bin/loop_claude/test_loop_claude.py` and `bin/routing_figures/test_routing_figures.py`), and the sentence on them in "loop-claude --help". Reword `route_from`'s docstring, and the comment at `Rung.__str__` in `bin/model_routing/ladder.py`, which gives the old lines as a reason `parse` must keep reading that spelling.
|
||||
PR #357 archived the route as JSON on each `Worker:` line, and kept `LEGACY_WORKER` to read the lines an older loop-claude wrote as sentences. The routing figures still count chunks whose archive holds such lines: 51 when PR #357 shipped, every chunk finished after the reset of 2026-10-21T22:16:41Z. A chunk that changes the routing steps has a `Reset` appended to `routing-calls-raised.md`, after which the figures count only chunks whose worker lines are JSON. This item waits for item 170 whether or not 170 changes the steps, as lilleman decided on 2026-10-23. Before starting, check that `routing-calls-raised.md` holds a `Reset` stamped after PR #357's merge; where none does, ask lilleman whether to append one. Delete `LEGACY_WORKER`, `legacy_fields`, the tests that read sentences (in `bin/loop_claude/test_loop_claude.py` and `bin/routing_figures/test_routing_figures.py`), and the sentence on them in "loop-claude --help". Reword `route_from`'s docstring, and the comment at `Rung.__str__` in `bin/model_routing/ladder.py`, which gives the old lines as a reason `parse` must keep reading that spelling.
|
||||
|
||||
Reference in New Issue
Block a user