Files
claude-config/bin/larv_review/scoring_setup.py
T

229 lines
12 KiB
Python

"""Set up a scoring run: the worktree its seats read and each seat's prompt, filled from skills/comprehension-panel/seat-prompts.md."""
import json
import pathlib
import re
import shutil
from review_rules.comprehension_history import DIMENSIONS
from failure import Failure
from text_files.markdown_text import FENCE, listed, section
from git_cli.git_repo import git
from project_config.main import located
from review_rules.review_folder import made_review_folder, review_folder
from project_config.settings import ProjectConfig
CLAUDE_MD = "CLAUDE.md"
PANEL = pathlib.Path(__file__).resolve().parents[2] / "skills" / "comprehension-panel"
# The file name of each seat's prompt, its label in seat-prompts.md → "Backgrounds", and the heading of its report there.
SEATS = (
("junior-a", "Junior A", "Report: Junior A, Mid A and the maintainability senior"),
("mid-a", "Mid A", "Report: Junior A, Mid A and the maintainability senior"),
("maintainability-senior", "Senior, maintainability", "Report: Junior A, Mid A and the maintainability senior"),
("inherited-architect", "Architect, inherited", "Report: the inherited architect"),
)
# The dimensions a seat scores beside the overall, which comes last in DIMENSIONS.
PANEL_DIMENSIONS = DIMENSIONS[:-1]
VALUE_NAMES = ("language", "kind", "domain", "domain docs", "3am question")
# The files no seat may open, by the setting that places each, and what each is.
FORBIDDEN = (("plan", "the plan"), ("decisions", "the decision log"), ("comprehension-history", "past panels' records"), ("reading-cost", "past chunks' reading cost"))
TEST_FILE = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.[^/]*$|[^/]*\.(test|spec)\.[^/]*$)")
TEST_FILES = "test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder"
def bullets(lines: list[str]) -> list[list[str]]:
"""Each run of `- ` bullets in lines, each bullet joined onto one line with its indented continuation lines."""
groups: list[list[str]] = []
inside = False
for line in lines:
if line.startswith("- "):
if not inside:
groups.append([])
groups[-1].append(line.rstrip())
inside = True
elif inside and line.startswith(" ") and line.strip():
groups[-1][-1] += " " + line.strip()
else:
inside = False
return groups
ANCHOR = re.compile(r"- \*\*\d+ — ")
DIMENSION = re.compile(r"- \*\*([^*]+)\*\* — ")
def score_bullets(skill: str) -> tuple[list[str], list[str]] | Failure:
"""The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line. A bullet is an anchor or a dimension by how it opens, `- **<n> — ` or `- **<name>** — `; the other bullets in the section are skipped, and the dimensions must come in the panel's order."""
lines = [line for group in bullets(section(skill, "The score") or []) for line in group]
anchors = [line.replace("**", "") for line in lines if ANCHOR.match(line)]
dimensions = [(match[1], line) for line in lines if (match := DIMENSION.match(line))]
if len(anchors) != 6 or tuple(name for name, _ in dimensions) != PANEL_DIMENSIONS:
return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer holds six bullets opening `- **<n> — ` and one opening `- **<name>** — ` for each of {listed(list(PANEL_DIMENSIONS), 'and')}, in that order")
return anchors, [line.replace("**", "") for _, line in dimensions]
def project_values(text: str, source: str) -> dict[str, str] | Failure:
"""Each value seat-prompts.md fills from the project, from the section of text, read from source, headed `Scoring run` or `<name>'s values`."""
lines = section(text, r"(\d+\.\s+)?Scoring run|.+'s values")
if lines is None:
return Failure(f"{source} has no \"Scoring run\" section to fill the seats' prompts from; add one with {listed([f'{{{name}}}' for name in VALUE_NAMES], 'and')}")
values = {}
for bullet in (line for group in bullets(lines) for line in group):
key, colon, value = bullet[2:].partition(":")
name = key.strip(" `*{}").lower()
if colon and name in VALUE_NAMES:
value = value.strip(" *")
values[name] = value[:-1] if value.endswith(".") and not value.endswith("..") else value
if missing := [name for name in VALUE_NAMES if name not in values]:
return Failure(f"{source} → \"Scoring run\" gives no {listed([f'{{{name}}}' for name in missing], 'or')}; add them as seat-prompts.md lists them")
return {name: values[name] for name in VALUE_NAMES}
def forbidden(config: ProjectConfig) -> tuple[list[str], str]:
"""The files no seat may open, as git names them, and the words that tell a seat so. A file the project's config puts outside the checkout is in neither: the scoring worktree doesn't hold it."""
held = [(path, what) for setting, what in FORBIDDEN if (path := config.tracked(setting))]
files = [path for path, _ in held]
return files, f"{listed(files, 'or')} ({listed([what for _, what in held], 'and')})" if held else "any record of a past panel"
def file_globs(paths: list[str], forbidden_files: list[str]) -> str:
"""Every tracked file of paths a seat reads, as one glob per top-level folder and each top-level file, with the test files the globs leave out."""
read = [path for path in paths if path not in forbidden_files and not TEST_FILE.search(path)]
folders = sorted({path.split("/")[0] for path in read if "/" in path})
tested = {path.split("/")[0] for path in paths if "/" in path and TEST_FILE.search(path)}
globs = [f"`{folder}/**`" + (" except test files" if folder in tested else "") for folder in folders]
return listed(globs + sorted(f"`{path}`" for path in read if "/" not in path), "and") + (f", {TEST_FILES}" if tested & set(folders) else "")
def preloaded_files(main: pathlib.Path, home: pathlib.Path) -> str:
"""The files Claude Code loads into a seat started in main before its first tool call: each CLAUDE.md, the files it imports, and each rule file without `paths:`, leaving out those `claudeMdExcludes` names."""
excludes = claude_md_excludes(main, home)
found = []
def kept(path: pathlib.Path) -> bool:
return not any(path.full_match(pattern) for pattern in excludes)
for shown, claude_md, rules in (("~/.claude/", home / ".claude" / CLAUDE_MD, home / ".claude" / "rules"), ("", main / CLAUDE_MD, main / ".claude" / "rules")):
if claude_md.is_file() and kept(claude_md):
found.append(f"`{shown}{CLAUDE_MD}`")
found += [f"`{shown}{name}`, which it imports" for name in re.findall(r"^@(\S+)", claude_md.read_text(), re.M)]
found += [f"`{shown}{rule.relative_to(claude_md.parent)}`" for rule in sorted(rules.glob("*.md")) if kept(rule) and not re.search(r"^paths:", rule.read_text(), re.M)]
return listed(found, "and") if found else "none"
def claude_md_excludes(main: pathlib.Path, home: pathlib.Path) -> list[str]:
"""The `claudeMdExcludes` globs of the user's settings and main's project and local settings, merged as Claude Code merges them across layers.
Managed settings are left out, since they may come from the claude.ai console or an MDM policy, where no file shows them."""
globs = []
for path in (home / ".claude" / "settings.json", main / ".claude" / "settings.json", main / ".claude" / "settings.local.json"):
try:
settings = json.loads(path.read_text())
except (OSError, ValueError):
continue
excludes = settings.get("claudeMdExcludes") if isinstance(settings, dict) else None
globs += [glob for glob in excludes if isinstance(glob, str)] if isinstance(excludes, list) else []
return globs
def prompt_parts() -> dict[str, str] | Failure:
"""seat-prompts.md's opening, shared brief and each report as their fenced text, and each seat's background, keyed by heading or label."""
path = PANEL / "seat-prompts.md"
try:
text = path.read_text()
except OSError as failure:
return Failure(f"cannot read {path}: {failure}")
parts = {}
for title in ("Opening", "Shared brief", *{report for _, _, report in SEATS}):
lines = section(text, re.escape(title)) or []
fences = [at for at, line in enumerate(lines) if FENCE.match(line)]
if len(fences) < 2:
return Failure(f"{path} → \"{title}\" holds no fenced prompt text")
parts[title] = "\n".join(lines[fences[0] + 1:fences[1]])
for line in (line for group in bullets(section(text, "Backgrounds") or []) for line in group):
label, _, background = line[2:].partition(": ")
parts[label] = background.strip("`")
if missing := [label for _, label, _ in SEATS if label not in parts]:
return Failure(f"{path} → \"Backgrounds\" names no {listed(missing, 'or')}")
return parts
def scoring_folder(worktree: str, home: pathlib.Path) -> pathlib.Path:
return pathlib.Path(review_folder(str(home), worktree)) / "scoring"
def prepare_scoring(worktree: str, preloaded: str | None) -> str | Failure:
"""Makes the scoring run's worktree at worktree's HEAD and writes each seat's prompt beside it, in the chunk's review folder; what it made, as the tool prints it."""
home = pathlib.Path.home()
folder = scoring_folder(worktree, home)
# A setup record-scoring never removed, as where the seats failed, would otherwise block every later run.
replaced = folder.exists()
if replaced and isinstance(removed := removed_setup(worktree), Failure):
return removed
config = located(worktree, home)
if isinstance(config, Failure):
return config
# Where AGENTS.md is missing, this repo keeps the values in seat-prompts.md: .claude/rules/this-repo-keeps-no-agents-md.md.
source = config.paths["working-rules"]
if not source.is_file() and (own := pathlib.Path(worktree) / "skills" / "comprehension-panel" / "seat-prompts.md").is_file():
source = own
values = project_values(source.read_text() if source.is_file() else "", source.name)
if isinstance(values, Failure):
return values
parts = prompt_parts()
if isinstance(parts, Failure):
return parts
score = score_bullets((PANEL / "SKILL.md").read_text())
if isinstance(score, Failure):
return score
listing = git("worktree", "list", "--porcelain", cwd=worktree)
branch = git("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
head = git("rev-parse", "--short", "HEAD", cwd=worktree)
if isinstance(listing, Failure):
return listing
if isinstance(head, Failure):
return head
main = listing.splitlines()[0].removeprefix("worktree ")
made = made_review_folder(worktree)
if isinstance(made, Failure):
return made
checkout = folder / "checkout"
# The seats score the chunk's head, which holds origin's default branch since larv-review §4 step 1, as larv-review → "Scoring run" says.
added = git("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
tracked = added if isinstance(added, Failure) else git("ls-files", cwd=str(checkout))
if isinstance(tracked, Failure):
shutil.rmtree(folder, ignore_errors=True)
return tracked
forbidden_files, forbidden_words = forbidden(config)
filled = {
**values,
"checkout": str(checkout),
"branch": head.strip() if isinstance(branch, Failure) else branch.strip(),
"main": main,
"forbidden": forbidden_words,
"git": f"`git -C {checkout} ls-files`",
"files": file_globs(tracked.splitlines(), forbidden_files),
"preloaded": preloaded if preloaded is not None else preloaded_files(pathlib.Path(main), home),
"anchors": "\n".join(score[0]),
"dimensions": "\n".join(score[1]),
}
placeholder = re.compile(r"\{(" + "|".join(re.escape(name) for name in filled) + r")\}")
written = []
for name, label, report in SEATS:
prompt = "\n\n".join((parts["Opening"], parts[label], parts["Shared brief"], parts[report]))
path = folder / f"{name}.md"
path.write_text(placeholder.sub(lambda found: filled[found.group(1)], prompt) + "\n")
written.append(str(path))
before = f"Removed the scoring run set up before in {folder}.\n" if replaced else ""
return before + f"Scoring worktree, at {head.strip()}: {checkout}\nPrompts:\n" + "\n".join(written)
def removed_setup(worktree: str) -> Failure | None:
"""Removes the scoring worktree and prompts prepare_scoring made for the chunk at worktree, where they exist."""
folder = scoring_folder(worktree, pathlib.Path.home())
if not folder.exists():
return None
if (folder / "checkout").exists() and isinstance(removed := git("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
return Failure(f"could not remove the scoring worktree {folder / 'checkout'}: {removed.reason}")
shutil.rmtree(folder, ignore_errors=True)
return None