213 lines
11 KiB
Python
213 lines
11 KiB
Python
"""Set up a scoring run: the worktree its seats read and each seat's prompt, filled from skills/comprehension-panel/seat-prompts.md."""
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
|
|
from review_rules.comprehension_history import DIMENSIONS
|
|
from failure import Failure
|
|
from text_files.markdown_text import FENCE, listed, section
|
|
from git_cli.git_repo import git
|
|
from project_config.main import located
|
|
from review_rules.review_folder import made_review_folder, review_folder
|
|
from project_config.settings import ProjectConfig
|
|
from record_store.stores import store
|
|
|
|
PANEL = pathlib.Path(__file__).resolve().parents[2] / "skills" / "comprehension-panel"
|
|
# The file name of each seat's prompt, its label in seat-prompts.md → "Backgrounds", and the heading of its report there.
|
|
SEATS = (
|
|
("junior-a", "Junior A", "Report: Junior A, Mid A and the maintainability senior"),
|
|
("mid-a", "Mid A", "Report: Junior A, Mid A and the maintainability senior"),
|
|
("maintainability-senior", "Senior, maintainability", "Report: Junior A, Mid A and the maintainability senior"),
|
|
("inherited-architect", "Architect, inherited", "Report: the inherited architect"),
|
|
)
|
|
# The dimensions a seat scores beside the overall, which comes last in DIMENSIONS.
|
|
PANEL_DIMENSIONS = DIMENSIONS[:-1]
|
|
# The agent under agents/ that runs each seat.
|
|
SEAT_AGENT = "comprehension-seat"
|
|
VALUE_NAMES = ("language", "kind", "domain", "domain docs", "3am question")
|
|
# The files no seat may open, by the setting that places each, and what each is.
|
|
FORBIDDEN = (("plan", "the plan"), ("decisions", "the decision log"), ("comprehension-history", "past panels' records"), ("reading-cost", "past chunks' reading cost"))
|
|
TEST_FILE = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.[^/]*$|[^/]*\.(test|spec)\.[^/]*$)")
|
|
TEST_FILES = "test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder"
|
|
|
|
|
|
def bullets(lines: list[str]) -> list[list[str]]:
|
|
"""Each run of `- ` bullets in lines, each bullet joined onto one line with its indented continuation lines."""
|
|
groups: list[list[str]] = []
|
|
inside = False
|
|
for line in lines:
|
|
if line.startswith("- "):
|
|
if not inside:
|
|
groups.append([])
|
|
groups[-1].append(line.rstrip())
|
|
inside = True
|
|
elif inside and line.startswith(" ") and line.strip():
|
|
groups[-1][-1] += " " + line.strip()
|
|
else:
|
|
inside = False
|
|
return groups
|
|
|
|
|
|
ANCHOR = re.compile(r"- \*\*\d+ — ")
|
|
DIMENSION = re.compile(r"- \*\*([^*]+)\*\* — ")
|
|
|
|
|
|
def score_bullets(skill: str) -> tuple[list[str], list[str]] | Failure:
|
|
"""The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line. A bullet is an anchor or a dimension by how it opens, `- **<n> — ` or `- **<name>** — `; the other bullets in the section are skipped, and the dimensions must come in the panel's order."""
|
|
lines = [line for group in bullets(section(skill, "The score") or []) for line in group]
|
|
anchors = [line.replace("**", "") for line in lines if ANCHOR.match(line)]
|
|
dimensions = [(match[1], line) for line in lines if (match := DIMENSION.match(line))]
|
|
if len(anchors) != 6 or tuple(name for name, _ in dimensions) != PANEL_DIMENSIONS:
|
|
return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer holds six bullets opening `- **<n> — ` and one opening `- **<name>** — ` for each of {listed(list(PANEL_DIMENSIONS), 'and')}, in that order")
|
|
return anchors, [line.replace("**", "") for _, line in dimensions]
|
|
|
|
|
|
def project_values(text: str, source: str) -> dict[str, str] | Failure:
|
|
"""Each value seat-prompts.md fills from the project, from the section of text, read from source, headed `Scoring run` or `<name>'s values`."""
|
|
lines = section(text, r"(\d+\.\s+)?Scoring run|.+'s values")
|
|
if lines is None:
|
|
return Failure(f"{source} has no \"Scoring run\" section to fill the seats' prompts from; add one with {listed([f'{{{name}}}' for name in VALUE_NAMES], 'and')}")
|
|
values = {}
|
|
for bullet in (line for group in bullets(lines) for line in group):
|
|
key, colon, value = bullet[2:].partition(":")
|
|
name = key.strip(" `*{}").lower()
|
|
if colon and name in VALUE_NAMES:
|
|
value = value.strip(" *")
|
|
values[name] = value[:-1] if value.endswith(".") and not value.endswith("..") else value
|
|
if missing := [name for name in VALUE_NAMES if name not in values]:
|
|
return Failure(f"{source} → \"Scoring run\" gives no {listed([f'{{{name}}}' for name in missing], 'or')}; add them as seat-prompts.md lists them")
|
|
return {name: values[name] for name in VALUE_NAMES}
|
|
|
|
|
|
def forbidden(config: ProjectConfig) -> tuple[list[str], str]:
|
|
"""The files no seat may open, as git names them, and the words that tell a seat so. The plan or a record that no commit holds is in neither: the scoring worktree doesn't hold it."""
|
|
held = [(path, what) for setting, what in FORBIDDEN if (path := config.tracked(setting) if setting == "plan" else store(config, setting).tracked)]
|
|
files = [path for path, _ in held]
|
|
return files, f"{listed(files, 'or')} ({listed([what for _, what in held], 'and')})" if held else "any record of a past panel"
|
|
|
|
|
|
def file_globs(paths: list[str], forbidden_files: list[str]) -> str:
|
|
"""Every tracked file of paths a seat reads, as one glob per top-level folder and each top-level file, with the test files the globs leave out."""
|
|
read = [path for path in paths if path not in forbidden_files and not TEST_FILE.search(path)]
|
|
folders = sorted({path.split("/")[0] for path in read if "/" in path})
|
|
tested = {path.split("/")[0] for path in paths if "/" in path and TEST_FILE.search(path)}
|
|
globs = [f"`{folder}/**`" + (" except test files" if folder in tested else "") for folder in folders]
|
|
return listed(globs + sorted(f"`{path}`" for path in read if "/" not in path), "and") + (f", {TEST_FILES}" if tested & set(folders) else "")
|
|
|
|
|
|
def read(path: pathlib.Path) -> str | Failure:
|
|
"""path's text, or a Failure naming path where it cannot be read or decoded."""
|
|
try:
|
|
return path.read_text()
|
|
except (OSError, UnicodeDecodeError) as failure:
|
|
return Failure(f"cannot read {path}: {failure}")
|
|
|
|
|
|
def prompt_parts() -> dict[str, str] | Failure:
|
|
"""seat-prompts.md's opening, shared brief and each report as their fenced text, and each seat's background, keyed by heading or label."""
|
|
path = PANEL / "seat-prompts.md"
|
|
text = read(PANEL / "seat-prompts.md")
|
|
if isinstance(text, Failure):
|
|
return text
|
|
parts = {}
|
|
for title in ("Opening", "Shared brief", *{report for _, _, report in SEATS}):
|
|
lines = section(text, re.escape(title)) or []
|
|
fences = [at for at, line in enumerate(lines) if FENCE.match(line)]
|
|
if len(fences) < 2:
|
|
return Failure(f"{path} → \"{title}\" holds no fenced prompt text")
|
|
parts[title] = "\n".join(lines[fences[0] + 1:fences[1]])
|
|
for line in (line for group in bullets(section(text, "Backgrounds") or []) for line in group):
|
|
label, _, background = line[2:].partition(": ")
|
|
parts[label] = background.strip("`")
|
|
if missing := [label for _, label, _ in SEATS if label not in parts]:
|
|
return Failure(f"{path} → \"Backgrounds\" names no {listed(missing, 'or')}")
|
|
return parts
|
|
|
|
|
|
def scoring_folder(worktree: str, home: pathlib.Path) -> pathlib.Path:
|
|
return pathlib.Path(review_folder(str(home), worktree)) / "scoring"
|
|
|
|
|
|
def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
|
|
"""Makes the scoring run's worktree at worktree's HEAD and writes each seat's prompt beside it, in the chunk's review folder; what it made, as the tool prints it."""
|
|
folder = scoring_folder(worktree, home)
|
|
# A setup record-scoring never removed, as where the seats failed, would otherwise block every later run.
|
|
replaced = folder.exists()
|
|
if replaced and isinstance(removed := removed_setup(worktree, home), Failure):
|
|
return removed
|
|
config = located(worktree, home)
|
|
if isinstance(config, Failure):
|
|
return config
|
|
# Where AGENTS.md is missing, this repo keeps the values in seat-prompts.md: .claude/rules/this-repo-keeps-no-agents-md.md.
|
|
source = config.paths["working-rules"]
|
|
if not source.is_file() and (own := pathlib.Path(worktree) / "skills" / "comprehension-panel" / "seat-prompts.md").is_file():
|
|
source = own
|
|
rules = read(source) if source.is_file() else ""
|
|
if isinstance(rules, Failure):
|
|
return rules
|
|
skill = read(PANEL / "SKILL.md")
|
|
if isinstance(skill, Failure):
|
|
return skill
|
|
values = project_values(rules, source.name)
|
|
if isinstance(values, Failure):
|
|
return values
|
|
parts = prompt_parts()
|
|
if isinstance(parts, Failure):
|
|
return parts
|
|
score = score_bullets(skill)
|
|
if isinstance(score, Failure):
|
|
return score
|
|
listing = git("worktree", "list", "--porcelain", cwd=worktree)
|
|
branch = git("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
|
|
head = git("rev-parse", "--short", "HEAD", cwd=worktree)
|
|
if isinstance(listing, Failure):
|
|
return listing
|
|
if isinstance(head, Failure):
|
|
return head
|
|
main = listing.splitlines()[0].removeprefix("worktree ")
|
|
made = made_review_folder(worktree, home)
|
|
if isinstance(made, Failure):
|
|
return made
|
|
checkout = folder / "checkout"
|
|
# The seats score the chunk's head, which holds origin's default branch since larv-review §4 step 1, as larv-review → "Scoring run" says.
|
|
added = git("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
|
|
tracked = added if isinstance(added, Failure) else git("ls-files", cwd=str(checkout))
|
|
if isinstance(tracked, Failure):
|
|
shutil.rmtree(folder, ignore_errors=True)
|
|
return tracked
|
|
forbidden_files, forbidden_words = forbidden(config)
|
|
filled = {
|
|
**values,
|
|
"checkout": str(checkout),
|
|
"branch": head.strip() if isinstance(branch, Failure) else branch.strip(),
|
|
"main": main,
|
|
"forbidden": forbidden_words,
|
|
"git": f"`git -C {checkout} ls-files`",
|
|
"files": file_globs(tracked.splitlines(), forbidden_files),
|
|
"anchors": "\n".join(score[0]),
|
|
"dimensions": "\n".join(score[1]),
|
|
}
|
|
placeholder = re.compile(r"\{(" + "|".join(re.escape(name) for name in filled) + r")\}")
|
|
written = []
|
|
for name, label, report in SEATS:
|
|
prompt = "\n\n".join((parts["Opening"], parts[label], parts["Shared brief"], parts[report]))
|
|
path = folder / f"{name}.md"
|
|
try:
|
|
path.write_text(placeholder.sub(lambda found: filled[found.group(1)], prompt) + "\n")
|
|
except OSError as failure:
|
|
return Failure(f"could not write the seat's prompt {path}: {failure}")
|
|
written.append(str(path))
|
|
before = f"Removed the scoring run set up before in {folder}.\n" if replaced else ""
|
|
return before + f"Scoring worktree, at {head.strip()}: {checkout}\nPrompts, each for one {SEAT_AGENT} agent:\n" + "\n".join(written)
|
|
|
|
|
|
def removed_setup(worktree: str, home: pathlib.Path) -> Failure | None:
|
|
"""Removes the scoring worktree and prompts prepare_scoring made for the chunk at worktree, where they exist."""
|
|
folder = scoring_folder(worktree, home)
|
|
if not folder.exists():
|
|
return None
|
|
if (folder / "checkout").exists() and isinstance(removed := git("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
|
|
return Failure(f"could not remove the scoring worktree {folder / 'checkout'}: {removed.reason}")
|
|
shutil.rmtree(folder, ignore_errors=True)
|
|
return None
|