Files
claude-config/bin/larv_review/test_scoring_setup.py
T

189 lines
12 KiB
Python

from unittest import mock
import os
import pathlib
import re
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from larv_review.scoring_setup import bullets, file_globs, prepare_scoring, project_values, removed_setup, score_bullets
from failure import Failure
from test_env import isolated_env
from project_config.test_project_config import write_config
TOOL = pathlib.Path(__file__).resolve().parents[1] / "larv-review"
SKILL = pathlib.Path(__file__).resolve().parents[2] / "skills" / "comprehension-panel" / "SKILL.md"
VALUES = {"language": "Python", "kind": "a CLI", "domain": "parsing", "domain docs": "the spec", "3am question": "the parse hangs"}
class ScoreBullets(unittest.TestCase):
def test_gives_the_six_anchors_and_four_dimensions_unbolded_each_on_one_line(self):
anchors, dimensions = score_bullets(SKILL.read_text())
self.assertEqual(len(anchors), 6)
self.assertEqual(anchors[0], "- 1 — Spaghetti. No boundary holds. Finding where a behaviour lives is grep archaeology; changing it safely is guesswork, and the reliable technique is to change it and see what breaks. No map the reader builds survives contact with the code.")
self.assertTrue(anchors[5].startswith("- 10 — Duress-proof. Hungover,"))
self.assertEqual([line.split(" — ")[0] for line in dimensions], ["- Navigation", "- Locality", "- Shape", "- Self-sufficiency"])
self.assertEqual(dimensions[1], "- Locality — understanding and changing one piece without holding the rest: hidden state, order dependence, action at a distance.")
def test_reads_the_bullets_by_how_they_open_whatever_else_the_section_holds(self):
skill = SKILL.read_text()
with_extra = skill.replace("## The score\n\n", "## The score\n\n- A bullet before the anchors.\n\n", 1)
self.assertEqual(score_bullets(with_extra), score_bullets(skill))
def test_fails_where_an_anchor_is_missing(self):
self.assertIsInstance(score_bullets(SKILL.read_text().replace("- **10 — Duress-proof.**", "- Dropped:", 1)), Failure)
def test_fails_where_a_dimension_is_missing(self):
self.assertIsInstance(score_bullets(SKILL.read_text().replace("- **Shape** —", "- Dropped:", 1)), Failure)
def test_fails_where_a_fifth_dimension_is_added(self):
skill = SKILL.read_text()
self.assertIsInstance(score_bullets(skill.replace("- **Shape** —", "- **Extra** — a new one.\n- **Shape** —", 1)), Failure)
class ProjectValues(unittest.TestCase):
def test_reads_each_spelling_the_projects_use(self):
for spelling in (
"## Scoring run\n\nThe values:\n\n- `{language}`: Python\n- `{kind}`: a CLI\n- `{domain}`: parsing\n- `{domain docs}`: the spec\n- `{3am question}`: the parse\n hangs\n\n## Next\n\n- `{kind}`: other\n",
"# Scoring run\n\n- **Language:** Python\n- **Kind:** a CLI\n- **Domain:** parsing\n- **Domain docs:** the spec\n- **3am question:** the parse hangs\n",
"## 8. Scoring run\n\n- Language: Python.\n- Kind: a CLI.\n- Domain: parsing.\n- Domain docs: the spec.\n- 3am question: the parse hangs.\n",
):
with self.subTest(spelling=spelling[:20]):
self.assertEqual(project_values(spelling, "AGENTS.md"), VALUES)
def test_keeps_a_colon_an_x_and_an_ellipsis_in_a_value_and_skips_a_bullet_without_a_colon(self):
text = "## Scoring run\n\n- Language: Python\n- Kind: a CLI: small\n- Kind\n- Domain: LaTeX\n- Domain docs: the spec\n- 3am question: it hangs...\n"
self.assertEqual(project_values(text, "AGENTS.md"), {**VALUES, "kind": "a CLI: small", "domain": "LaTeX", "3am question": "it hangs..."})
def test_names_the_values_a_section_lacks_and_a_file_without_one(self):
self.assertEqual(project_values("## Scoring run\n\n- Language: Python\n- Kind: a CLI\n", "AGENTS.md"), Failure("AGENTS.md → \"Scoring run\" gives no {domain}, {domain docs} or {3am question}; add them as seat-prompts.md lists them"))
self.assertEqual(project_values("# P\n", "AGENTS.md"), Failure("AGENTS.md has no \"Scoring run\" section to fill the seats' prompts from; add one with {language}, {kind}, {domain}, {domain docs} and {3am question}"))
class FileGlobs(unittest.TestCase):
def test_names_no_test_files_where_every_glob_holds_none(self):
self.assertEqual(file_globs(["bin/a.py", "tests/x.py", "README.md"], []), "`bin/**` and `README.md`")
def test_gives_one_glob_per_directory_and_each_top_level_file_leaving_out_tests_and_the_forbidden_files(self):
paths = ["README.md", "ci.sh", "bin/run", "bin/a/main.py", "bin/a/test_main.py", "docs/decisions.md", "docs/guide.md", "plan/todo.md", "tests/x.py", "web/app.test.ts"]
self.assertEqual(file_globs(paths, ["plan/todo.md", "docs/decisions.md"]), "`bin/**` except test files, `docs/**`, `README.md` and `ci.sh`, test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder")
class Bullets(unittest.TestCase):
def test_joins_continuations_and_splits_runs_at_other_lines(self):
lines = [" stray", "- a ", " more", "Text", "- b", "", "- c"]
self.assertEqual(bullets(lines), [["- a more"], ["- b"], ["- c"]])
class PrepareScoring(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = pathlib.Path(os.path.realpath(temp.name))
self.env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(self.root)}
self.checkout = self.root / "proj"
self.checkout.mkdir()
self.git("init", "-q", "-b", "item-9")
(self.checkout / "README.md").write_text("# P\n")
(self.checkout / "bin").mkdir()
(self.checkout / "bin" / "tool.py").write_text("")
self.config = write_config(self.root, self.checkout) / "config.toml"
def git(self, *args):
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.checkout, check=True, env=self.env, capture_output=True, text=True).stdout
def commit(self, agents=None):
if agents is not None:
(self.checkout / "AGENTS.md").write_text(agents)
self.git("add", "-A")
self.git("commit", "-q", "-m", "start")
def prepare(self):
"""What prepare-scoring returns in the checkout, run in this process so mutation tests see the lines it runs."""
with mock.patch.dict(os.environ, self.env, clear=True):
return prepare_scoring(str(self.checkout), pathlib.Path.home())
def run_tool(self, *args):
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=self.checkout, env=self.env, text=True)
def test_writes_each_seats_prompt_with_every_placeholder_filled_and_a_worktree_at_head(self):
self.commit("# A\n\n## Scoring run\n\n- Language: Python\n- Kind: a CLI\n- Domain: parsing\n- Domain docs: the spec\n- 3am question: the parse hangs\n")
ran = self.prepare()
folder = self.root / ".local" / "state" / "larv-review" / "proj" / "scoring"
worktree = folder / "checkout"
prompts = [folder / f"{seat}.md" for seat in ("junior-a", "mid-a", "maintainability-senior", "inherited-architect")]
self.assertEqual(ran, f"Scoring worktree, at {self.git('rev-parse', '--short', 'HEAD').strip()}: {worktree}\nPrompts, each for one comprehension-seat agent:\n" + "\n".join(str(prompt) for prompt in prompts))
self.assertEqual(subprocess.run(["git", "rev-parse", "HEAD"], cwd=worktree, env=self.env, capture_output=True, text=True).stdout, self.git("rev-parse", "HEAD"))
junior, _, senior, architect = (prompt.read_text() for prompt in prompts)
for prompt in (junior, architect):
self.assertEqual(re.findall(r"\{[a-z0-9 ]+\}", prompt), [])
self.assertTrue(junior.startswith(f"You are one seat of a comprehension panel. Tools: Read, Grep, Glob, and Bash only for `git -C {worktree} ls-files` and `ls`."))
self.assertIn("\n\nYour background: ~2 years in Python, first codebase of this kind (a CLI); no exposure to the domain at all.\n\n", junior)
self.assertIn(f"Target: the project checked out at {worktree} (branch `item-9`). Read ONLY there — never {self.checkout}, which is a different version.", junior)
self.assertIn("every non-test tracked file: `bin/**`, `AGENTS.md` and `README.md`. Do NOT open todo.md, docs/decisions.md,", junior)
self.assertIn("\n- 7 — Predictable. The layout answers", junior)
self.assertIn("Report, in this shape:", senior)
self.assertIn("~15 years taking over other people's systems", architect)
self.assertIn("The 3am question: the parse hangs — walking in cold", architect)
self.assertNotIn("Report, in this shape:", architect)
(folder / "mid-a.md").write_text("stale")
again = self.prepare()
self.assertTrue(again.startswith(f"Removed the scoring run set up before in {folder}.\nScoring worktree, at "))
self.assertNotEqual((folder / "mid-a.md").read_text(), "stale")
self.assertEqual(len(self.git("worktree", "list").splitlines()), 2)
def test_says_it_could_not_remove_a_checkout_git_does_not_hold(self):
self.commit()
checkout = self.root / ".local" / "state" / "larv-review" / "proj" / "scoring" / "checkout"
checkout.mkdir(parents=True)
with mock.patch.dict(os.environ, self.env, clear=True):
removed = removed_setup(str(self.checkout), pathlib.Path.home())
self.assertIsInstance(removed, Failure)
self.assertTrue(removed.reason.startswith(f"could not remove the scoring worktree {checkout}: "), removed.reason)
self.assertTrue(checkout.exists())
def test_reads_the_values_and_forbids_the_records_where_the_config_puts_them(self):
values = "# A\n\n## Scoring run\n\n- Language: Go\n- Kind: a CLI\n- Domain: parsing\n- Domain docs: the spec\n- 3am question: the parse hangs\n"
self.config.write_text(self.config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"').replace('"checkout:todo.md"', '"checkout:plan/todo.md"').replace('"checkout:docs/comprehension-history.md"', '"folder:comprehension-history.md"'))
(self.config.parent / "AGENTS.md").write_text(values)
(self.checkout / "plan").mkdir()
(self.checkout / "plan" / "todo.md").write_text("# todo\n")
self.commit("# The owners' own AGENTS.md, with no values\n")
ran = self.prepare()
assert isinstance(ran, str), ran
junior = pathlib.Path(ran.splitlines()[2]).read_text()
self.assertIn("Your background: ~2 years in Go,", junior)
self.assertIn("every non-test tracked file: `bin/**`, `AGENTS.md` and `README.md`. Do NOT open plan/todo.md, docs/decisions.md or docs/reading-cost.md (the plan, the decision log and past chunks' reading cost), nor any review output.", junior)
def test_forbids_any_record_of_a_past_panel_where_the_config_puts_every_record_outside_the_checkout(self):
self.config.write_text(self.config.read_text().replace('"checkout:', '"folder:').replace('"folder:README.md"', '"checkout:README.md"'))
(self.config.parent / "AGENTS.md").write_text("# A\n\n## Scoring run\n\n- Language: Go\n- Kind: a CLI\n- Domain: parsing\n- Domain docs: the spec\n- 3am question: the parse hangs\n")
self.commit()
ran = self.prepare()
assert isinstance(ran, str), ran
self.assertIn("every non-test tracked file: `bin/**` and `README.md`. Do NOT open any record of a past panel, nor any review output.", pathlib.Path(ran.splitlines()[2]).read_text())
def test_refuses_writing_nothing_where_the_project_gives_no_values(self):
self.commit()
ran = self.run_tool("prepare-scoring")
self.assertEqual(ran.returncode, 1)
self.assertIn("AGENTS.md has no \"Scoring run\" section", ran.stderr)
self.assertFalse((self.root / ".local" / "state" / "larv-review" / "proj" / "scoring").exists())
self.assertEqual(len(self.git("worktree", "list").splitlines()), 1)
def test_fails_before_it_makes_the_worktree_where_a_file_it_reads_does_not_decode(self):
values = "## Scoring run\n\n- Language: Python\n- Kind: a CLI\n- Domain: parsing\n- Domain docs: the spec\n- 3am question: the parse hangs\n"
self.commit(values)
(self.checkout / "AGENTS.md").write_bytes(b"\xff")
unread = self.prepare()
assert isinstance(unread, Failure), unread
self.assertTrue(unread.reason.startswith(f"cannot read {self.checkout}/AGENTS.md: "), unread.reason)
self.assertEqual(len(self.git("worktree", "list").splitlines()), 1)
if __name__ == "__main__":
unittest.main()