Test the scorer's code-health weight and each failure it names
This commit is contained in:
@@ -46,7 +46,7 @@ def ranked(items: list[EpicItem], formula: str, epic_goals: str, repos: dict[str
|
||||
if each.repo not in scorings:
|
||||
if each.repo is not None and each.repo not in repos:
|
||||
return Failure(f"{name} is in the repo {each.repo}, whose goals were not read")
|
||||
scoring = chain_scoring(parsed, epic_goals, each.repo, repos.get(each.repo or ""))
|
||||
scoring = chain_scoring(parsed, epic_goals, each.repo, repos[each.repo] if each.repo else None)
|
||||
if isinstance(scoring, Failure):
|
||||
return scoring
|
||||
scorings[each.repo] = scoring
|
||||
@@ -69,6 +69,7 @@ def chain_scoring(formula: ast.Expression, epic_goals: str, repo: str | None, pr
|
||||
overall = latest_overall(premises.history) if premises and CODE_HEALTH in dict(goals) else None
|
||||
if isinstance(overall, Failure):
|
||||
return Failure(f"{repo}: {overall.reason}")
|
||||
# goal_weights fails only on a goal without a mark, which the check above already named by whose it is.
|
||||
weights = goal_weights(goals, overall, premises.floor if premises else decimal.Decimal(0))
|
||||
return weights if isinstance(weights, Failure) else ranking.Scoring(SHAPE, formula, 0, 0, dict(goals), weights)
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import dataclasses
|
||||
import decimal
|
||||
import pathlib
|
||||
import sys
|
||||
@@ -12,6 +13,8 @@ from plan.todo_table import Item, parse_moves
|
||||
EPIC_GOALS = "1. **Achieve:** Ship invoices.\n2. **Achieve:** Cut support mail.\n3. **Maintain:** Leak no customer data.\n"
|
||||
VOIDED = RepoPremises("## Goals\n\n**Void:** code health, since it is a one-off script.\n\n1. **Achieve:** Fast builds.\n", "", "", decimal.Decimal("7.0"))
|
||||
HEALTHY = RepoPremises("## Goals\n\n1. **Achieve:** Fast builds.\n2. **Maintain:** Stay on the LTS.\n", "", "", decimal.Decimal("7.0"))
|
||||
# A scoring run of a repo whose overall mean is {overall}.
|
||||
SCORED = "# Comprehension history\n\n## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\n| Seat | Overall |\n|---|---|\n| Mid A | 6 |\n| Mean | {overall} |\n"
|
||||
# What the three achieve goals of the chain epic 1, epic 2, repo 1 weigh together before they are scaled to sum to 1, each 0.97 of the one above.
|
||||
TOTAL = 1 + 0.97 + 0.97**2
|
||||
|
||||
@@ -95,6 +98,31 @@ class RankedTest(unittest.TestCase):
|
||||
def test_a_repos_goals_file_that_cannot_be_read_fails_naming_the_repo(self) -> None:
|
||||
self.assertEqual(order([item(4, "epic 1 by 5")], {"billing-api": RepoPremises("# Billing\n", "", "", decimal.Decimal("7.0"))}), Failure("billing-api: the goals file has no ## Goals heading"))
|
||||
|
||||
def test_code_health_weighs_four_times_its_place_while_the_repo_is_unmeasured_or_under_its_floor(self) -> None:
|
||||
total = 1 + 0.97 + 0.97**2 + 0.97**3
|
||||
for history, times in (("", 4), (SCORED.format(overall="6.9"), 4), (SCORED.format(overall="7.0"), 1)):
|
||||
with self.subTest(history=history):
|
||||
found = ranked([item(1, "code health by 10", repo="web"), item(2, "repo 1 by 10", repo="web")], "V", EPIC_GOALS, {"web": dataclasses.replace(HEALTHY, history=history)})
|
||||
assert not isinstance(found, Failure), found
|
||||
self.assertAlmostEqual(found[0][1], times * 10 * 0.97**2 / total)
|
||||
self.assertAlmostEqual(found[1][1], 10 * 0.97**3 / total)
|
||||
|
||||
def test_a_repo_under_its_own_floor_weighs_code_health_by_that_floor(self) -> None:
|
||||
found = ranked([item(1, "code health by 10", repo="web")], "V", EPIC_GOALS, {"web": dataclasses.replace(HEALTHY, history=SCORED.format(overall="7.5"), floor=decimal.Decimal("8.0"))})
|
||||
assert not isinstance(found, Failure), found
|
||||
self.assertAlmostEqual(found[0][1], 4 * 10 * 0.97**2 / (1 + 0.97 + 0.97**2 + 0.97**3))
|
||||
|
||||
def test_a_history_that_cannot_be_read_fails_naming_the_repo(self) -> None:
|
||||
found = order([item(1, "code health by 10", repo="web")], {"web": dataclasses.replace(HEALTHY, history=SCORED.format(overall="high"))})
|
||||
assert isinstance(found, Failure), found
|
||||
self.assertTrue(found.reason.startswith("web: ") and len(found.reason) > len("web: None"), found.reason)
|
||||
|
||||
def test_a_formula_that_gives_an_item_no_finite_score_fails_naming_the_item(self) -> None:
|
||||
self.assertEqual(order([item(4, "epic 1 by 5")], {"billing-api": VOIDED}, "10 ** 400 * V * 10 ** 400"), Failure("the formula gives item 4 no finite score"))
|
||||
|
||||
def test_each_goal_an_item_names_is_checked(self) -> None:
|
||||
self.assertEqual(order([item(4, "epic 1 by 5, repo 9 by 5")], {"billing-api": VOIDED}), Failure("item 4 names repo 9, which the ## Goals of billing-api's goals file does not number"))
|
||||
|
||||
def test_a_question_then_a_security_finding_rank_before_any_score(self) -> None:
|
||||
self.assertEqual(order([item(1, "epic 1 by 10"), item(2, "epic 2 by 1", exempt="security"), item(3, "", exempt="question")], {"billing-api": VOIDED}), [3, 2, 1])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user