204 lines
9.4 KiB
Python
204 lines
9.4 KiB
Python
"""task-bench's planter: which files it mutates, which mutants it draws, how it re-applies one and what it tells Haiku."""
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from task_bench.plant import drawn, go_files, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
|
|
from failure import Failure
|
|
from mutate_changes.tool_process import Ran
|
|
|
|
PLACES = """## 2026-11-19T22:33:15Z, PR #646 at a357cc7b
|
|
|
|
Junior A, hardest first:
|
|
|
|
- Locality, Shape: `internal/render/pick.go:287` `parked`: three steps held at once
|
|
- Shape: `internal/render/pick_test.go:12` `TestPick`: a long table
|
|
- Self-sufficiency: `README.md:89` "Round cap": three sections
|
|
- Locality: `cli/report.go:41` the comment: points to `cli/gone.go`
|
|
|
|
Mid A, hardest first:
|
|
|
|
- Locality: `internal/render/pick.go:354` `resumed`: a loop
|
|
- Locality: `data-import/geo.py:3` `load`: a long chain
|
|
"""
|
|
# go test's report of an assertion that failed in a subtest, then of one whose message runs over two lines.
|
|
GO_FAILURE = """ok \texample.com/calc/cli\t0.004s
|
|
--- FAIL: TestPick (0.00s)
|
|
--- FAIL: TestPick/empty_input (0.00s)
|
|
pick_test.go:42: Pick() = 3,
|
|
want 4
|
|
pick_test.go:50: got 1 row, want 2
|
|
FAIL
|
|
FAIL\texample.com/calc/render\t0.012s
|
|
FAIL
|
|
"""
|
|
# testify prints the test's file and name inside the message.
|
|
TESTIFY_FAILURE = """--- FAIL: TestTotal (0.00s)
|
|
total_test.go:12:
|
|
\tError Trace:\t/work/total_test.go:12
|
|
\tError: \tNot equal:
|
|
\t \texpected: 1
|
|
\t \tactual : 2
|
|
\tTest: \tTestTotal
|
|
FAIL
|
|
"""
|
|
PANIC = """--- FAIL: TestPick (0.00s)
|
|
panic: runtime error: index out of range [3] with length 3 [recovered]
|
|
FAIL\texample.com/calc/render\t0.012s
|
|
"""
|
|
|
|
|
|
class PlaceFilesTest(unittest.TestCase):
|
|
def test_names_each_go_file_that_holds_a_place_once_and_no_test_file_or_other_language(self):
|
|
self.assertEqual(place_files(PLACES), ["cli/report.go", "internal/render/pick.go"])
|
|
|
|
def test_names_none_where_the_seats_found_no_places(self):
|
|
self.assertEqual(place_files("Seats found no places hard."), [])
|
|
|
|
|
|
class DrawnTest(unittest.TestCase):
|
|
MUTANTS = [Mutant("a.go", line, 3, "ARITHMETIC_BASE") for line in range(1, 40)]
|
|
|
|
def test_a_seed_draws_the_same_order_whatever_order_gremlins_listed_the_mutants_in(self):
|
|
self.assertEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS[::-1], 7))
|
|
|
|
def test_draws_every_mutant_once(self):
|
|
self.assertEqual(sorted(drawn(self.MUTANTS, 7)), self.MUTANTS)
|
|
|
|
def test_another_seed_draws_another_order(self):
|
|
self.assertNotEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS, 8))
|
|
|
|
|
|
class MutatedTest(unittest.TestCase):
|
|
SOURCE = b"package calc\n\nfunc f(a, b int) int {\n\tif a >= b {\n\t\treturn a - b\n\t}\n\tn -= 1\n\treturn a\n}\n"
|
|
|
|
def planted(self, line: int, column: int, mutator: str, source: bytes = SOURCE) -> str | None:
|
|
found = mutated(source, Mutant("calc.go", line, column, mutator))
|
|
return None if found is None else found.decode().splitlines()[line - 1]
|
|
|
|
def test_swaps_the_token_at_the_line_and_column_as_gremlins_mutator_does(self):
|
|
self.assertEqual(self.planted(4, 7, "CONDITIONALS_NEGATION"), "\tif a < b {")
|
|
self.assertEqual(self.planted(4, 7, "CONDITIONALS_BOUNDARY"), "\tif a > b {")
|
|
self.assertEqual(self.planted(5, 12, "ARITHMETIC_BASE"), "\t\treturn a + b")
|
|
self.assertEqual(self.planted(7, 4, "INVERT_ASSIGNMENTS"), "\tn += 1")
|
|
self.assertEqual(self.planted(7, 4, "REMOVE_SELF_ASSIGNMENTS"), "\tn = 1")
|
|
|
|
def test_changes_no_other_byte(self):
|
|
found = mutated(self.SOURCE, Mutant("calc.go", 5, 12, "ARITHMETIC_BASE"))
|
|
self.assertEqual(found, self.SOURCE.replace(b"a - b", b"a + b"))
|
|
|
|
def test_counts_the_column_in_bytes_as_go_does(self):
|
|
self.assertEqual(self.planted(1, 15, "CONDITIONALS_NEGATION", 'x := "åäö" == y\n'.encode()), 'x := "åäö" != y')
|
|
|
|
def test_is_none_where_the_place_holds_no_token_of_the_mutator(self):
|
|
self.assertIsNone(self.planted(4, 6, "CONDITIONALS_NEGATION"))
|
|
self.assertIsNone(self.planted(5, 12, "CONDITIONALS_NEGATION"))
|
|
self.assertIsNone(self.planted(40, 1, "ARITHMETIC_BASE"))
|
|
self.assertIsNone(self.planted(4, 70, "ARITHMETIC_BASE"))
|
|
self.assertIsNone(self.planted(4, 7, "A_MUTATOR_GREMLINS_ADDS_LATER"))
|
|
|
|
|
|
class TaskTextTest(unittest.TestCase):
|
|
def test_is_the_first_failing_assertions_message_without_the_tests_name_file_or_line(self):
|
|
self.assertEqual(task_text(GO_FAILURE), "Pick() = 3,\nwant 4")
|
|
|
|
def test_leaves_out_each_line_that_names_the_test_or_its_file(self):
|
|
self.assertEqual(task_text(TESTIFY_FAILURE), "Error: \tNot equal:\nexpected: 1\nactual : 2")
|
|
|
|
def test_is_none_where_no_assertion_failed_as_on_a_panic_or_a_passing_run(self):
|
|
self.assertIsNone(task_text(PANIC))
|
|
self.assertIsNone(task_text("ok \texample.com/calc\t0.004s\n"))
|
|
|
|
def test_is_none_where_the_message_says_nothing_beyond_the_test(self):
|
|
self.assertIsNone(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: TestPick failed\nFAIL\n"))
|
|
|
|
|
|
class LineAtHeadTest(unittest.TestCase):
|
|
# Lines 3 and 4 became one, two lines went in after line 9, and line 20 went.
|
|
DIFF = "diff --git a/calc.go b/calc.go\n--- a/calc.go\n+++ b/calc.go\n@@ -3,2 +3 @@\n-a\n-b\n+c\n@@ -9,0 +9,2 @@\n+d\n+e\n@@ -20 +20,0 @@\n-f\n"
|
|
|
|
def test_a_line_above_every_change_keeps_its_number(self):
|
|
self.assertEqual(line_at_head(self.DIFF, 2), 2)
|
|
|
|
def test_a_line_below_a_change_moves_by_the_lines_it_added_and_removed(self):
|
|
self.assertEqual([line_at_head(self.DIFF, line) for line in (5, 9, 10, 19, 21)], [4, 8, 11, 20, 21])
|
|
|
|
def test_a_line_the_head_changed_or_removed_has_none(self):
|
|
self.assertEqual([line_at_head(self.DIFF, line) for line in (3, 4, 20)], [None, None, None])
|
|
|
|
def test_no_diff_keeps_every_line(self):
|
|
self.assertEqual(line_at_head("", 12), 12)
|
|
|
|
|
|
class KilledMutantsTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.top = pathlib.Path(folder.name)
|
|
for path in ("calc.go", "calc_test.go", "render/pick.go", "render/a+b.go", "render/notes.md", "cli/main.go"):
|
|
(self.top / path).parent.mkdir(parents=True, exist_ok=True)
|
|
(self.top / path).write_text("package x\n")
|
|
self.ran: list[tuple[list[str], str]] = []
|
|
|
|
def gremlins(self, report: dict | None, code: int = 0):
|
|
def run(argv: list[str], cwd: str) -> Ran:
|
|
self.ran.append((argv, cwd))
|
|
if report is not None:
|
|
pathlib.Path(cwd, argv[argv.index("--output") + 1]).write_text(json.dumps(report))
|
|
return Ran(code, "gremlins said so")
|
|
|
|
return run
|
|
|
|
def test_names_each_go_file_gremlins_would_mutate_by_its_path_from_the_top(self):
|
|
self.assertEqual(sorted(go_files(str(self.top))), ["calc.go", "cli/main.go", "render/a+b.go", "render/pick.go"])
|
|
|
|
def test_has_gremlins_leave_out_every_other_go_file_by_its_whole_path(self):
|
|
killed_mutants(str(self.top), ["render/pick.go"], self.gremlins({"files": []}), 3)
|
|
argv, cwd = self.ran[0]
|
|
self.assertEqual(cwd, str(self.top))
|
|
self.assertEqual(argv[:8], ["gremlins", "unleash", "--exclude-files", r"^calc\.go$", "--exclude-files", r"^cli/main\.go$", "--exclude-files", r"^render/a\+b\.go$"])
|
|
self.assertEqual((argv[8], argv[10:]), ("--output", ["--workers", "3"]))
|
|
|
|
def test_keeps_the_killed_mutants_of_the_files_asked_for(self):
|
|
report = {"files": [
|
|
{"file_name": "render/pick.go", "mutations": [{"line": 4, "column": 7, "status": "KILLED", "type": "ARITHMETIC_BASE"}, {"line": 5, "column": 2, "status": "LIVED", "type": "INVERT_LOGICAL"}, {"line": 6, "column": 2, "status": "NOT COVERED", "type": "INVERT_LOGICAL"}]},
|
|
{"file_name": "calc.go", "mutations": [{"line": 1, "column": 1, "status": "KILLED", "type": "ARITHMETIC_BASE"}]},
|
|
]}
|
|
self.assertEqual(killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(report), 3), [Mutant("render/pick.go", 4, 7, "ARITHMETIC_BASE")])
|
|
|
|
def test_a_gremlins_that_fails_is_a_failure_with_what_it_printed(self):
|
|
found = killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(None, code=1), 3)
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("gremlins said so", found.reason)
|
|
|
|
|
|
class EdgesTest(unittest.TestCase):
|
|
def test_mutates_the_last_line_of_a_file(self):
|
|
self.assertEqual(mutated(b"a\nb + c", Mutant("calc.go", 2, 3, "ARITHMETIC_BASE")), b"a\nb - c")
|
|
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 3, 3, "ARITHMETIC_BASE")))
|
|
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 0, 3, "ARITHMETIC_BASE")))
|
|
|
|
def test_takes_the_longest_token_at_the_column(self):
|
|
self.assertEqual(mutated(b"a >= b", Mutant("calc.go", 1, 3, "CONDITIONALS_BOUNDARY")), b"a > b")
|
|
self.assertEqual(mutated(b"a &^ b", Mutant("calc.go", 1, 3, "INVERT_BITWISE")), b"a & b")
|
|
|
|
def test_a_message_ends_at_the_next_line_indented_as_its_first(self):
|
|
output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n pick_test.go:50: got 5\n"
|
|
self.assertEqual(task_text(output), "got 3\nwant 4")
|
|
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n a line the test logged\n"), "got 3")
|
|
|
|
def test_a_message_runs_to_the_end_of_the_output(self):
|
|
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4")
|
|
|
|
def test_a_message_may_name_a_part_of_a_subtests_name_that_is_no_test(self):
|
|
output = "--- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: empty input: got 3\n in TestPick\n in TestPick/empty\n"
|
|
self.assertEqual(task_text(output), "empty input: got 3")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|