Files

226 lines
11 KiB
Python

"""task-bench's planter: which files it mutates, which mutants it draws, how it re-applies one and what it tells Haiku."""
import json
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from task_bench.plant import drawn, go_files, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
from failure import Failure
from mutate_changes.tool_process import Ran
PLACES = """## 2026-11-19T22:33:15Z, PR #646 at a357cc7b
Junior A, hardest first:
- Locality, Shape: `internal/render/pick.go:287` `parked`: three steps held at once
- Shape: `internal/render/pick_test.go:12` `TestPick`: a long table
- Self-sufficiency: `README.md:89` "Round cap": three sections
- Locality: `cli/report.go:41` the comment: points to `cli/gone.go`
Mid A, hardest first:
- Locality: `internal/render/pick.go:354` `resumed`: a loop
- Locality: `data-import/geo.py:3` `load`: a long chain
"""
# go test's report of an assertion that failed in a subtest, then of one whose message runs over two lines.
GO_FAILURE = """ok \texample.com/calc/cli\t0.004s
--- FAIL: TestPick (0.00s)
--- FAIL: TestPick/empty_input (0.00s)
pick_test.go:42: Pick() = 3,
want 4
pick_test.go:50: got 1 row, want 2
FAIL
FAIL\texample.com/calc/render\t0.012s
FAIL
"""
# testify prints the test's file and name inside the message.
TESTIFY_FAILURE = """--- FAIL: TestTotal (0.00s)
total_test.go:12:
\tError Trace:\t/work/total_test.go:12
\tError: \tNot equal:
\t \texpected: 1
\t \tactual : 2
\tTest: \tTestTotal
FAIL
"""
PANIC = """--- FAIL: TestPick (0.00s)
panic: runtime error: index out of range [3] with length 3 [recovered]
FAIL\texample.com/calc/render\t0.012s
"""
class PlaceFilesTest(unittest.TestCase):
def test_names_each_go_file_that_holds_a_place_once_and_no_test_file_or_other_language(self):
self.assertEqual(place_files(PLACES), ["cli/report.go", "internal/render/pick.go"])
def test_names_none_where_the_seats_found_no_places(self):
self.assertEqual(place_files("Seats found no places hard."), [])
class DrawnTest(unittest.TestCase):
MUTANTS = [Mutant("a.go", line, 3, "ARITHMETIC_BASE") for line in range(1, 40)]
def test_a_seed_draws_the_same_order_whatever_order_gremlins_listed_the_mutants_in(self):
self.assertEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS[::-1], 7))
def test_draws_every_mutant_once(self):
self.assertEqual(sorted(drawn(self.MUTANTS, 7)), self.MUTANTS)
def test_another_seed_draws_another_order(self):
self.assertNotEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS, 8))
class MutatedTest(unittest.TestCase):
SOURCE = b"package calc\n\nfunc f(a, b int) int {\n\tif a >= b {\n\t\treturn a - b\n\t}\n\tn -= 1\n\treturn a\n}\n"
def planted(self, line: int, column: int, mutator: str, source: bytes = SOURCE) -> str | None:
found = mutated(source, Mutant("calc.go", line, column, mutator))
return None if found is None else found.decode().splitlines()[line - 1]
def test_swaps_the_token_at_the_line_and_column_as_gremlins_mutator_does(self):
self.assertEqual(self.planted(4, 7, "CONDITIONALS_NEGATION"), "\tif a < b {")
self.assertEqual(self.planted(4, 7, "CONDITIONALS_BOUNDARY"), "\tif a > b {")
self.assertEqual(self.planted(5, 12, "ARITHMETIC_BASE"), "\t\treturn a + b")
self.assertEqual(self.planted(7, 4, "INVERT_ASSIGNMENTS"), "\tn += 1")
self.assertEqual(self.planted(7, 4, "REMOVE_SELF_ASSIGNMENTS"), "\tn = 1")
def test_changes_no_other_byte(self):
found = mutated(self.SOURCE, Mutant("calc.go", 5, 12, "ARITHMETIC_BASE"))
self.assertEqual(found, self.SOURCE.replace(b"a - b", b"a + b"))
def test_counts_the_column_in_bytes_as_go_does(self):
self.assertEqual(self.planted(1, 15, "CONDITIONALS_NEGATION", 'x := "åäö" == y\n'.encode()), 'x := "åäö" != y')
def test_is_none_where_the_place_holds_no_token_of_the_mutator(self):
self.assertIsNone(self.planted(4, 6, "CONDITIONALS_NEGATION"))
self.assertIsNone(self.planted(5, 12, "CONDITIONALS_NEGATION"))
self.assertIsNone(self.planted(40, 1, "ARITHMETIC_BASE"))
self.assertIsNone(self.planted(4, 70, "ARITHMETIC_BASE"))
self.assertIsNone(self.planted(4, 7, "A_MUTATOR_GREMLINS_ADDS_LATER"))
class TaskTextTest(unittest.TestCase):
def test_is_the_first_failed_tests_message_without_the_tests_name_file_or_line(self):
self.assertEqual(task_text(GO_FAILURE), "Pick() = 3,\nwant 4")
def test_leaves_out_each_line_that_names_the_test_or_its_file(self):
self.assertEqual(task_text(TESTIFY_FAILURE), "Error: \tNot equal:\nexpected: 1\nactual : 2")
def test_is_none_where_no_assertion_failed_as_on_a_panic_or_a_passing_run(self):
self.assertIsNone(task_text(PANIC))
self.assertIsNone(task_text("ok \texample.com/calc\t0.004s\n"))
def test_takes_the_message_of_a_test_that_failed_not_what_a_passing_test_logged(self):
output = "=== RUN TestServer\n server_test.go:12: listening on 127.0.0.1:0\n--- PASS: TestServer (0.00s)\n=== RUN TestPick\n pick_test.go:42: got 3, want 4\n--- FAIL: TestPick (0.00s)\nFAIL\n"
self.assertEqual(task_text(output), "got 3, want 4")
def test_takes_the_last_message_of_the_failed_test_since_a_test_logs_before_it_asserts(self):
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:40: input [1 2 3]\n pick_test.go:42: got 3, want 4\nFAIL\n"), "got 3, want 4")
def test_takes_the_first_failed_test_that_has_a_message(self):
output = "--- FAIL: TestPick (0.00s)\n --- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: got 3\n--- FAIL: TestLast (0.00s)\n pick_test.go:50: got 5\nFAIL\n"
self.assertEqual(task_text(output), "got 3")
def test_a_line_printed_before_any_tests_mark_is_no_tests_message(self):
self.assertEqual(task_text(" setup_test.go:3: starting\n--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\nFAIL\n"), "got 3")
self.assertIsNone(task_text(" setup_test.go:3: starting\n--- FAIL: TestPick (0.00s)\nFAIL\n"))
def test_takes_the_failed_subtests_message_over_what_its_parent_logged_before_it(self):
output = "--- FAIL: TestPick (0.00s)\n pick_test.go:10: three cases\n --- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: got 3\nFAIL\n"
self.assertEqual(task_text(output), "got 3")
def test_is_none_where_only_passing_tests_logged(self):
self.assertIsNone(task_text("=== RUN TestServer\n server_test.go:12: listening\n--- PASS: TestServer (0.00s)\nPASS\n"))
def test_is_none_where_the_message_does_not_say_anything_beyond_the_test(self):
self.assertIsNone(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: TestPick failed\nFAIL\n"))
class LineAtHeadTest(unittest.TestCase):
# Lines 3 and 4 became one, two lines went in after line 9, and line 20 went.
DIFF = "diff --git a/calc.go b/calc.go\n--- a/calc.go\n+++ b/calc.go\n@@ -3,2 +3 @@\n-a\n-b\n+c\n@@ -9,0 +9,2 @@\n+d\n+e\n@@ -20 +20,0 @@\n-f\n"
def test_a_line_above_every_change_keeps_its_number(self):
self.assertEqual(line_at_head(self.DIFF, 2), 2)
def test_a_line_below_a_change_moves_by_the_lines_it_added_and_removed(self):
self.assertEqual([line_at_head(self.DIFF, line) for line in (5, 9, 10, 19, 21)], [4, 8, 11, 20, 21])
def test_a_line_the_head_changed_or_removed_has_none(self):
self.assertEqual([line_at_head(self.DIFF, line) for line in (3, 4, 20)], [None, None, None])
def test_no_diff_keeps_every_line(self):
self.assertEqual(line_at_head("", 12), 12)
class KilledMutantsTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.top = pathlib.Path(folder.name)
for path in ("calc.go", "calc_test.go", "render/pick.go", "render/a+b.go", "render/notes.md", "cli/main.go"):
(self.top / path).parent.mkdir(parents=True, exist_ok=True)
(self.top / path).write_text("package x\n")
self.ran: list[tuple[list[str], str]] = []
def gremlins(self, report: dict | None, code: int = 0):
def run(argv: list[str], cwd: str) -> Ran:
self.ran.append((argv, cwd))
if report is not None:
pathlib.Path(cwd, argv[argv.index("--output") + 1]).write_text(json.dumps(report))
return Ran(code, "gremlins said so")
return run
def test_names_each_go_file_gremlins_would_mutate_by_its_path_from_the_top(self):
self.assertEqual(sorted(go_files(str(self.top))), ["calc.go", "cli/main.go", "render/a+b.go", "render/pick.go"])
def test_has_gremlins_leave_out_every_other_go_file_by_its_whole_path(self):
killed_mutants(str(self.top), ["render/pick.go"], self.gremlins({"files": []}), 3)
argv, cwd = self.ran[0]
self.assertEqual(cwd, str(self.top))
self.assertEqual(argv[:8], ["gremlins", "unleash", "--exclude-files", r"^calc\.go$", "--exclude-files", r"^cli/main\.go$", "--exclude-files", r"^render/a\+b\.go$"])
self.assertEqual((argv[8], argv[10:]), ("--output", ["--workers", "3"]))
def test_keeps_the_killed_mutants_of_the_files_asked_for(self):
report = {"files": [
{"file_name": "render/pick.go", "mutations": [{"line": 4, "column": 7, "status": "KILLED", "type": "ARITHMETIC_BASE"}, {"line": 5, "column": 2, "status": "LIVED", "type": "INVERT_LOGICAL"}, {"line": 6, "column": 2, "status": "NOT COVERED", "type": "INVERT_LOGICAL"}]},
{"file_name": "calc.go", "mutations": [{"line": 1, "column": 1, "status": "KILLED", "type": "ARITHMETIC_BASE"}]},
]}
self.assertEqual(killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(report), 3), [Mutant("render/pick.go", 4, 7, "ARITHMETIC_BASE")])
def test_a_gremlins_that_fails_is_a_failure_with_what_it_printed(self):
found = killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(None, code=1), 3)
self.assertIsInstance(found, Failure)
self.assertIn("gremlins said so", found.reason)
class EdgesTest(unittest.TestCase):
def test_mutates_the_last_line_of_a_file(self):
self.assertEqual(mutated(b"a\nb + c", Mutant("calc.go", 2, 3, "ARITHMETIC_BASE")), b"a\nb - c")
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 3, 3, "ARITHMETIC_BASE")))
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 0, 3, "ARITHMETIC_BASE")))
def test_takes_the_longest_token_at_the_column(self):
self.assertEqual(mutated(b"a >= b", Mutant("calc.go", 1, 3, "CONDITIONALS_BOUNDARY")), b"a > b")
self.assertEqual(mutated(b"a &^ b", Mutant("calc.go", 1, 3, "INVERT_BITWISE")), b"a & b")
def test_a_message_ends_at_the_next_line_indented_as_its_first(self):
output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n--- FAIL: TestLast (0.00s)\n pick_test.go:50: got 5\n"
self.assertEqual(task_text(output), "got 3\nwant 4")
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n a line the test logged\n"), "got 3")
def test_a_message_runs_to_the_end_of_the_output(self):
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4")
def test_a_message_may_name_a_part_of_a_subtests_name_that_is_no_test(self):
output = "--- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: empty input: got 3\n in TestPick\n in TestPick/empty\n"
self.assertEqual(task_text(output), "empty input: got 3")
if __name__ == "__main__":
unittest.main()