Test the value after an equals sign, -h, a line next to a change and the proxy's refusals
CI / tests (pull_request) Successful in 3m54s

This commit is contained in:
2026-11-21 02:04:37 +01:00
parent ebbe926ab7
commit 1e96a397af
3 changed files with 14 additions and 8 deletions
+2 -2
View File
@@ -99,8 +99,8 @@ class TableTest(unittest.TestCase):
])
def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self):
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=90, reads=2)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +50 % | +2 |")
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=120, reads=2)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +100 % | +2 |")
def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self):
results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)]
+1
View File
@@ -189,6 +189,7 @@ class EdgesTest(unittest.TestCase):
def test_a_message_ends_at_the_next_line_indented_as_its_first(self):
output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n pick_test.go:50: got 5\n"
self.assertEqual(task_text(output), "got 3\nwant 4")
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n a line the test logged\n"), "got 3")
def test_a_message_runs_to_the_end_of_the_output(self):
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4")
+11 -6
View File
@@ -22,8 +22,8 @@ from mutate_changes.tool_process import Ran
from task_bench.figures import Result
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
# The head moves Add down a line and rewrites Twice's line.
HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a")
# The head adds a line right above Add, which moves it down, and rewrites Twice's line.
HEAD_CODE = BASE_CODE.replace("func Add", "// Add adds.\nfunc Add").replace("a * 2", "2 * a")
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
TESTS = textwrap.dedent("""\
grep -q 'a + b' calc.go && exit 0
@@ -71,7 +71,7 @@ CLAUDE = textwrap.dedent("""\
print(json.dumps(call("Read", "/work/README.md")))
print(json.dumps(call("Read", "/work/calc.go")))
code = pathlib.Path("calc.go")
if code.read_text().startswith("//"):
if "// Add adds." in code.read_text():
code.write_text(code.read_text().replace("a - b", "a + b"))
else:
pathlib.Path("calc_test.go").write_text("exit 0\\n")
@@ -279,6 +279,11 @@ class MainTest(unittest.TestCase):
code, errors = self.run_main(*argv)
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
def test_takes_a_value_after_an_equals_sign_too(self):
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["--image=golang:1.27.4", "--test=go test ./...", "--bugs=2"], {"CLAUDE_CODE_OAUTH_TOKEN": "t", "HOME": "/nonexistent"}, lambda home: Failure("read the places")), 1)
self.assertEqual(errors.getvalue(), "read the places\n")
def test_refuses_a_missing_image_or_test_command(self):
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
@@ -292,8 +297,8 @@ class MainTest(unittest.TestCase):
def test_prints_its_help(self):
with contextlib.redirect_stdout(io.StringIO()) as out:
self.assertEqual(main.main(["--help"], {}), 0)
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
self.assertEqual((main.main(["--help"], {}), main.main(["-h"], {})), (0, 0))
self.assertEqual(out.getvalue().count("usage: task-bench --image <image> --test <shell text>"), 2)
def test_records_a_run_that_did_not_finish_without_figures(self):
@@ -483,7 +488,7 @@ class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
async def test_answers_403_to_any_other_target_or_method(self):
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"):
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), f"GET {self.allowed} HTTP/1.1\r\n\r\n".encode(), f"CONNECT {self.allowed}\r\n\r\n".encode(), b"junk\r\n\r\n"):
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")