Test the value after an equals sign, -h, a line next to a change and the proxy's refusals
CI / tests (pull_request) Successful in 3m54s
CI / tests (pull_request) Successful in 3m54s
This commit is contained in:
@@ -99,8 +99,8 @@ class TableTest(unittest.TestCase):
|
||||
])
|
||||
|
||||
def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self):
|
||||
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=90, reads=2)]
|
||||
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +50 % | +2 |")
|
||||
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=120, reads=2)]
|
||||
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +100 % | +2 |")
|
||||
|
||||
def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self):
|
||||
results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)]
|
||||
|
||||
@@ -189,6 +189,7 @@ class EdgesTest(unittest.TestCase):
|
||||
def test_a_message_ends_at_the_next_line_indented_as_its_first(self):
|
||||
output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n pick_test.go:50: got 5\n"
|
||||
self.assertEqual(task_text(output), "got 3\nwant 4")
|
||||
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n a line the test logged\n"), "got 3")
|
||||
|
||||
def test_a_message_runs_to_the_end_of_the_output(self):
|
||||
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4")
|
||||
|
||||
@@ -22,8 +22,8 @@ from mutate_changes.tool_process import Ran
|
||||
from task_bench.figures import Result
|
||||
|
||||
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
|
||||
# The head moves Add down a line and rewrites Twice's line.
|
||||
HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a")
|
||||
# The head adds a line right above Add, which moves it down, and rewrites Twice's line.
|
||||
HEAD_CODE = BASE_CODE.replace("func Add", "// Add adds.\nfunc Add").replace("a * 2", "2 * a")
|
||||
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
|
||||
TESTS = textwrap.dedent("""\
|
||||
grep -q 'a + b' calc.go && exit 0
|
||||
@@ -71,7 +71,7 @@ CLAUDE = textwrap.dedent("""\
|
||||
print(json.dumps(call("Read", "/work/README.md")))
|
||||
print(json.dumps(call("Read", "/work/calc.go")))
|
||||
code = pathlib.Path("calc.go")
|
||||
if code.read_text().startswith("//"):
|
||||
if "// Add adds." in code.read_text():
|
||||
code.write_text(code.read_text().replace("a - b", "a + b"))
|
||||
else:
|
||||
pathlib.Path("calc_test.go").write_text("exit 0\\n")
|
||||
@@ -279,6 +279,11 @@ class MainTest(unittest.TestCase):
|
||||
code, errors = self.run_main(*argv)
|
||||
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
|
||||
|
||||
def test_takes_a_value_after_an_equals_sign_too(self):
|
||||
with contextlib.redirect_stderr(io.StringIO()) as errors:
|
||||
self.assertEqual(main.main(["--image=golang:1.27.4", "--test=go test ./...", "--bugs=2"], {"CLAUDE_CODE_OAUTH_TOKEN": "t", "HOME": "/nonexistent"}, lambda home: Failure("read the places")), 1)
|
||||
self.assertEqual(errors.getvalue(), "read the places\n")
|
||||
|
||||
def test_refuses_a_missing_image_or_test_command(self):
|
||||
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
|
||||
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
|
||||
@@ -292,8 +297,8 @@ class MainTest(unittest.TestCase):
|
||||
|
||||
def test_prints_its_help(self):
|
||||
with contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
self.assertEqual(main.main(["--help"], {}), 0)
|
||||
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
|
||||
self.assertEqual((main.main(["--help"], {}), main.main(["-h"], {})), (0, 0))
|
||||
self.assertEqual(out.getvalue().count("usage: task-bench --image <image> --test <shell text>"), 2)
|
||||
|
||||
|
||||
def test_records_a_run_that_did_not_finish_without_figures(self):
|
||||
@@ -483,7 +488,7 @@ class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
|
||||
|
||||
async def test_answers_403_to_any_other_target_or_method(self):
|
||||
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"):
|
||||
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), f"GET {self.allowed} HTTP/1.1\r\n\r\n".encode(), f"CONNECT {self.allowed}\r\n\r\n".encode(), b"junk\r\n\r\n"):
|
||||
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user