import contextlib import fcntl import json import os import subprocess import pytest from adf_codec import convert from failure import Failure from machine_load import slots DOC = {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Hi"}]}]} def fake(status: int, out: str = "", err: str = ""): calls: list[tuple[str, str]] = [] def run(mode: str, text: str) -> convert.Ran: calls.append((mode, text)) return convert.Ran(status, out, err) run.calls = calls # type: ignore[attr-defined] return run def test_markdown_to_adf_returns_the_document_the_shim_printed(): run = fake(0, json.dumps(DOC)) assert convert.markdown_to_adf("Hi\n", run) == DOC assert run.calls == [("to-adf", "Hi\n")] def test_adfs_to_markdown_sends_every_document_in_one_run_and_returns_the_texts_in_order(): run = fake(0, json.dumps(["Hi\n", "Yo\n"])) assert convert.adfs_to_markdown([DOC, DOC], run) == ["Hi\n", "Yo\n"] assert run.calls == [("to-markdown", json.dumps([DOC, DOC]))] def test_adfs_to_markdown_of_no_documents_runs_no_container(): run = fake(1) assert convert.adfs_to_markdown([], run) == [] assert run.calls == [] def test_a_refused_document_names_its_index(): refusal = {"index": 1, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}} refused = convert.adfs_to_markdown([DOC, DOC], fake(2, json.dumps(refusal))) assert refused == convert.Refused(1, Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)")) def test_a_refusal_whose_index_is_past_the_documents_becomes_a_failure(): refusal = {"index": 2, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}} failure = convert.adfs_to_markdown([DOC, DOC], fake(2, json.dumps(refusal))) assert failure == Failure("the adf-codec shim refused a document at index 2, past the 2 it was handed") @pytest.mark.parametrize("out", ["not json", json.dumps(["Hi\n"]), json.dumps({"0": "Hi\n"}), json.dumps(["Hi\n", 7])]) def test_texts_that_do_not_match_the_documents_become_a_failure(out): failure = convert.adfs_to_markdown([DOC, DOC], fake(0, out)) assert failure == Failure(f"the adf-codec shim printed output that is not a JSON array of 2 texts: {out}") def test_adf_to_markdown_returns_the_text_of_its_one_document(): run = fake(0, json.dumps(["Hi\n"])) assert convert.adf_to_markdown(DOC, run) == "Hi\n" assert run.calls == [("to-markdown", json.dumps([DOC]))] def test_markdown_to_adf_lossless_runs_the_lossless_mode(): run = fake(0, json.dumps(DOC)) assert convert.markdown_to_adf_lossless("Hi\n", run) == DOC assert run.calls == [("to-adf-lossless", "Hi\n")] def test_adfs_to_markdown_lossless_sends_every_document_in_one_lossless_run(): run = fake(0, json.dumps(["a\n", "b\n"])) assert convert.adfs_to_markdown_lossless([DOC, DOC], run) == ["a\n", "b\n"] assert run.calls == [("to-markdown-lossless", json.dumps([DOC, DOC]))] def test_adf_to_markdown_lossless_returns_the_text_of_its_one_document(): run = fake(0, json.dumps(["a\n"])) assert convert.adf_to_markdown_lossless(DOC, run) == "a\n" assert run.calls == [("to-markdown-lossless", json.dumps([DOC]))] def test_adfs_to_markdown_lossless_names_a_refused_documents_index(): refusal = {"index": 1, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}} refused = convert.adfs_to_markdown_lossless([DOC, DOC], fake(2, json.dumps(refusal))) assert refused == convert.Refused(1, Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)")) def test_adfs_to_markdown_lossless_of_no_documents_runs_no_container(): run = fake(1) assert convert.adfs_to_markdown_lossless([], run) == [] assert run.calls == [] def test_a_refusal_becomes_a_failure_with_adf_codecs_code_and_message(): refusal = {"error": {"code": "malformed-pipe-table", "message": "row has too many cells", "path": [], "position": {"line": 3, "offset": 9}}} failure = convert.markdown_to_adf("| a |\n", fake(2, json.dumps(refusal))) assert failure == Failure("adf-codec refused the input with malformed-pipe-table: row has too many cells (at line 3)") def test_a_refusal_without_a_position_names_the_path(): refusal = {"index": 0, "error": {"code": "unsupported-node-shape", "message": "bad node", "path": ["content", 0]}} failure = convert.adf_to_markdown(DOC, fake(2, json.dumps(refusal))) assert failure == Failure("adf-codec refused the input with unsupported-node-shape: bad node (at content.0)") def test_a_shim_that_could_not_run_becomes_a_failure_with_its_stderr(): failure = convert.adf_to_markdown(DOC, fake(1, "", "Unexpected token")) assert failure == Failure("the adf-codec shim failed with exit 1: Unexpected token") def test_a_shim_killed_for_memory_becomes_a_failure_with_its_exit_status(): failure = convert.markdown_to_adf("x", fake(137)) assert isinstance(failure, Failure) and "137" in failure.reason def test_output_that_is_not_json_becomes_a_failure(): failure = convert.markdown_to_adf("x", fake(0, "not json")) assert failure == Failure("the adf-codec shim printed output that is not JSON: not json") def test_a_refusal_that_is_not_the_expected_json_becomes_a_failure(): failure = convert.markdown_to_adf("x", fake(2, "oops")) assert failure == Failure("the adf-codec shim refused with output that is not an error: oops") def test_a_failure_from_the_runner_passes_through(): def run(mode: str, text: str) -> Failure: return Failure("docker is not installed") assert convert.markdown_to_adf("x", run) == Failure("docker is not installed") def test_the_image_tag_changes_with_the_shim_files(tmp_path): for name in convert.SHIM_FILES: (tmp_path / name).write_text("a") before = convert.image_tag(tmp_path) (tmp_path / "shim.mjs").write_text("b") assert convert.image_tag(tmp_path) != before def test_the_container_command_line_is_exactly_this(): assert convert.docker_args("image:1", "to-adf") == [ "docker", "run", "--rm", "-i", "--memory", "256m", "--memory-swap", "256m", "--network", "none", "--read-only", "image:1", "to-adf", ] def test_a_refusal_at_the_root_says_the_document(): refusal = {"index": 0, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}} failure = convert.adf_to_markdown(DOC, fake(2, json.dumps(refusal))) assert failure == Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)") def test_docker_passes_the_text_on_stdin_and_returns_the_output(): seen = {} def call(args, **kwargs): seen.update(kwargs, args=args) return subprocess.CompletedProcess(args, 3, "out", "err") assert convert.docker_ran(["docker", "x"], "text", 7, call) == convert.Ran(3, "out", "err") assert seen == {"args": ["docker", "x"], "input": "text", "capture_output": True, "text": True, "encoding": "utf-8", "timeout": 7} def test_docker_missing_becomes_a_failure(): def call(args, **kwargs): raise FileNotFoundError failure = convert.docker_ran(["docker", "x"], "", 7, call) assert failure == Failure("docker is not installed, and the adf-codec shim runs in a container") def test_docker_that_takes_too_long_becomes_a_failure(): def call(args, **kwargs): raise subprocess.TimeoutExpired(args, 7) assert convert.docker_ran(["docker", "build", "x"], "", 7, call) == Failure("`docker build` took more than 7 seconds") def shim_dir(tmp_path): for name in convert.SHIM_FILES: (tmp_path / name).write_text("a") return tmp_path def scripted(*answers): lines: list[tuple[list[str], str, int]] = [] queue = list(answers) def run_docker(args, text, seconds): lines.append((args, text, seconds)) return queue.pop(0) run_docker.lines = lines # type: ignore[attr-defined] return run_docker def test_an_image_that_exists_is_not_built(tmp_path): run_docker = scripted(convert.Ran(0, "", "")) tag = convert.image_tag(shim_dir(tmp_path)) assert convert.ensure_image(tmp_path, run_docker) == tag assert run_docker.lines == [(["docker", "image", "inspect", tag], "", convert.RUN_SECONDS)] class Slot: """Stands in for a heavy slot: records the docker lines run while it is held, or yields the given Failure.""" def __init__(self, run_docker, failure: Failure | None = None): self.run_docker, self.failure, self.held_lines = run_docker, failure, None @contextlib.contextmanager def __call__(self): before = len(self.run_docker.lines) yield self.failure self.held_lines = self.run_docker.lines[before:] def test_a_missing_image_is_built_from_the_shim_folder_with_a_memory_limit_inside_a_heavy_slot(tmp_path): missing = convert.Ran(1, "", "") run_docker = scripted(missing, missing, convert.Ran(0, "", "")) slot = Slot(run_docker) tag = convert.image_tag(shim_dir(tmp_path)) assert convert.ensure_image(tmp_path, run_docker, slot) == tag inspect = (["docker", "image", "inspect", tag], "", convert.RUN_SECONDS) build = (["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(tmp_path)], "", convert.BUILD_SECONDS) assert run_docker.lines == [inspect, inspect, build] assert slot.held_lines == [inspect, build] def test_an_image_another_process_built_while_this_one_waited_for_a_slot_is_not_built_again(tmp_path): run_docker = scripted(convert.Ran(1, "", ""), convert.Ran(0, "", "")) tag = convert.image_tag(shim_dir(tmp_path)) assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == tag assert len(run_docker.lines) == 2 def test_a_heavy_slot_that_cannot_be_taken_fails_before_the_build(tmp_path): run_docker = scripted(convert.Ran(1, "", "")) shim_dir(tmp_path) down = Failure("cannot take a heavy slot in /x: Permission denied") assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker, down)) == down assert len(run_docker.lines) == 1 def test_a_failed_build_becomes_a_failure_with_its_stderr(tmp_path): run_docker = scripted(convert.Ran(1, "", ""), convert.Ran(1, "", ""), convert.Ran(5, "", "no network\n")) shim_dir(tmp_path) assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == Failure("building the adf-codec shim's image failed with exit 5: no network") DOWN, MISSING = Failure("docker is not installed"), convert.Ran(1, "", "") @pytest.mark.parametrize("answers", [(DOWN,), (MISSING, DOWN), (MISSING, MISSING, DOWN)], ids=["first inspect", "second inspect", "build"]) def test_a_failure_to_start_docker_passes_through_inspect_and_build(tmp_path, answers): shim_dir(tmp_path) run_docker = scripted(*answers) assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == DOWN def not_said(text): raise AssertionError(f"said {text}") def not_slept(seconds): raise AssertionError(f"slept {seconds} seconds") def test_a_heavy_slot_is_held_until_the_block_ends(tmp_path): with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path)}, tmp_path, 1, not_said, not_slept, fcntl.flock) as held: assert held is None assert slots.taken_slot(tmp_path / "run-heavy", 1, fcntl.flock) is None slot, fd = slots.taken_slot(tmp_path / "run-heavy", 1, fcntl.flock) os.close(fd) def test_a_process_under_run_heavy_takes_no_second_slot(tmp_path): (tmp_path / "run-heavy").mkdir() slot, fd = slots.taken_slot(tmp_path / "run-heavy", 1, fcntl.flock) try: with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path), slots.HELD: "0"}, tmp_path, 1, not_said, not_slept, fcntl.flock) as held: assert held is None finally: os.close(fd) def test_a_heavy_slot_folder_that_cannot_be_made_yields_a_failure(tmp_path): (tmp_path / "run-heavy").write_text("a file where the folder goes") with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path)}, tmp_path, 1, not_said, not_slept, fcntl.flock) as held: assert isinstance(held, Failure) and held.reason.startswith(f"cannot take a heavy slot in {tmp_path / 'run-heavy'}: ") def test_container_run_runs_the_image_on_the_text(tmp_path): run_docker = scripted(convert.Ran(0, "", ""), convert.Ran(0, "answer", "")) assert convert.container_run("to-adf", "text", run_docker, shim_dir(tmp_path)) == convert.Ran(0, "answer", "") assert run_docker.lines[1] == (convert.docker_args(convert.image_tag(tmp_path), "to-adf"), "text", convert.RUN_SECONDS) def test_container_run_passes_an_image_failure_through(tmp_path): down = Failure("docker is not installed") assert convert.container_run("to-adf", "text", scripted(down), shim_dir(tmp_path)) == down def test_round_trip_through_the_real_container(): doc = convert.markdown_to_adf("# Title\n\nSome **bold** text.\n") assert isinstance(doc, dict), doc assert doc["type"] == "doc" and doc["content"][0]["type"] == "heading" markdown = convert.adf_to_markdown(doc) assert isinstance(markdown, str) and markdown.startswith("# Title") and "**bold**" in markdown refused = convert.adf_to_markdown({"type": "nope"}) assert isinstance(refused, Failure) and "not-an-adf-document" in refused.reason assert convert.adfs_to_markdown([doc, doc]) == [markdown, markdown] second = convert.adfs_to_markdown([doc, {"type": "nope"}]) assert isinstance(second, convert.Refused) and second.index == 1 and "not-an-adf-document" in second.failure.reason crashed = convert.answer("to-markdown", "not json", convert.container_run) assert isinstance(crashed, Failure) and crashed.reason.startswith("the adf-codec shim failed with exit 1: ") # A Jira description as Jira stores it: every block carries a localId, and an empty paragraph spaces two blocks. JIRA_DESCRIPTION = { "type": "doc", "version": 1, "content": [ {"type": "heading", "attrs": {"level": 2, "localId": "h-1"}, "content": [{"type": "text", "text": "Goal"}]}, {"type": "paragraph", "attrs": {"localId": "p-1"}, "content": [{"type": "text", "text": "Keep "}, {"type": "text", "text": "this", "marks": [{"type": "strong"}]}]}, {"type": "paragraph", "attrs": {"localId": "p-2"}}, {"type": "bulletList", "attrs": {"localId": "l-1"}, "content": [ {"type": "listItem", "attrs": {"localId": "i-1"}, "content": [{"type": "paragraph", "attrs": {"localId": "p-3"}, "content": [{"type": "text", "text": "one"}]}]}, ]}, ], } def test_the_lossless_pair_gives_back_a_jira_description_identical_and_the_portable_pair_does_not(): markdown = convert.adf_to_markdown_lossless(JIRA_DESCRIPTION) assert isinstance(markdown, str), markdown assert convert.markdown_to_adf_lossless(markdown) == JIRA_DESCRIPTION assert convert.adfs_to_markdown_lossless([JIRA_DESCRIPTION, JIRA_DESCRIPTION]) == [markdown, markdown] portable = convert.markdown_to_adf(convert.adf_to_markdown(JIRA_DESCRIPTION)) assert portable != JIRA_DESCRIPTION second = convert.adfs_to_markdown_lossless([JIRA_DESCRIPTION, {"type": "nope"}]) assert isinstance(second, convert.Refused) and second.index == 1