335 lines
14 KiB
Python
335 lines
14 KiB
Python
import contextlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from adf_codec import convert
|
|
from failure import Failure
|
|
from machine_load import slots
|
|
|
|
DOC = {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Hi"}]}]}
|
|
|
|
|
|
def fake(status: int, out: str = "", err: str = ""):
|
|
calls: list[tuple[str, str]] = []
|
|
|
|
def run(mode: str, text: str) -> convert.Ran:
|
|
calls.append((mode, text))
|
|
return convert.Ran(status, out, err)
|
|
|
|
run.calls = calls # type: ignore[attr-defined]
|
|
return run
|
|
|
|
|
|
def test_markdown_to_adf_returns_the_document_the_shim_printed():
|
|
run = fake(0, json.dumps(DOC))
|
|
assert convert.markdown_to_adf("Hi\n", run) == DOC
|
|
assert run.calls == [("to-adf", "Hi\n")]
|
|
|
|
|
|
def test_adfs_to_markdown_sends_every_document_in_one_run_and_returns_the_texts_in_order():
|
|
run = fake(0, json.dumps(["Hi\n", "Yo\n"]))
|
|
assert convert.adfs_to_markdown([DOC, DOC], run) == ["Hi\n", "Yo\n"]
|
|
assert run.calls == [("to-markdown", json.dumps([DOC, DOC]))]
|
|
|
|
|
|
def test_adfs_to_markdown_of_no_documents_runs_no_container():
|
|
run = fake(1)
|
|
assert convert.adfs_to_markdown([], run) == []
|
|
assert run.calls == []
|
|
|
|
|
|
def test_a_refused_document_names_its_index():
|
|
refusal = {"index": 1, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}}
|
|
refused = convert.adfs_to_markdown([DOC, DOC], fake(2, json.dumps(refusal)))
|
|
assert refused == convert.Refused(1, Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)"))
|
|
|
|
|
|
def test_a_refusal_whose_index_is_past_the_documents_becomes_a_failure():
|
|
refusal = {"index": 2, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}}
|
|
failure = convert.adfs_to_markdown([DOC, DOC], fake(2, json.dumps(refusal)))
|
|
assert failure == Failure("the adf-codec shim refused a document at index 2, past the 2 it was handed")
|
|
|
|
|
|
@pytest.mark.parametrize("out", ["not json", json.dumps(["Hi\n"]), json.dumps({"0": "Hi\n"}), json.dumps(["Hi\n", 7])])
|
|
def test_texts_that_do_not_match_the_documents_become_a_failure(out):
|
|
failure = convert.adfs_to_markdown([DOC, DOC], fake(0, out))
|
|
assert failure == Failure(f"the adf-codec shim printed output that is not a JSON array of 2 texts: {out}")
|
|
|
|
|
|
def test_adf_to_markdown_returns_the_text_of_its_one_document():
|
|
run = fake(0, json.dumps(["Hi\n"]))
|
|
assert convert.adf_to_markdown(DOC, run) == "Hi\n"
|
|
assert run.calls == [("to-markdown", json.dumps([DOC]))]
|
|
|
|
|
|
def test_markdown_to_adf_lossless_runs_the_lossless_mode():
|
|
run = fake(0, json.dumps(DOC))
|
|
assert convert.markdown_to_adf_lossless("Hi\n", run) == DOC
|
|
assert run.calls == [("to-adf-lossless", "Hi\n")]
|
|
|
|
|
|
def test_adfs_to_markdown_lossless_sends_every_document_in_one_lossless_run():
|
|
run = fake(0, json.dumps(["a\n", "b\n"]))
|
|
assert convert.adfs_to_markdown_lossless([DOC, DOC], run) == ["a\n", "b\n"]
|
|
assert run.calls == [("to-markdown-lossless", json.dumps([DOC, DOC]))]
|
|
|
|
|
|
def test_adf_to_markdown_lossless_returns_the_text_of_its_one_document():
|
|
run = fake(0, json.dumps(["a\n"]))
|
|
assert convert.adf_to_markdown_lossless(DOC, run) == "a\n"
|
|
assert run.calls == [("to-markdown-lossless", json.dumps([DOC]))]
|
|
|
|
|
|
def test_adfs_to_markdown_lossless_names_a_refused_documents_index():
|
|
refusal = {"index": 1, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}}
|
|
refused = convert.adfs_to_markdown_lossless([DOC, DOC], fake(2, json.dumps(refusal)))
|
|
assert refused == convert.Refused(1, Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)"))
|
|
|
|
|
|
def test_adfs_to_markdown_lossless_of_no_documents_runs_no_container():
|
|
run = fake(1)
|
|
assert convert.adfs_to_markdown_lossless([], run) == []
|
|
assert run.calls == []
|
|
|
|
|
|
def test_a_refusal_becomes_a_failure_with_adf_codecs_code_and_message():
|
|
refusal = {"error": {"code": "malformed-pipe-table", "message": "row has too many cells", "path": [], "position": {"line": 3, "offset": 9}}}
|
|
failure = convert.markdown_to_adf("| a |\n", fake(2, json.dumps(refusal)))
|
|
assert failure == Failure("adf-codec refused the input with malformed-pipe-table: row has too many cells (at line 3)")
|
|
|
|
|
|
def test_a_refusal_without_a_position_names_the_path():
|
|
refusal = {"index": 0, "error": {"code": "unsupported-node-shape", "message": "bad node", "path": ["content", 0]}}
|
|
failure = convert.adf_to_markdown(DOC, fake(2, json.dumps(refusal)))
|
|
assert failure == Failure("adf-codec refused the input with unsupported-node-shape: bad node (at content.0)")
|
|
|
|
|
|
def test_a_shim_that_could_not_run_becomes_a_failure_with_its_stderr():
|
|
failure = convert.adf_to_markdown(DOC, fake(1, "", "Unexpected token"))
|
|
assert failure == Failure("the adf-codec shim failed with exit 1: Unexpected token")
|
|
|
|
|
|
def test_a_shim_killed_for_memory_becomes_a_failure_with_its_exit_status():
|
|
failure = convert.markdown_to_adf("x", fake(137))
|
|
assert isinstance(failure, Failure) and "137" in failure.reason
|
|
|
|
|
|
def test_output_that_is_not_json_becomes_a_failure():
|
|
failure = convert.markdown_to_adf("x", fake(0, "not json"))
|
|
assert failure == Failure("the adf-codec shim printed output that is not JSON: not json")
|
|
|
|
|
|
def test_a_refusal_that_is_not_the_expected_json_becomes_a_failure():
|
|
failure = convert.markdown_to_adf("x", fake(2, "oops"))
|
|
assert failure == Failure("the adf-codec shim refused with output that is not an error: oops")
|
|
|
|
|
|
def test_a_failure_from_the_runner_passes_through():
|
|
def run(mode: str, text: str) -> Failure:
|
|
return Failure("docker is not installed")
|
|
|
|
assert convert.markdown_to_adf("x", run) == Failure("docker is not installed")
|
|
|
|
|
|
def test_the_image_tag_changes_with_the_shim_files(tmp_path):
|
|
for name in convert.SHIM_FILES:
|
|
(tmp_path / name).write_text("a")
|
|
before = convert.image_tag(tmp_path)
|
|
(tmp_path / "shim.mjs").write_text("b")
|
|
assert convert.image_tag(tmp_path) != before
|
|
|
|
|
|
def test_the_container_command_line_is_exactly_this():
|
|
assert convert.docker_args("image:1", "to-adf") == [
|
|
"docker", "run", "--rm", "-i", "--memory", "256m", "--memory-swap", "256m", "--network", "none", "--read-only", "image:1", "to-adf",
|
|
]
|
|
|
|
|
|
def test_a_refusal_at_the_root_says_the_document():
|
|
refusal = {"index": 0, "error": {"code": "not-an-adf-document", "message": "no type", "path": []}}
|
|
failure = convert.adf_to_markdown(DOC, fake(2, json.dumps(refusal)))
|
|
assert failure == Failure("adf-codec refused the input with not-an-adf-document: no type (at the document)")
|
|
|
|
|
|
def test_docker_passes_the_text_on_stdin_and_returns_the_output():
|
|
seen = {}
|
|
|
|
def call(args, **kwargs):
|
|
seen.update(kwargs, args=args)
|
|
return subprocess.CompletedProcess(args, 3, "out", "err")
|
|
|
|
assert convert.docker(["docker", "x"], "text", 7, call) == convert.Ran(3, "out", "err")
|
|
assert seen == {"args": ["docker", "x"], "input": "text", "capture_output": True, "text": True, "encoding": "utf-8", "timeout": 7}
|
|
|
|
|
|
def test_docker_missing_becomes_a_failure():
|
|
def call(args, **kwargs):
|
|
raise FileNotFoundError
|
|
|
|
failure = convert.docker(["docker", "x"], "", 7, call)
|
|
assert failure == Failure("docker is not installed, and the adf-codec shim runs in a container")
|
|
|
|
|
|
def test_docker_that_takes_too_long_becomes_a_failure():
|
|
def call(args, **kwargs):
|
|
raise subprocess.TimeoutExpired(args, 7)
|
|
|
|
assert convert.docker(["docker", "build", "x"], "", 7, call) == Failure("`docker build` took more than 7 seconds")
|
|
|
|
|
|
def shim_dir(tmp_path):
|
|
for name in convert.SHIM_FILES:
|
|
(tmp_path / name).write_text("a")
|
|
return tmp_path
|
|
|
|
|
|
def scripted(*answers):
|
|
lines: list[tuple[list[str], str, int]] = []
|
|
queue = list(answers)
|
|
|
|
def run_docker(args, text, seconds):
|
|
lines.append((args, text, seconds))
|
|
return queue.pop(0)
|
|
|
|
run_docker.lines = lines # type: ignore[attr-defined]
|
|
return run_docker
|
|
|
|
|
|
def test_an_image_that_exists_is_not_built(tmp_path):
|
|
run_docker = scripted(convert.Ran(0, "", ""))
|
|
tag = convert.image_tag(shim_dir(tmp_path))
|
|
assert convert.ensure_image(tmp_path, run_docker) == tag
|
|
assert run_docker.lines == [(["docker", "image", "inspect", tag], "", convert.RUN_SECONDS)]
|
|
|
|
|
|
class Slot:
|
|
"""Stands in for a heavy slot: records the docker lines run while it is held, or yields the given Failure."""
|
|
def __init__(self, run_docker, failure: Failure | None = None):
|
|
self.run_docker, self.failure, self.held_lines = run_docker, failure, None
|
|
|
|
@contextlib.contextmanager
|
|
def __call__(self):
|
|
before = len(self.run_docker.lines)
|
|
yield self.failure
|
|
self.held_lines = self.run_docker.lines[before:]
|
|
|
|
|
|
def test_a_missing_image_is_built_from_the_shim_folder_with_a_memory_limit_inside_a_heavy_slot(tmp_path):
|
|
missing = convert.Ran(1, "", "")
|
|
run_docker = scripted(missing, missing, convert.Ran(0, "", ""))
|
|
slot = Slot(run_docker)
|
|
tag = convert.image_tag(shim_dir(tmp_path))
|
|
assert convert.ensure_image(tmp_path, run_docker, slot) == tag
|
|
inspect = (["docker", "image", "inspect", tag], "", convert.RUN_SECONDS)
|
|
build = (["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(tmp_path)], "", convert.BUILD_SECONDS)
|
|
assert run_docker.lines == [inspect, inspect, build]
|
|
assert slot.held_lines == [inspect, build]
|
|
|
|
|
|
def test_an_image_another_process_built_while_this_one_waited_for_a_slot_is_not_built_again(tmp_path):
|
|
run_docker = scripted(convert.Ran(1, "", ""), convert.Ran(0, "", ""))
|
|
tag = convert.image_tag(shim_dir(tmp_path))
|
|
assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == tag
|
|
assert len(run_docker.lines) == 2
|
|
|
|
|
|
def test_a_heavy_slot_that_cannot_be_taken_fails_before_the_build(tmp_path):
|
|
run_docker = scripted(convert.Ran(1, "", ""))
|
|
shim_dir(tmp_path)
|
|
down = Failure("cannot take a heavy slot in /x: Permission denied")
|
|
assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker, down)) == down
|
|
assert len(run_docker.lines) == 1
|
|
|
|
|
|
def test_a_failed_build_becomes_a_failure_with_its_stderr(tmp_path):
|
|
run_docker = scripted(convert.Ran(1, "", ""), convert.Ran(1, "", ""), convert.Ran(5, "", "no network\n"))
|
|
shim_dir(tmp_path)
|
|
assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == Failure("building the adf-codec shim's image failed with exit 5: no network")
|
|
|
|
|
|
DOWN, MISSING = Failure("docker is not installed"), convert.Ran(1, "", "")
|
|
|
|
|
|
@pytest.mark.parametrize("answers", [(DOWN,), (MISSING, DOWN), (MISSING, MISSING, DOWN)], ids=["first inspect", "second inspect", "build"])
|
|
def test_a_failure_to_start_docker_passes_through_inspect_and_build(tmp_path, answers):
|
|
shim_dir(tmp_path)
|
|
run_docker = scripted(*answers)
|
|
assert convert.ensure_image(tmp_path, run_docker, Slot(run_docker)) == DOWN
|
|
|
|
|
|
def test_a_heavy_slot_is_held_until_the_block_ends(tmp_path):
|
|
with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path)}, 1) as held:
|
|
assert held is None
|
|
assert slots.taken_slot(tmp_path / "run-heavy", 1) is None
|
|
slot, fd = slots.taken_slot(tmp_path / "run-heavy", 1)
|
|
os.close(fd)
|
|
|
|
|
|
def test_a_process_under_run_heavy_takes_no_second_slot(tmp_path):
|
|
(tmp_path / "run-heavy").mkdir()
|
|
slot, fd = slots.taken_slot(tmp_path / "run-heavy", 1)
|
|
try:
|
|
with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path), slots.HELD: "0"}, 1) as held:
|
|
assert held is None
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def test_a_heavy_slot_folder_that_cannot_be_made_yields_a_failure(tmp_path):
|
|
(tmp_path / "run-heavy").write_text("a file where the folder goes")
|
|
with convert.heavy_slot({"XDG_STATE_HOME": str(tmp_path)}, 1) as held:
|
|
assert isinstance(held, Failure) and held.reason.startswith(f"cannot take a heavy slot in {tmp_path / 'run-heavy'}: ")
|
|
|
|
|
|
def test_container_run_runs_the_image_on_the_text(tmp_path):
|
|
run_docker = scripted(convert.Ran(0, "", ""), convert.Ran(0, "answer", ""))
|
|
assert convert.container_run("to-adf", "text", run_docker, shim_dir(tmp_path)) == convert.Ran(0, "answer", "")
|
|
assert run_docker.lines[1] == (convert.docker_args(convert.image_tag(tmp_path), "to-adf"), "text", convert.RUN_SECONDS)
|
|
|
|
|
|
def test_container_run_passes_an_image_failure_through(tmp_path):
|
|
down = Failure("docker is not installed")
|
|
assert convert.container_run("to-adf", "text", scripted(down), shim_dir(tmp_path)) == down
|
|
|
|
|
|
def test_round_trip_through_the_real_container():
|
|
doc = convert.markdown_to_adf("# Title\n\nSome **bold** text.\n")
|
|
assert isinstance(doc, dict), doc
|
|
assert doc["type"] == "doc" and doc["content"][0]["type"] == "heading"
|
|
markdown = convert.adf_to_markdown(doc)
|
|
assert isinstance(markdown, str) and markdown.startswith("# Title") and "**bold**" in markdown
|
|
refused = convert.adf_to_markdown({"type": "nope"})
|
|
assert isinstance(refused, Failure) and "not-an-adf-document" in refused.reason
|
|
assert convert.adfs_to_markdown([doc, doc]) == [markdown, markdown]
|
|
second = convert.adfs_to_markdown([doc, {"type": "nope"}])
|
|
assert isinstance(second, convert.Refused) and second.index == 1 and "not-an-adf-document" in second.failure.reason
|
|
crashed = convert.answer("to-markdown", "not json", convert.container_run)
|
|
assert isinstance(crashed, Failure) and crashed.reason.startswith("the adf-codec shim failed with exit 1: ")
|
|
|
|
|
|
# A Jira description as Jira stores it: every block carries a localId, and an empty paragraph spaces two blocks.
|
|
JIRA_DESCRIPTION = {
|
|
"type": "doc", "version": 1, "content": [
|
|
{"type": "heading", "attrs": {"level": 2, "localId": "h-1"}, "content": [{"type": "text", "text": "Goal"}]},
|
|
{"type": "paragraph", "attrs": {"localId": "p-1"}, "content": [{"type": "text", "text": "Keep "}, {"type": "text", "text": "this", "marks": [{"type": "strong"}]}]},
|
|
{"type": "paragraph", "attrs": {"localId": "p-2"}},
|
|
{"type": "bulletList", "attrs": {"localId": "l-1"}, "content": [
|
|
{"type": "listItem", "attrs": {"localId": "i-1"}, "content": [{"type": "paragraph", "attrs": {"localId": "p-3"}, "content": [{"type": "text", "text": "one"}]}]},
|
|
]},
|
|
],
|
|
}
|
|
|
|
|
|
def test_the_lossless_pair_gives_back_a_jira_description_identical_and_the_portable_pair_does_not():
|
|
markdown = convert.adf_to_markdown_lossless(JIRA_DESCRIPTION)
|
|
assert isinstance(markdown, str), markdown
|
|
assert convert.markdown_to_adf_lossless(markdown) == JIRA_DESCRIPTION
|
|
assert convert.adfs_to_markdown_lossless([JIRA_DESCRIPTION, JIRA_DESCRIPTION]) == [markdown, markdown]
|
|
portable = convert.markdown_to_adf(convert.adf_to_markdown(JIRA_DESCRIPTION))
|
|
assert portable != JIRA_DESCRIPTION
|
|
second = convert.adfs_to_markdown_lossless([JIRA_DESCRIPTION, {"type": "nope"}])
|
|
assert isinstance(second, convert.Refused) and second.index == 1
|