"""Converts markdown to ADF (Atlassian Document Format, the rich text of Jira Cloud and Confluence) and back. It runs @larvit/adf-codec, a Node package, in a throwaway container that holds node_shim/shim.mjs; the shim's header says what its exit status means. adf-codec has two pairs of calls, and each function here names the one it runs: - The portable pair, with no suffix, which the loop's tools use. It drops each block's localId and the empty paragraphs Jira uses for spacing, so saving a document read by it changes the document; a tool that saves one unedited sends nothing instead (`docs/decisions.md` → `project config: the loop shows a Jira description and a Confluence page as adf-codec's portable markdown, …`). - The lossless pair, `*_lossless`, which keeps both, so a document read and saved unedited comes back identical. Its markdown wraps every block in `!adf:` marker lines, so no tool reads by it yet. """ import contextlib import dataclasses import fcntl import hashlib import json import os import pathlib import subprocess import sys import time import typing from failure import Failure from machine_load.slots import HELD, Lock, slot_count, slot_folder, usable_cpus, waited_slot SHIM = pathlib.Path(__file__).resolve().parent / "node_shim" SHIM_FILES = ("Dockerfile", "package.json", "package-lock.json", "shim.mjs") # `--memory` and `--memory-swap` both take this, so the container can't swap: a document too big for it ends in a Failure. MEMORY = "256m" RUN_SECONDS = 60 BUILD_SECONDS = 300 REFUSED = 2 @dataclasses.dataclass(frozen=True) class Ran: status: int out: str err: str # Runs the shim in the given mode (`to-adf`, `to-markdown`, `to-adf-lossless` or `to-markdown-lossless`) on a text, or fails to run it. Run = typing.Callable[[str, str], Ran | Failure] # Runs a docker command line on a text, in at most the given seconds, or fails to run it. Docker = typing.Callable[[list[str], str, int], Ran | Failure] # Holds one of the machine's heavy slots for a with block, or yields why none could be taken. Slot = typing.Callable[[], typing.ContextManager[Failure | None]] @dataclasses.dataclass(frozen=True) class Refused: """Why adf-codec refused one of the documents handed to adfs_to_markdown, and that document's index among them.""" index: int failure: Failure def image_tag(shim: pathlib.Path) -> str: """A tag that changes with every file the image is built from, so an edited shim or a bumped adf-codec builds a new image.""" digest = hashlib.sha256() for name in SHIM_FILES: digest.update((shim / name).read_bytes()) return f"claude-config-adf-codec:{digest.hexdigest()[:12]}" def docker_args(image: str, mode: str) -> list[str]: return ["docker", "run", "--rm", "-i", "--memory", MEMORY, "--memory-swap", MEMORY, "--network", "none", "--read-only", image, mode] def docker(args: list[str], text: str, seconds: int) -> Ran | Failure: """Runs a docker command line as a process of this one: the shim's shell, as git_repo.git is git's.""" return docker_ran(args, text, seconds, subprocess.run) def docker_ran(args: list[str], text: str, seconds: int, call: typing.Callable[..., typing.Any]) -> Ran | Failure: """What call, which runs a command line as subprocess.run does, made of a docker command line on text.""" try: done = call(args, input=text, capture_output=True, text=True, encoding="utf-8", timeout=seconds) except FileNotFoundError: return Failure("docker is not installed, and the adf-codec shim runs in a container") except subprocess.TimeoutExpired: return Failure(f"`{' '.join(args[:2])}` took more than {seconds} seconds") return Ran(done.returncode, done.stdout, done.stderr) @contextlib.contextmanager def machine_slot() -> typing.Iterator[Failure | None]: """heavy_slot on this process's environment, home, processors, stderr and clock: the shim's shell, as docker is.""" with heavy_slot(os.environ, pathlib.Path.home(), slot_count(usable_cpus()), lambda text: print(text, file=sys.stderr, flush=True), time.sleep, fcntl.flock) as refused: yield refused @contextlib.contextmanager def heavy_slot(env: typing.Mapping[str, str], home: pathlib.Path, slots: int, say: typing.Callable[[str], None], sleep: typing.Callable[[float], None], lock: Lock) -> typing.Iterator[Failure | None]: """Holds one of the machine's heavy slots, which run-heavy shares, for the with block, telling say where it waits for one. Under run-heavy the process already holds one, so it doesn't wait.""" if HELD in env: yield None return folder = slot_folder(env, home) try: folder.mkdir(parents=True, exist_ok=True) _, fd = waited_slot(folder, slots, lambda: say(f"adf-codec: waiting for a heavy slot, of {slots} on this machine, in {folder}, to build the shim's image; `fuser {folder}/slot-*` names the processes that hold them"), sleep, lock) except OSError as failure: yield Failure(f"cannot take a heavy slot in {folder}: {failure.strerror}") return try: yield None finally: os.close(fd) def ensure_image(shim: pathlib.Path, run_docker: Docker = docker, slot: Slot = machine_slot) -> str | Failure: """The tag of the image for the shim folder, built from that folder first where this machine has no image under that tag. The build waits for a heavy slot. Once this process holds the slot, it inspects again, as another process may have built the image meanwhile.""" tag = image_tag(shim) found = image_found(tag, run_docker) if found is not False: return tag if found is True else found with slot() as refused: if refused is not None: return refused found = image_found(tag, run_docker) if found is not False: return tag if found is True else found return build_image(shim, tag, run_docker) def image_found(tag: str, run_docker: Docker) -> bool | Failure: found = run_docker(["docker", "image", "inspect", tag], "", RUN_SECONDS) return found if isinstance(found, Failure) else found.status == 0 def build_image(shim: pathlib.Path, tag: str, run_docker: Docker) -> str | Failure: ran = run_docker(["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(shim)], "", BUILD_SECONDS) if isinstance(ran, Failure): return ran if ran.status != 0: return Failure(f"building the adf-codec shim's image failed with exit {ran.status}: {ran.err.strip()}") return tag def container_run(mode: str, text: str, run_docker: Docker = docker, shim: pathlib.Path = SHIM) -> Ran | Failure: image = ensure_image(shim, run_docker) if isinstance(image, Failure): return image return run_docker(docker_args(image, mode), text, RUN_SECONDS) def answer(mode: str, text: str, run: Run) -> str | Refused | Failure: """What the shim printed for text, or why it gave no answer: a Refused where it named the refused document's index.""" ran = run(mode, text) if isinstance(ran, Failure): return ran if ran.status == 0: return ran.out if ran.status == REFUSED: return refusal(ran.out) return Failure(f"the adf-codec shim failed with exit {ran.status}: {ran.err.strip()}") def refusal(out: str) -> Refused | Failure: try: refused = json.loads(out) error = refused["error"] where = f"line {error['position']['line']}" if "position" in error else ".".join(str(step) for step in error["path"]) or "the document" failure = Failure(f"adf-codec refused the input with {error['code']}: {error['message']} (at {where})") return Refused(refused["index"], failure) if isinstance(refused.get("index"), int) else failure except (ValueError, KeyError, TypeError, AttributeError): return Failure(f"the adf-codec shim refused with output that is not an error: {out.strip()}") def to_adf(mode: str, markdown: str, run: Run) -> dict[str, typing.Any] | Failure: out = answer(mode, markdown, run) if isinstance(out, Refused): return out.failure if isinstance(out, Failure): return out try: document = json.loads(out) except ValueError: return Failure(f"the adf-codec shim printed output that is not JSON: {out.strip()}") return document def to_markdowns(mode: str, documents: list[dict[str, typing.Any]], run: Run) -> list[str] | Refused | Failure: if not documents: return [] out = answer(mode, json.dumps(documents), run) if isinstance(out, Refused) and not 0 <= out.index < len(documents): return Failure(f"the adf-codec shim refused a document at index {out.index}, past the {len(documents)} it was handed") if isinstance(out, Refused | Failure): return out try: texts = json.loads(out) except ValueError: texts = None if not isinstance(texts, list) or len(texts) != len(documents) or not all(isinstance(text, str) for text in texts): return Failure(f"the adf-codec shim printed output that is not a JSON array of {len(documents)} texts: {out.strip()}") return texts def one_markdown(texts: list[str] | Refused | Failure) -> str | Failure: if isinstance(texts, Refused): return texts.failure return texts if isinstance(texts, Failure) else texts[0] def markdown_to_adf(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure: """The ADF document for the markdown, by the portable pair.""" return to_adf("to-adf", markdown, run) def markdown_to_adf_lossless(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure: """The ADF document for the markdown, by the lossless pair.""" return to_adf("to-adf-lossless", markdown, run) def adfs_to_markdown(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure: """The markdown for each ADF document, in their order, by the portable pair, from one container run, so a caller with many documents pays one container start.""" return to_markdowns("to-markdown", documents, run) def adfs_to_markdown_lossless(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure: """The markdown for each ADF document, in their order, by the lossless pair, from one container run.""" return to_markdowns("to-markdown-lossless", documents, run) def adf_to_markdown(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure: """The markdown for the ADF document, by the portable pair.""" return one_markdown(adfs_to_markdown([document], run)) def adf_to_markdown_lossless(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure: """The markdown for the ADF document, by the lossless pair.""" return one_markdown(adfs_to_markdown_lossless([document], run))