234 lines
10 KiB
Python
234 lines
10 KiB
Python
"""Converts markdown to ADF (Atlassian Document Format, the rich text of Jira Cloud and Confluence) and back.
|
|
|
|
It runs @larvit/adf-codec, a Node package, in a throwaway container that holds node_shim/shim.mjs; the shim's header says what its exit status means.
|
|
|
|
adf-codec has two pairs of calls, and each function here names the one it runs:
|
|
- The portable pair, with no suffix, which the loop's tools use. It drops each block's localId and the empty paragraphs Jira uses for spacing, so saving a document read by it changes the document; a tool that saves one unedited sends nothing instead (`docs/decisions.md` → `project config: the loop shows a Jira description and a Confluence page as adf-codec's portable markdown, …`).
|
|
- The lossless pair, `*_lossless`, which keeps both, so a document read and saved unedited comes back identical. Its markdown wraps every block in `!adf:` marker lines, so no tool reads by it yet.
|
|
"""
|
|
import contextlib
|
|
import dataclasses
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import typing
|
|
|
|
from failure import Failure
|
|
from machine_load.slots import HELD, Lock, slot_count, slot_folder, usable_cpus, waited_slot
|
|
|
|
SHIM = pathlib.Path(__file__).resolve().parent / "node_shim"
|
|
SHIM_FILES = ("Dockerfile", "package.json", "package-lock.json", "shim.mjs")
|
|
# `--memory` and `--memory-swap` both take this, so the container can't swap: a document too big for it ends in a Failure.
|
|
MEMORY = "256m"
|
|
RUN_SECONDS = 60
|
|
BUILD_SECONDS = 300
|
|
REFUSED = 2
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Ran:
|
|
status: int
|
|
out: str
|
|
err: str
|
|
|
|
|
|
# Runs the shim in the given mode (`to-adf`, `to-markdown`, `to-adf-lossless` or `to-markdown-lossless`) on a text, or fails to run it.
|
|
Run = typing.Callable[[str, str], Ran | Failure]
|
|
# Runs a docker command line on a text, in at most the given seconds, or fails to run it.
|
|
Docker = typing.Callable[[list[str], str, int], Ran | Failure]
|
|
# Holds one of the machine's heavy slots for a with block, or yields why none could be taken.
|
|
Slot = typing.Callable[[], typing.ContextManager[Failure | None]]
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Refused:
|
|
"""Why adf-codec refused one of the documents handed to adfs_to_markdown, and that document's index among them."""
|
|
index: int
|
|
failure: Failure
|
|
|
|
|
|
def image_tag(shim: pathlib.Path) -> str:
|
|
"""A tag that changes with every file the image is built from, so an edited shim or a bumped adf-codec builds a new image."""
|
|
digest = hashlib.sha256()
|
|
for name in SHIM_FILES:
|
|
digest.update((shim / name).read_bytes())
|
|
return f"claude-config-adf-codec:{digest.hexdigest()[:12]}"
|
|
|
|
|
|
def docker_args(image: str, mode: str) -> list[str]:
|
|
return ["docker", "run", "--rm", "-i", "--memory", MEMORY, "--memory-swap", MEMORY, "--network", "none", "--read-only", image, mode]
|
|
|
|
|
|
def docker(args: list[str], text: str, seconds: int) -> Ran | Failure:
|
|
"""Runs a docker command line as a process of this one: the shim's shell, as git_repo.git is git's."""
|
|
return docker_ran(args, text, seconds, subprocess.run)
|
|
|
|
|
|
def docker_ran(args: list[str], text: str, seconds: int, call: typing.Callable[..., typing.Any]) -> Ran | Failure:
|
|
"""What call, which runs a command line as subprocess.run does, made of a docker command line on text."""
|
|
try:
|
|
done = call(args, input=text, capture_output=True, text=True, encoding="utf-8", timeout=seconds)
|
|
except FileNotFoundError:
|
|
return Failure("docker is not installed, and the adf-codec shim runs in a container")
|
|
except subprocess.TimeoutExpired:
|
|
return Failure(f"`{' '.join(args[:2])}` took more than {seconds} seconds")
|
|
return Ran(done.returncode, done.stdout, done.stderr)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def machine_slot() -> typing.Iterator[Failure | None]:
|
|
"""heavy_slot on this process's environment, home, processors, stderr and clock: the shim's shell, as docker is."""
|
|
with heavy_slot(os.environ, pathlib.Path.home(), slot_count(usable_cpus()), lambda text: print(text, file=sys.stderr, flush=True), time.sleep, fcntl.flock) as refused:
|
|
yield refused
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def heavy_slot(env: typing.Mapping[str, str], home: pathlib.Path, slots: int, say: typing.Callable[[str], None], sleep: typing.Callable[[float], None], lock: Lock) -> typing.Iterator[Failure | None]:
|
|
"""Holds one of the machine's heavy slots, which run-heavy shares, for the with block, telling say where it waits for one. Under run-heavy the process already holds one, so it doesn't wait."""
|
|
if HELD in env:
|
|
yield None
|
|
return
|
|
folder = slot_folder(env, home)
|
|
try:
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
_, fd = waited_slot(folder, slots, lambda: say(f"adf-codec: waiting for a heavy slot, of {slots} on this machine, in {folder}, to build the shim's image; `fuser {folder}/slot-*` names the processes that hold them"), sleep, lock)
|
|
except OSError as failure:
|
|
yield Failure(f"cannot take a heavy slot in {folder}: {failure.strerror}")
|
|
return
|
|
try:
|
|
yield None
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def ensure_image(shim: pathlib.Path, run_docker: Docker = docker, slot: Slot = machine_slot) -> str | Failure:
|
|
"""The tag of the image for the shim folder, built from that folder first where this machine has no image under that tag.
|
|
|
|
The build waits for a heavy slot. Once this process holds the slot, it inspects again, as another process may have built the image meanwhile."""
|
|
tag = image_tag(shim)
|
|
found = image_found(tag, run_docker)
|
|
if found is not False:
|
|
return tag if found is True else found
|
|
with slot() as refused:
|
|
if refused is not None:
|
|
return refused
|
|
found = image_found(tag, run_docker)
|
|
if found is not False:
|
|
return tag if found is True else found
|
|
return build_image(shim, tag, run_docker)
|
|
|
|
|
|
def image_found(tag: str, run_docker: Docker) -> bool | Failure:
|
|
found = run_docker(["docker", "image", "inspect", tag], "", RUN_SECONDS)
|
|
return found if isinstance(found, Failure) else found.status == 0
|
|
|
|
|
|
def build_image(shim: pathlib.Path, tag: str, run_docker: Docker) -> str | Failure:
|
|
ran = run_docker(["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(shim)], "", BUILD_SECONDS)
|
|
if isinstance(ran, Failure):
|
|
return ran
|
|
if ran.status != 0:
|
|
return Failure(f"building the adf-codec shim's image failed with exit {ran.status}: {ran.err.strip()}")
|
|
return tag
|
|
|
|
|
|
def container_run(mode: str, text: str, run_docker: Docker = docker, shim: pathlib.Path = SHIM) -> Ran | Failure:
|
|
image = ensure_image(shim, run_docker)
|
|
if isinstance(image, Failure):
|
|
return image
|
|
return run_docker(docker_args(image, mode), text, RUN_SECONDS)
|
|
|
|
|
|
def answer(mode: str, text: str, run: Run) -> str | Refused | Failure:
|
|
"""What the shim printed for text, or why it gave no answer: a Refused where it named the refused document's index."""
|
|
ran = run(mode, text)
|
|
if isinstance(ran, Failure):
|
|
return ran
|
|
if ran.status == 0:
|
|
return ran.out
|
|
if ran.status == REFUSED:
|
|
return refusal(ran.out)
|
|
return Failure(f"the adf-codec shim failed with exit {ran.status}: {ran.err.strip()}")
|
|
|
|
|
|
def refusal(out: str) -> Refused | Failure:
|
|
try:
|
|
refused = json.loads(out)
|
|
error = refused["error"]
|
|
where = f"line {error['position']['line']}" if "position" in error else ".".join(str(step) for step in error["path"]) or "the document"
|
|
failure = Failure(f"adf-codec refused the input with {error['code']}: {error['message']} (at {where})")
|
|
return Refused(refused["index"], failure) if isinstance(refused.get("index"), int) else failure
|
|
except (ValueError, KeyError, TypeError, AttributeError):
|
|
return Failure(f"the adf-codec shim refused with output that is not an error: {out.strip()}")
|
|
|
|
|
|
def to_adf(mode: str, markdown: str, run: Run) -> dict[str, typing.Any] | Failure:
|
|
out = answer(mode, markdown, run)
|
|
if isinstance(out, Refused):
|
|
return out.failure
|
|
if isinstance(out, Failure):
|
|
return out
|
|
try:
|
|
document = json.loads(out)
|
|
except ValueError:
|
|
return Failure(f"the adf-codec shim printed output that is not JSON: {out.strip()}")
|
|
return document
|
|
|
|
|
|
def to_markdowns(mode: str, documents: list[dict[str, typing.Any]], run: Run) -> list[str] | Refused | Failure:
|
|
if not documents:
|
|
return []
|
|
out = answer(mode, json.dumps(documents), run)
|
|
if isinstance(out, Refused) and not 0 <= out.index < len(documents):
|
|
return Failure(f"the adf-codec shim refused a document at index {out.index}, past the {len(documents)} it was handed")
|
|
if isinstance(out, Refused | Failure):
|
|
return out
|
|
try:
|
|
texts = json.loads(out)
|
|
except ValueError:
|
|
texts = None
|
|
if not isinstance(texts, list) or len(texts) != len(documents) or not all(isinstance(text, str) for text in texts):
|
|
return Failure(f"the adf-codec shim printed output that is not a JSON array of {len(documents)} texts: {out.strip()}")
|
|
return texts
|
|
|
|
|
|
def one_markdown(texts: list[str] | Refused | Failure) -> str | Failure:
|
|
if isinstance(texts, Refused):
|
|
return texts.failure
|
|
return texts if isinstance(texts, Failure) else texts[0]
|
|
|
|
|
|
def markdown_to_adf(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure:
|
|
"""The ADF document for the markdown, by the portable pair."""
|
|
return to_adf("to-adf", markdown, run)
|
|
|
|
|
|
def markdown_to_adf_lossless(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure:
|
|
"""The ADF document for the markdown, by the lossless pair."""
|
|
return to_adf("to-adf-lossless", markdown, run)
|
|
|
|
|
|
def adfs_to_markdown(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure:
|
|
"""The markdown for each ADF document, in their order, by the portable pair, from one container run, so a caller with many documents pays one container start."""
|
|
return to_markdowns("to-markdown", documents, run)
|
|
|
|
|
|
def adfs_to_markdown_lossless(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure:
|
|
"""The markdown for each ADF document, in their order, by the lossless pair, from one container run."""
|
|
return to_markdowns("to-markdown-lossless", documents, run)
|
|
|
|
|
|
def adf_to_markdown(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure:
|
|
"""The markdown for the ADF document, by the portable pair."""
|
|
return one_markdown(adfs_to_markdown([document], run))
|
|
|
|
|
|
def adf_to_markdown_lossless(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure:
|
|
"""The markdown for the ADF document, by the lossless pair."""
|
|
return one_markdown(adfs_to_markdown_lossless([document], run))
|