Files
claude-config/bin/adf_codec/convert.py
T

234 lines
10 KiB
Python

"""Converts markdown to ADF (Atlassian Document Format, the rich text of Jira Cloud and Confluence) and back.
It runs @larvit/adf-codec, a Node package, in a throwaway container that holds node_shim/shim.mjs; the shim's header says what its exit status means.
adf-codec has two pairs of calls, and each function here names the one it runs:
- The portable pair, with no suffix, which the loop's tools use. It drops each block's localId and the empty paragraphs Jira uses for spacing, so saving a document read by it changes the document; a tool that saves one unedited sends nothing instead (`docs/decisions.md` → `project config: the loop shows a Jira description and a Confluence page as adf-codec's portable markdown, …`).
- The lossless pair, `*_lossless`, which keeps both, so a document read and saved unedited comes back identical. Its markdown wraps every block in `!adf:` marker lines, so no tool reads by it yet.
"""
import contextlib
import dataclasses
import fcntl
import hashlib
import json
import os
import pathlib
import subprocess
import sys
import time
import typing
from failure import Failure
from machine_load.slots import HELD, Lock, slot_count, slot_folder, usable_cpus, waited_slot
SHIM = pathlib.Path(__file__).resolve().parent / "node_shim"
SHIM_FILES = ("Dockerfile", "package.json", "package-lock.json", "shim.mjs")
# `--memory` and `--memory-swap` both take this, so the container can't swap: a document too big for it ends in a Failure.
MEMORY = "256m"
RUN_SECONDS = 60
BUILD_SECONDS = 300
REFUSED = 2
@dataclasses.dataclass(frozen=True)
class Ran:
status: int
out: str
err: str
# Runs the shim in the given mode (`to-adf`, `to-markdown`, `to-adf-lossless` or `to-markdown-lossless`) on a text, or fails to run it.
Run = typing.Callable[[str, str], Ran | Failure]
# Runs a docker command line on a text, in at most the given seconds, or fails to run it.
Docker = typing.Callable[[list[str], str, int], Ran | Failure]
# Holds one of the machine's heavy slots for a with block, or yields why none could be taken.
Slot = typing.Callable[[], typing.ContextManager[Failure | None]]
@dataclasses.dataclass(frozen=True)
class Refused:
"""Why adf-codec refused one of the documents handed to adfs_to_markdown, and that document's index among them."""
index: int
failure: Failure
def image_tag(shim: pathlib.Path) -> str:
"""A tag that changes with every file the image is built from, so an edited shim or a bumped adf-codec builds a new image."""
digest = hashlib.sha256()
for name in SHIM_FILES:
digest.update((shim / name).read_bytes())
return f"claude-config-adf-codec:{digest.hexdigest()[:12]}"
def docker_args(image: str, mode: str) -> list[str]:
return ["docker", "run", "--rm", "-i", "--memory", MEMORY, "--memory-swap", MEMORY, "--network", "none", "--read-only", image, mode]
def docker(args: list[str], text: str, seconds: int) -> Ran | Failure:
"""Runs a docker command line as a process of this one: the shim's shell, as git_repo.git is git's."""
return docker_ran(args, text, seconds, subprocess.run)
def docker_ran(args: list[str], text: str, seconds: int, call: typing.Callable[..., typing.Any]) -> Ran | Failure:
"""What call, which runs a command line as subprocess.run does, made of a docker command line on text."""
try:
done = call(args, input=text, capture_output=True, text=True, encoding="utf-8", timeout=seconds)
except FileNotFoundError:
return Failure("docker is not installed, and the adf-codec shim runs in a container")
except subprocess.TimeoutExpired:
return Failure(f"`{' '.join(args[:2])}` took more than {seconds} seconds")
return Ran(done.returncode, done.stdout, done.stderr)
@contextlib.contextmanager
def machine_slot() -> typing.Iterator[Failure | None]:
"""heavy_slot on this process's environment, home, processors, stderr and clock: the shim's shell, as docker is."""
with heavy_slot(os.environ, pathlib.Path.home(), slot_count(usable_cpus()), lambda text: print(text, file=sys.stderr, flush=True), time.sleep, fcntl.flock) as refused:
yield refused
@contextlib.contextmanager
def heavy_slot(env: typing.Mapping[str, str], home: pathlib.Path, slots: int, say: typing.Callable[[str], None], sleep: typing.Callable[[float], None], lock: Lock) -> typing.Iterator[Failure | None]:
"""Holds one of the machine's heavy slots, which run-heavy shares, for the with block, telling say where it waits for one. Under run-heavy the process already holds one, so it doesn't wait."""
if HELD in env:
yield None
return
folder = slot_folder(env, home)
try:
folder.mkdir(parents=True, exist_ok=True)
_, fd = waited_slot(folder, slots, lambda: say(f"adf-codec: waiting for a heavy slot, of {slots} on this machine, in {folder}, to build the shim's image; `fuser {folder}/slot-*` names the processes that hold them"), sleep, lock)
except OSError as failure:
yield Failure(f"cannot take a heavy slot in {folder}: {failure.strerror}")
return
try:
yield None
finally:
os.close(fd)
def ensure_image(shim: pathlib.Path, run_docker: Docker = docker, slot: Slot = machine_slot) -> str | Failure:
"""The tag of the image for the shim folder, built from that folder first where this machine has no image under that tag.
The build waits for a heavy slot. Once this process holds the slot, it inspects again, as another process may have built the image meanwhile."""
tag = image_tag(shim)
found = image_found(tag, run_docker)
if found is not False:
return tag if found is True else found
with slot() as refused:
if refused is not None:
return refused
found = image_found(tag, run_docker)
if found is not False:
return tag if found is True else found
return build_image(shim, tag, run_docker)
def image_found(tag: str, run_docker: Docker) -> bool | Failure:
found = run_docker(["docker", "image", "inspect", tag], "", RUN_SECONDS)
return found if isinstance(found, Failure) else found.status == 0
def build_image(shim: pathlib.Path, tag: str, run_docker: Docker) -> str | Failure:
ran = run_docker(["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(shim)], "", BUILD_SECONDS)
if isinstance(ran, Failure):
return ran
if ran.status != 0:
return Failure(f"building the adf-codec shim's image failed with exit {ran.status}: {ran.err.strip()}")
return tag
def container_run(mode: str, text: str, run_docker: Docker = docker, shim: pathlib.Path = SHIM) -> Ran | Failure:
image = ensure_image(shim, run_docker)
if isinstance(image, Failure):
return image
return run_docker(docker_args(image, mode), text, RUN_SECONDS)
def answer(mode: str, text: str, run: Run) -> str | Refused | Failure:
"""What the shim printed for text, or why it gave no answer: a Refused where it named the refused document's index."""
ran = run(mode, text)
if isinstance(ran, Failure):
return ran
if ran.status == 0:
return ran.out
if ran.status == REFUSED:
return refusal(ran.out)
return Failure(f"the adf-codec shim failed with exit {ran.status}: {ran.err.strip()}")
def refusal(out: str) -> Refused | Failure:
try:
refused = json.loads(out)
error = refused["error"]
where = f"line {error['position']['line']}" if "position" in error else ".".join(str(step) for step in error["path"]) or "the document"
failure = Failure(f"adf-codec refused the input with {error['code']}: {error['message']} (at {where})")
return Refused(refused["index"], failure) if isinstance(refused.get("index"), int) else failure
except (ValueError, KeyError, TypeError, AttributeError):
return Failure(f"the adf-codec shim refused with output that is not an error: {out.strip()}")
def to_adf(mode: str, markdown: str, run: Run) -> dict[str, typing.Any] | Failure:
out = answer(mode, markdown, run)
if isinstance(out, Refused):
return out.failure
if isinstance(out, Failure):
return out
try:
document = json.loads(out)
except ValueError:
return Failure(f"the adf-codec shim printed output that is not JSON: {out.strip()}")
return document
def to_markdowns(mode: str, documents: list[dict[str, typing.Any]], run: Run) -> list[str] | Refused | Failure:
if not documents:
return []
out = answer(mode, json.dumps(documents), run)
if isinstance(out, Refused) and not 0 <= out.index < len(documents):
return Failure(f"the adf-codec shim refused a document at index {out.index}, past the {len(documents)} it was handed")
if isinstance(out, Refused | Failure):
return out
try:
texts = json.loads(out)
except ValueError:
texts = None
if not isinstance(texts, list) or len(texts) != len(documents) or not all(isinstance(text, str) for text in texts):
return Failure(f"the adf-codec shim printed output that is not a JSON array of {len(documents)} texts: {out.strip()}")
return texts
def one_markdown(texts: list[str] | Refused | Failure) -> str | Failure:
if isinstance(texts, Refused):
return texts.failure
return texts if isinstance(texts, Failure) else texts[0]
def markdown_to_adf(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure:
"""The ADF document for the markdown, by the portable pair."""
return to_adf("to-adf", markdown, run)
def markdown_to_adf_lossless(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure:
"""The ADF document for the markdown, by the lossless pair."""
return to_adf("to-adf-lossless", markdown, run)
def adfs_to_markdown(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure:
"""The markdown for each ADF document, in their order, by the portable pair, from one container run, so a caller with many documents pays one container start."""
return to_markdowns("to-markdown", documents, run)
def adfs_to_markdown_lossless(documents: list[dict[str, typing.Any]], run: Run = container_run) -> list[str] | Refused | Failure:
"""The markdown for each ADF document, in their order, by the lossless pair, from one container run."""
return to_markdowns("to-markdown-lossless", documents, run)
def adf_to_markdown(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure:
"""The markdown for the ADF document, by the portable pair."""
return one_markdown(adfs_to_markdown([document], run))
def adf_to_markdown_lossless(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure:
"""The markdown for the ADF document, by the lossless pair."""
return one_markdown(adfs_to_markdown_lossless([document], run))