Track the adf_codec package, which the allowlist ignored
CI / tests (pull_request) Successful in 2m13s
CI / tests (pull_request) Successful in 2m13s
This commit is contained in:
@@ -23,6 +23,13 @@
|
||||
!agents/
|
||||
!agents/*.md
|
||||
!bin/
|
||||
!bin/adf_codec/
|
||||
!bin/adf_codec/*.py
|
||||
!bin/adf_codec/node_shim/
|
||||
!bin/adf_codec/node_shim/Dockerfile
|
||||
!bin/adf_codec/node_shim/package-lock.json
|
||||
!bin/adf_codec/node_shim/package.json
|
||||
!bin/adf_codec/node_shim/shim.mjs
|
||||
!bin/branch_chunks/
|
||||
!bin/branch_chunks/*.py
|
||||
!bin/chunk-pr
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Converts markdown to ADF (Atlassian Document Format, the rich text of Jira Cloud and Confluence) and back.
|
||||
|
||||
It runs @larvit/adf-codec's portable calls in a throwaway container, since adf-codec is a Node package:
|
||||
`portableMarkdownToAdf` to write ADF and `adfToPortableMarkdown` to read it. The container holds node_shim/shim.mjs,
|
||||
whose exit status says what happened: 0 answered, 2 adf-codec refused the input (its error code is in the Failure), 1 the shim itself failed.
|
||||
"""
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
from failure import Failure
|
||||
|
||||
SHIM = pathlib.Path(__file__).resolve().parent / "node_shim"
|
||||
SHIM_FILES = ("Dockerfile", "package.json", "package-lock.json", "shim.mjs")
|
||||
# Both limits are equal, so the container can't swap: a document too big for it ends in a Failure, not in a slow machine.
|
||||
MEMORY = "256m"
|
||||
RUN_SECONDS = 60
|
||||
BUILD_SECONDS = 300
|
||||
REFUSED = 2
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Ran:
|
||||
status: int
|
||||
out: str
|
||||
err: str
|
||||
|
||||
|
||||
# Runs the shim in the given mode (`to-adf` or `to-markdown`) on a text, or fails to start it.
|
||||
Run = typing.Callable[[str, str], Ran | Failure]
|
||||
|
||||
|
||||
def image_tag(shim: pathlib.Path) -> str:
|
||||
"""A tag that changes with every file the image is built from, so an edited shim or a bumped adf-codec rebuilds it."""
|
||||
digest = hashlib.sha256()
|
||||
for name in SHIM_FILES:
|
||||
digest.update((shim / name).read_bytes())
|
||||
return f"claude-config-adf-codec:{digest.hexdigest()[:12]}"
|
||||
|
||||
|
||||
def docker_args(image: str, mode: str) -> list[str]:
|
||||
return ["docker", "run", "--rm", "-i", "--memory", MEMORY, "--memory-swap", MEMORY, "--network", "none", "--read-only", image, mode]
|
||||
|
||||
|
||||
def docker(args: list[str], text: str, seconds: int) -> Ran | Failure:
|
||||
try:
|
||||
done = subprocess.run(args, input=text, capture_output=True, text=True, encoding="utf-8", timeout=seconds)
|
||||
except FileNotFoundError:
|
||||
return Failure("docker is not installed, and the adf-codec shim runs in a container")
|
||||
except subprocess.TimeoutExpired:
|
||||
return Failure(f"`{' '.join(args[:2])}` took more than {seconds} seconds")
|
||||
return Ran(done.returncode, done.stdout, done.stderr)
|
||||
|
||||
|
||||
def ensure_image(shim: pathlib.Path) -> str | Failure:
|
||||
"""The tag of the shim's image, built from node_shim/ first where this machine has no image under that tag."""
|
||||
tag = image_tag(shim)
|
||||
found = docker(["docker", "image", "inspect", tag], "", RUN_SECONDS)
|
||||
if isinstance(found, Failure) or found.status == 0:
|
||||
return found if isinstance(found, Failure) else tag
|
||||
built = docker(["docker", "build", "--quiet", "--memory", "1g", "--memory-swap", "1g", "--tag", tag, str(shim)], "", BUILD_SECONDS)
|
||||
if isinstance(built, Failure):
|
||||
return built
|
||||
if built.status != 0:
|
||||
return Failure(f"building the adf-codec shim's image failed with exit {built.status}: {built.err.strip()}")
|
||||
return tag
|
||||
|
||||
|
||||
def container_run(mode: str, text: str) -> Ran | Failure:
|
||||
image = ensure_image(SHIM)
|
||||
if isinstance(image, Failure):
|
||||
return image
|
||||
return docker(docker_args(image, mode), text, RUN_SECONDS)
|
||||
|
||||
|
||||
def answer(mode: str, text: str, run: Run) -> str | Failure:
|
||||
"""What the shim printed for text, or a Failure saying why it printed none."""
|
||||
ran = run(mode, text)
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
if ran.status == 0:
|
||||
return ran.out
|
||||
if ran.status == REFUSED:
|
||||
return refusal(ran.out)
|
||||
return Failure(f"the adf-codec shim failed with exit {ran.status}: {ran.err.strip()}")
|
||||
|
||||
|
||||
def refusal(out: str) -> Failure:
|
||||
try:
|
||||
error = json.loads(out)["error"]
|
||||
where = f"line {error['position']['line']}" if "position" in error else ".".join(str(step) for step in error["path"]) or "the document"
|
||||
return Failure(f"adf-codec refused the input with {error['code']}: {error['message']} (at {where})")
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return Failure(f"the adf-codec shim refused with output that is not an error: {out.strip()}")
|
||||
|
||||
|
||||
def markdown_to_adf(markdown: str, run: Run = container_run) -> dict[str, typing.Any] | Failure:
|
||||
"""The ADF document markdown reads as."""
|
||||
out = answer("to-adf", markdown, run)
|
||||
if isinstance(out, Failure):
|
||||
return out
|
||||
try:
|
||||
document = json.loads(out)
|
||||
except ValueError:
|
||||
return Failure(f"the adf-codec shim printed output that is not JSON: {out.strip()}")
|
||||
return document
|
||||
|
||||
|
||||
def adf_to_markdown(document: dict[str, typing.Any], run: Run = container_run) -> str | Failure:
|
||||
"""The markdown an ADF document reads as."""
|
||||
return answer("to-markdown", json.dumps(document), run)
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM node:24.19.0-alpine3.24
|
||||
WORKDIR /shim
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --ignore-scripts --omit=dev
|
||||
COPY shim.mjs ./
|
||||
USER node
|
||||
ENTRYPOINT ["node", "shim.mjs"]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "adf-codec-shim",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "adf-codec-shim",
|
||||
"dependencies": {
|
||||
"@larvit/adf-codec": "0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@larvit/adf-codec": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@larvit/adf-codec/-/adf-codec-0.3.1.tgz",
|
||||
"integrity": "sha512-hI15WrztaGU90nEc2+IeFj16LtqpTxb2hahdhfj/YvdFTwd2lzhdQN1MRG4/65i6FiEZR2lIlFxyIYqx6EXjvg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "adf-codec-shim",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@larvit/adf-codec": "0.3.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Converts between markdown and ADF with @larvit/adf-codec's portable calls. The mode is the one argument:
|
||||
// to-adf stdin is markdown, stdout is the ADF document as JSON
|
||||
// to-markdown stdin is an ADF document as JSON, stdout is markdown
|
||||
// Exit 0: stdout holds the answer. Exit 2: adf-codec refused the input, and stdout holds {"error": {code, message, path, position?}}.
|
||||
// Exit 1: the shim could not run, and stderr says why.
|
||||
import { adfToPortableMarkdown, portableMarkdownToAdf } from '@larvit/adf-codec'
|
||||
|
||||
const REFUSED = 2
|
||||
|
||||
async function readStdin() {
|
||||
const chunks = []
|
||||
for await (const chunk of process.stdin) chunks.push(chunk)
|
||||
return Buffer.concat(chunks).toString('utf8')
|
||||
}
|
||||
|
||||
function finish(result, render) {
|
||||
if (result.ok) {
|
||||
process.stdout.write(render(result.value))
|
||||
return 0
|
||||
}
|
||||
process.stdout.write(JSON.stringify({ error: result.error }))
|
||||
return REFUSED
|
||||
}
|
||||
|
||||
async function main(mode) {
|
||||
const input = await readStdin()
|
||||
if (mode === 'to-adf') return finish(portableMarkdownToAdf(input), (doc) => JSON.stringify(doc))
|
||||
if (mode === 'to-markdown') return finish(adfToPortableMarkdown(JSON.parse(input)), (markdown) => markdown)
|
||||
throw new Error(`mode must be to-adf or to-markdown, not ${mode}`)
|
||||
}
|
||||
|
||||
main(process.argv[2]).then(
|
||||
(code) => { process.exitCode = code },
|
||||
(failure) => {
|
||||
process.stderr.write(`${failure instanceof Error ? failure.message : failure}\n`)
|
||||
process.exitCode = 1
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from adf_codec import convert
|
||||
from failure import Failure
|
||||
|
||||
DOC = {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Hi"}]}]}
|
||||
|
||||
|
||||
def fake(status: int, out: str = "", err: str = ""):
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
def run(mode: str, text: str) -> convert.Ran:
|
||||
calls.append((mode, text))
|
||||
return convert.Ran(status, out, err)
|
||||
|
||||
run.calls = calls # type: ignore[attr-defined]
|
||||
return run
|
||||
|
||||
|
||||
def test_markdown_to_adf_returns_the_document_the_shim_printed():
|
||||
run = fake(0, json.dumps(DOC))
|
||||
assert convert.markdown_to_adf("Hi\n", run) == DOC
|
||||
assert run.calls == [("to-adf", "Hi\n")]
|
||||
|
||||
|
||||
def test_adf_to_markdown_sends_the_document_as_json_and_returns_the_text():
|
||||
run = fake(0, "Hi\n")
|
||||
assert convert.adf_to_markdown(DOC, run) == "Hi\n"
|
||||
assert run.calls == [("to-markdown", json.dumps(DOC))]
|
||||
|
||||
|
||||
def test_a_refusal_becomes_a_failure_with_adf_codecs_code_and_message():
|
||||
refusal = {"error": {"code": "malformed-pipe-table", "message": "row has too many cells", "path": [], "position": {"line": 3, "offset": 9}}}
|
||||
failure = convert.markdown_to_adf("| a |\n", fake(2, json.dumps(refusal)))
|
||||
assert failure == Failure("adf-codec refused the input with malformed-pipe-table: row has too many cells (at line 3)")
|
||||
|
||||
|
||||
def test_a_refusal_without_a_position_names_the_path():
|
||||
refusal = {"error": {"code": "unsupported-node-shape", "message": "bad node", "path": ["content", 0]}}
|
||||
failure = convert.adf_to_markdown(DOC, fake(2, json.dumps(refusal)))
|
||||
assert failure == Failure("adf-codec refused the input with unsupported-node-shape: bad node (at content.0)")
|
||||
|
||||
|
||||
def test_a_shim_that_could_not_run_becomes_a_failure_with_its_stderr():
|
||||
failure = convert.adf_to_markdown(DOC, fake(1, "", "Unexpected token"))
|
||||
assert failure == Failure("the adf-codec shim failed with exit 1: Unexpected token")
|
||||
|
||||
|
||||
def test_a_container_that_was_killed_for_memory_says_so():
|
||||
failure = convert.markdown_to_adf("x", fake(137))
|
||||
assert isinstance(failure, Failure) and "137" in failure.reason
|
||||
|
||||
|
||||
def test_output_that_is_not_json_becomes_a_failure():
|
||||
failure = convert.markdown_to_adf("x", fake(0, "not json"))
|
||||
assert failure == Failure("the adf-codec shim printed output that is not JSON: not json")
|
||||
|
||||
|
||||
def test_a_refusal_that_is_not_the_expected_json_becomes_a_failure():
|
||||
failure = convert.markdown_to_adf("x", fake(2, "oops"))
|
||||
assert failure == Failure("the adf-codec shim refused with output that is not an error: oops")
|
||||
|
||||
|
||||
def test_a_failure_from_the_runner_passes_through():
|
||||
def run(mode: str, text: str) -> Failure:
|
||||
return Failure("docker is not installed")
|
||||
|
||||
assert convert.markdown_to_adf("x", run) == Failure("docker is not installed")
|
||||
|
||||
|
||||
def test_the_container_has_a_memory_and_swap_limit_and_no_network():
|
||||
args = convert.docker_args("image:1", "to-adf")
|
||||
assert args[:3] == ["docker", "run", "--rm"]
|
||||
assert args[args.index("--memory") + 1] == args[args.index("--memory-swap") + 1]
|
||||
assert args[args.index("--network") + 1] == "none"
|
||||
assert args[-2:] == ["image:1", "to-adf"]
|
||||
|
||||
|
||||
def test_the_image_tag_changes_with_the_shim_files(tmp_path):
|
||||
for name in convert.SHIM_FILES:
|
||||
(tmp_path / name).write_text("a")
|
||||
before = convert.image_tag(tmp_path)
|
||||
(tmp_path / "shim.mjs").write_text("b")
|
||||
assert convert.image_tag(tmp_path) != before
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0, reason="needs a docker daemon")
|
||||
def test_round_trip_through_the_real_container():
|
||||
doc = convert.markdown_to_adf("# Title\n\nSome **bold** text.\n")
|
||||
assert isinstance(doc, dict) and doc["type"] == "doc" and doc["content"][0]["type"] == "heading"
|
||||
markdown = convert.adf_to_markdown(doc)
|
||||
assert isinstance(markdown, str) and markdown.startswith("# Title") and "**bold**" in markdown
|
||||
refused = convert.adf_to_markdown({"type": "nope"})
|
||||
assert isinstance(refused, Failure) and "not-an-adf-document" in refused.reason
|
||||
Reference in New Issue
Block a user