Emitter 2d: the opaque carry in both positions and the reserved info string

README.md

@@ -3,8 +3,8 @@
 Lossless conversion between **Atlassian Document Format** (ADF), an extended markdown flavour, and
 an HTML dialect.
 
-**Status: pre-release — `adfToMarkdown` emits every node the flavour spells but the opaque carry,
+**Status: pre-release — `adfToMarkdown` emits every node the flavour spells, nothing else is
-nothing else is built.**
+built.**
 Plan: `todo.md`. Decisions: `AGENTS.md`. The flavour's grammar:
 [`spec/flavour.md`](spec/flavour.md).
 

corpus/round-trip/opaque-carry/code-block-adf.json

@@ -0,0 +1,30 @@
+{
+  "content": [
+    {
+      "attrs": {
+        "language": "adf"
+      },
+      "content": [
+        {
+          "text": "{\n  \"type\": \"blockCard\"\n}",
+          "type": "text"
+        }
+      ],
+      "type": "codeBlock"
+    },
+    {
+      "attrs": {
+        "language": "adf"
+      },
+      "content": [
+        {
+          "text": "```",
+          "type": "text"
+        }
+      ],
+      "type": "codeBlock"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/round-trip/opaque-carry/code-block-adf.md

@@ -0,0 +1,29 @@
+```adf
+{
+  "attrs": {
+    "language": "adf"
+  },
+  "content": [
+    {
+      "text": "{\n  \"type\": \"blockCard\"\n}",
+      "type": "text"
+    }
+  ],
+  "type": "codeBlock"
+}
+```
+
+````adf
+{
+  "attrs": {
+    "language": "adf"
+  },
+  "content": [
+    {
+      "text": "```",
+      "type": "text"
+    }
+  ],
+  "type": "codeBlock"
+}
+````

corpus/round-trip/opaque-carry/unknown-block.json

@@ -0,0 +1,37 @@
+{
+  "content": [
+    {
+      "attrs": {
+        "url": "https://example.com/roadmap"
+      },
+      "type": "blockCard"
+    },
+    {
+      "attrs": {
+        "panelType": "info"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "The card below has no spelling yet.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "attrs": {
+            "layout": "wide",
+            "url": "https://example.com/board",
+            "width": 100
+          },
+          "type": "embedCard"
+        }
+      ],
+      "type": "panel"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/round-trip/opaque-carry/unknown-block.md

@@ -0,0 +1,23 @@
+```adf
+{
+  "attrs": {
+    "url": "https://example.com/roadmap"
+  },
+  "type": "blockCard"
+}
+```
+
+:::panel info
+The card below has no spelling yet.
+
+```adf
+{
+  "attrs": {
+    "layout": "wide",
+    "url": "https://example.com/board",
+    "width": 100
+  },
+  "type": "embedCard"
+}
+```
+:::

corpus/round-trip/opaque-carry/unknown-inline.json

@@ -0,0 +1,59 @@
+{
+  "content": [
+    {
+      "content": [
+        {
+          "text": "Draft: ",
+          "type": "text"
+        },
+        {
+          "attrs": {
+            "text": "Add a summary"
+          },
+          "type": "placeholder"
+        },
+        {
+          "text": " — expected Friday.",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "content": [
+        {
+          "marks": [
+            {
+              "type": "strong"
+            }
+          ],
+          "text": "Blocked",
+          "type": "text"
+        },
+        {
+          "text": " on ",
+          "type": "text"
+        },
+        {
+          "attrs": {
+            "extensionKey": "jira-issue",
+            "extensionType": "com.atlassian.jira.macro"
+          },
+          "marks": [
+            {
+              "type": "strong"
+            }
+          ],
+          "type": "inlineExtension"
+        },
+        {
+          "text": " until the rebuild lands.",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/round-trip/opaque-carry/unknown-inline.md

@@ -0,0 +1,3 @@
+Draft: :adf{json="{\"attrs\":{\"text\":\"Add a summary\"},\"type\":\"placeholder\"}"} — expected Friday.
+
+**Blocked** on :adf{json="{\"attrs\":{\"extensionKey\":\"jira-issue\",\"extensionType\":\"com.atlassian.jira.macro\"},\"marks\":[{\"type\":\"strong\"}],\"type\":\"inlineExtension\"}"} until the rebuild lands.

src/adf-to-markdown.test.ts

@@ -29,7 +29,7 @@ test('names the node a refusal came from', () => {
   const unspellable: AdfNode = { attrs: { localId: 'a' }, type: 'paragraph' }
   const list: AdfNode = { content: [{ content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }, { content: [unspellable], type: 'listItem' }], type: 'bulletList' }
   assert.deepEqual(path(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }), list))), ['content', 1, 'content', 1, 'content', 0])
-  assert.deepEqual(path(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, { type: 'blockCard' })))), ['content', 0, 'content', 1])
+  assert.deepEqual(path(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, { type: 'text' })))), ['content', 0, 'content', 1])
   assert.deepEqual(path(adfToMarkdown({ type: 'doc', version: 2 })), [])
 })
 
@@ -60,7 +60,6 @@ test('refuses an ordered list whose markdown start is ambiguous', () => {
 
 test('refuses the code block info strings the fence cannot hold', () => {
   assert.equal(code(adfToMarkdown(document({ attrs: { language: '' }, type: 'codeBlock' }))), 'ambiguous-attribute-spelling')
-  assert.equal(code(adfToMarkdown(document({ attrs: { language: 'adf' }, type: 'codeBlock' }))), 'reserved-adf-language')
   assert.equal(code(adfToMarkdown(document({ attrs: { language: 'a`b' }, type: 'codeBlock' }))), 'unspellable-code-block-language')
   assert.equal(code(adfToMarkdown(document({ attrs: { language: ' sql' }, type: 'codeBlock' }))), 'unspellable-code-block-language')
 })
@@ -113,10 +112,33 @@ test('refuses two adjacent lists of the same kind', () => {
   assert.equal(code(adfToMarkdown(document(list, list))), 'unspellable-adjacent-lists')
 })
 
-test('refuses a node type the canonical form does not cover', () => {
+test('carries a node type no section spells', () => {
-  assert.equal(code(adfToMarkdown(document({ type: 'blockCard' }))), 'unsupported-node-type')
+  assert.equal(markdown(adfToMarkdown(document({ type: 'blockCard' }))), '```adf\n{\n  "type": "blockCard"\n}\n```\n')
-  assert.equal(code(adfToMarkdown(document({ type: 'toString' }))), 'unsupported-node-type')
+  assert.equal(markdown(adfToMarkdown(document({ type: 'toString' }))), '```adf\n{\n  "type": "toString"\n}\n```\n')
-  assert.equal(code(adfToMarkdown(document(paragraph({ type: 'blockCard' })))), 'unsupported-node-type')
+  assert.equal(markdown(adfToMarkdown(document(paragraph({ type: 'blockCard' })))), ':adf{json="{\\"type\\":\\"blockCard\\"}"}\n')
+})
+
+test('carries the code block whose language is the reserved info string', () => {
+  assert.equal(
+    markdown(adfToMarkdown(document({ attrs: { language: 'adf' }, type: 'codeBlock' }))),
+    '```adf\n{\n  "attrs": {\n    "language": "adf"\n  },\n  "type": "codeBlock"\n}\n```\n',
+  )
+})
+
+test('breaks a mark run at the node it carries', () => {
+  const strong: AdfMark = { type: 'strong' }
+  const carried: AdfNode = { marks: [strong], type: 'placeholder' }
+  assert.equal(
+    markdown(adfToMarkdown(document(paragraph({ marks: [strong], text: 'a', type: 'text' }, carried, { marks: [strong], text: 'b', type: 'text' })))),
+    '**a**:adf{json="{\\"marks\\":[{\\"type\\":\\"strong\\"}],\\"type\\":\\"placeholder\\"}"}**b**\n',
+  )
+})
+
+test('refuses a carried node nested deeper than the emitter carries', () => {
+  let node: AdfNode = { type: 'blockCard' }
+  for (let depth = 0; depth < 600; depth += 1) node = { content: [node], type: 'blockCard' }
+  assert.equal(code(adfToMarkdown(document(node))), 'unsupported-node-shape')
+  assert.equal(code(adfToMarkdown(document(paragraph(node)))), 'unsupported-node-shape')
 })
 
 test('refuses a node whose content model the canonical form cannot emit', () => {
@@ -373,7 +395,10 @@ test('spells a table as a pipe table only where every row and cell is plain', ()
   assert.ok(directive(adfToMarkdown(table(row(cell('tableHeader'))))))
   assert.ok(directive(adfToMarkdown(table(row(cell('tableHeader', text('a'), text('b')))))))
   assert.equal(code(adfToMarkdown(table(row(cell('tableHeader', { attrs: { localId: 'a' }, type: 'paragraph' }))))), 'unspelled-node-attribute')
-  assert.equal(code(adfToMarkdown(table(row(cell('tableHeader', { content: [{ type: 'blockCard' }], type: 'paragraph' }))))), 'unsupported-node-type')
+  assert.equal(
+    markdown(adfToMarkdown(table(row(cell('tableHeader', { content: [{ type: 'blockCard' }], type: 'paragraph' }))))),
+    '| :adf{json="{\\"type\\":\\"blockCard\\"}"} |\n| --- |\n',
+  )
   assert.equal(code(adfToMarkdown(table(row(cell('tableHeader', { content: [{ text: '\fa', type: 'text' }], type: 'paragraph' }))))), 'unspellable-whitespace')
   assert.ok(directive(adfToMarkdown(table(row(cell('tableHeader', { attrs: { level: 1 }, type: 'heading' }))))))
   assert.equal(markdown(adfToMarkdown(table(row(cell('tableHeader', { content: [{ text: ' a', type: 'text' }], type: 'paragraph' }))))), '| :text{text=" "}a |\n| --- |\n')
@@ -383,6 +408,7 @@ test('spells a table as a pipe table only where every row and cell is plain', ()
   const fallsBack = (node: AdfNode): boolean => directive(adfToMarkdown(table(row(cell('tableHeader', { content: [node], type: 'paragraph' })))))
   assert.ok(fallsBack({ marks: [{ type: 'code' }], text: 'a|b', type: 'text' }))
   assert.ok(fallsBack({ attrs: { style: 'a|b' }, type: 'status' }))
+  assert.ok(fallsBack({ attrs: { url: 'https://example.com/?a|b' }, type: 'blockCard' }))
   assert.equal(markdown(adfToMarkdown(marked({ attrs: { href: 'https://example.com/x' }, type: 'link' }))), '| [l](https://example.com/x) |\n| --- |\n')
 })
 

src/adf-to-markdown.ts

@@ -5,6 +5,7 @@ import { blockDirective, spellDirectiveHeader } from './block-directives.ts'
 import { tryImage } from './markdown-image.ts'
 import { emitInlineLine } from './markdown-inline.ts'
 import { tryPipeTable } from './markdown-pipe-table.ts'
+import { carriedBlock, carryName } from './opaque-carry.ts'
 import { failure, success, type ConvertErrorPath, type Result } from './result.ts'
 import { holdsNullCharacter, isThematicBreak } from './commonmark-grammar.ts'
 import { isAdfDocument } from './adf-document.ts'
@@ -80,7 +81,7 @@ function interruptsParagraph(node: AdfNode): boolean {
 function emitBlock(node: AdfNode, path: ConvertErrorPath, depth: number): Result<EmittedBlock> {
   if (node.type === 'blockquote') return commonMarkContainer(emitBlockquote(node, path, depth))
   if (node.type === 'bulletList' || node.type === 'orderedList') return commonMarkContainer(emitList(node, path, depth))
-  if (node.type === 'codeBlock') return commonMarkLine(emitCodeBlock(node, path))
+  if (node.type === 'codeBlock') return commonMarkLine(node.attrs?.['language'] === carryName ? carriedBlock(node, path) : emitCodeBlock(node, path))
   if (node.type === 'heading') return commonMarkLine(emitHeading(node, path))
   if (node.type === 'paragraph') return emitParagraph(node, path)
   if (node.type === 'rule') return commonMarkLine(emitRule(node, path))
@@ -93,7 +94,7 @@ function emitBlock(node: AdfNode, path: ConvertErrorPath, depth: number): Result
   if (node.type === 'hardBreak' || node.type === 'listItem' || node.type === 'text') {
     return failure('unsupported-node-shape', `a ${node.type} node cannot stand where a block belongs`, path)
   }
-  return failure('unsupported-node-type', `the canonical form spells no block node of type ${node.type}`, path)
+  return commonMarkLine(carriedBlock(node, path))
 }
 
 function commonMarkLine(text: Result<string>): Result<EmittedBlock> {
@@ -184,7 +185,6 @@ function spellCodeFenceInfo(language: JsonValue | undefined, path: ConvertErrorP
   if (language === '') {
     return failure('ambiguous-attribute-spelling', 'an empty codeBlock language and an absent one share one markdown spelling', path)
   }
-  if (language === 'adf') return failure('reserved-adf-language', 'the adf info string is reserved for the opaque carry', path)
   if (/[`\n\r]/.test(language) || language !== language.trim()) {
     return failure('unspellable-code-block-language', 'a fence info string holds no backtick and no edge whitespace', path)
   }

src/corpus.test.ts

@@ -13,7 +13,7 @@ const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus')
 const roundTripRoot = join(corpusRoot, 'round-trip')
 const unspellableRoot = join(corpusRoot, 'unspellable')
 
-const emittingDirectories = ['block-nodes', 'commonmark-subset', 'inline-nodes']
+const emittingDirectories = ['block-nodes', 'commonmark-subset', 'inline-nodes', 'opaque-carry']
 
 function directoryNames(root: string): string[] {
   return readdirSync(root, { withFileTypes: true })

src/markdown-inline.ts

@@ -4,6 +4,7 @@ import { assembleInlineLine, type InlineEscaping, type InlineSegment, type LineC
 import { inlineDirective, markDirective, spellInlineNodeAttributes, spellMarkAttributes } from './inline-directives.ts'
 import { largestNesting } from './nesting.ts'
 import { claimsLine, holdsControlCharacter, holdsEntityReference, holdsNullCharacter, isAutolink, isUnicodeWhitespace } from './commonmark-grammar.ts'
+import { carriedInline } from './opaque-carry.ts'
 import { failure, success, type ConvertErrorPath, type Result } from './result.ts'
 import { longestBacktickRun } from './backtick-runs.ts'
 import { serializeCanonicalJson } from './canonical-json.ts'
@@ -131,7 +132,7 @@ function inlineRuns(nodes: readonly AdfNode[], depth: number, firstIndex: number
   const runs: InlineRun[] = []
   for (const [offset, node] of nodes.entries()) {
     const index = firstIndex + offset
-    const mark = (node.marks ?? [])[depth]
+    const mark = carries(node) ? undefined : (node.marks ?? [])[depth]
     if (mark === undefined) {
       runs.push({ index, kind: 'plain', node })
       continue
@@ -147,15 +148,24 @@ function nodePath(context: InlineContext, index: number): ConvertErrorPath {
   return [...context.path, 'content', index]
 }
 
+// spec/flavour.md, Marks: a run breaks at every carried node, so no emitted carry sits inside a mark spelling.
+function carries(node: AdfNode): boolean {
+  return node.type !== 'hardBreak' && node.type !== 'text' && inlineDirective(node.type) === undefined
+}
+
 function emitLeaf(node: AdfNode, context: InlineContext, index: number): Result<InlineSegment[]> {
   const path = nodePath(context, index)
+  if (carries(node)) {
+    const carried = carriedInline(node, path)
+    if (!carried.ok) return carried
+    return success([syntax(carried.value)])
+  }
   const types = (node.marks ?? []).map((mark) => mark.type)
   if (new Set(types).size !== types.length) return failure('unsupported-node-shape', `a ${node.type} node carries one mark type twice`, path)
-  if (node.type === 'text') return emitText(node, context, path)
-  if (node.type === 'hardBreak') return emitHardBreak(node, context, path)
   const directive = inlineDirective(node.type)
-  if (directive === undefined) return failure('unsupported-node-type', `the canonical form spells no inline node of type ${node.type}`, path)
+  if (directive !== undefined) return emitInlineDirective(node, directive, path)
-  return emitInlineDirective(node, directive, path)
+  if (node.type === 'text') return emitText(node, context, path)
+  return emitHardBreak(node, context, path)
 }
 
 function emitHardBreak(node: AdfNode, context: InlineContext, path: ConvertErrorPath): Result<InlineSegment[]> {

src/opaque-carry.ts

@@ -0,0 +1,30 @@
+import type { AdfNode } from './adf-document.ts'
+import type { JsonSpelling } from './canonical-json.ts'
+import { failure, success, type ConvertErrorPath, type Result } from './result.ts'
+import { isJsonValue } from './json-value.ts'
+import { largestNesting } from './nesting.ts'
+import { longestBacktickRun } from './backtick-runs.ts'
+import { serializeCanonicalJson } from './canonical-json.ts'
+import { spellAttributes, spellStringAttribute } from './directive-attributes.ts'
+
+export const carryName = 'adf'
+
+export function carriedBlock(node: AdfNode, path: ConvertErrorPath): Result<string> {
+  const json = carriedJson(node, 'two-space', path)
+  if (!json.ok) return json
+  const fence = '`'.repeat(Math.max(3, longestBacktickRun(json.value) + 1))
+  return success(`${fence}${carryName}\n${json.value}\n${fence}`)
+}
+
+export function carriedInline(node: AdfNode, path: ConvertErrorPath): Result<string> {
+  const json = carriedJson(node, 'compact', path)
+  if (!json.ok) return json
+  return success(`:${carryName}${spellAttributes([['json', spellStringAttribute(json.value)]])}`)
+}
+
+function carriedJson(node: AdfNode, spelling: JsonSpelling, path: ConvertErrorPath): Result<string> {
+  if (!isJsonValue(node)) {
+    return failure('unsupported-node-shape', `a carried node nests deeper than the ${largestNesting} levels the emitter carries`, path)
+  }
+  return success(serializeCanonicalJson(node, spelling))
+}

src/result.ts

@@ -1,7 +1,6 @@
 export type ConvertErrorCode =
   | 'ambiguous-attribute-spelling'
   | 'not-an-adf-document'
-  | 'reserved-adf-language'
   | 'unspellable-adjacent-lists'
   | 'unspellable-character'
   | 'unspellable-code-block-language'
@@ -17,7 +16,6 @@ export type ConvertErrorCode =
   | 'unsupported-document-version'
   | 'unsupported-heading-level'
   | 'unsupported-node-shape'
-  | 'unsupported-node-type'
 
 export type ConvertErrorPath = readonly (number | string)[]
 

todo.md

@@ -97,6 +97,12 @@ detail is settled at its own milestone.
         error `spec/flavour.md` promises. And `escaping: 'attribute'` earns its keep at the
         `\u007c` rule or collapses into `none`: nothing the escaper does tells the two apart
         today, since a carried segment holds only spaces, tabs and newlines.
+        The carry's fallback triggers land here too: `spec/flavour.md` carries a node its section
+        cannot spell — an attrs key no section lists, a value that is not the section's type, an
+        arg slot holding no bare token, marks no nesting spells — where the emitter still refuses,
+        which leaves 1d's blocked CommonMark nodes as the only refusal. The `\u007c` rule covers
+        the inline carry's `json` value with them: the carry spells `escaping: 'none'` today so
+        that `tryPipeCell` refuses the pipe form for a carry whose JSON holds a `|`.
         The gate gains the collision property here: no two corpus documents may emit the same
         bytes — one spelling for two documents is a round-trip break no parser can undo, and it is
         provable without one. It also settles the emitter's one known approximation: delimiter