Read the inline nodes and the marks back, and keep the whitespace CommonMark does not strip

src/markdown/emit/adf-to-markdown.test.ts

@@ -130,7 +130,6 @@ test('carries a mark the canonical spellings cannot nest', () => {
 })
 
 test('refuses whitespace CommonMark cannot hold', () => {
-  assert.equal(code(adfToMarkdown(document(paragraph({ text: '\fa', type: 'text' })))), 'unspellable-whitespace')
   assert.equal(code(adfToMarkdown(document(paragraph({ text: 'a\rb', type: 'text' })))), 'unspellable-whitespace')
 })
 
@@ -535,7 +534,7 @@ test('spells a table as a pipe table only where every row and cell is plain', ()
     markdown(adfToMarkdown(table(row(cell('tableHeader', { content: [{ attrs: { url: 'a|b' }, type: 'blockCard' }], type: 'paragraph' }))))),
     '| :adf{json="{\\"attrs\\":{\\"url\\":\\"a\\u007cb\\"},\\"type\\":\\"blockCard\\"}"} |\n| --- |\n',
   )
-  assert.equal(code(adfToMarkdown(table(row(cell('tableHeader', { content: [{ text: '\fa', type: 'text' }], type: 'paragraph' }))))), 'unspellable-whitespace')
+  assert.equal(markdown(adfToMarkdown(table(row(cell('tableHeader', { content: [{ text: '\fa', type: 'text' }], type: 'paragraph' }))))), '| \fa |\n| --- |\n')
   assert.ok(directive(adfToMarkdown(table(row(cell('tableHeader', { attrs: { level: 1 }, type: 'heading' }))))))
   assert.equal(markdown(adfToMarkdown(table(row(cell('tableHeader', { content: [{ text: ' a', type: 'text' }], type: 'paragraph' }))))), '| :text{text=" "}a |\n| --- |\n')
   const marked = (mark: AdfMark): AdfDocument => table(row(cell('tableHeader', { content: [{ marks: [mark], text: 'l', type: 'text' }], type: 'paragraph' })))
@@ -621,4 +620,6 @@ test('carries whitespace CommonMark strips in the reserved text directive', () =
   assert.equal(emitted({ text: 'a ', type: 'text' }, { type: 'hardBreak' }, { text: ' b', type: 'text' }), 'a:text{text=" "}\\\n:text{text=" "}b\n')
   assert.equal(emitted({ marks: [{ type: 'em' }], text: ' a ', type: 'text' }), '_:text{text=" "}a:text{text=" "}_\n')
   assert.equal(markdown(adfToMarkdown(document({ attrs: { level: 1 }, content: [{ text: 'x ', type: 'text' }], type: 'heading' }))), '# x:text{text=" "}\n')
+  // CommonMark strips spaces and tabs alone, so the whitespace beside them is plain text.
+  assert.equal(emitted({ text: '\va\f', type: 'text' }), '\va\f\n')
 })

src/markdown/emit/inline-line.ts

@@ -10,9 +10,9 @@ import { largestNesting } from '../../nesting.ts'
 import { longestBacktickRun } from '../backtick-runs.ts'
 import { markSpelling, spellMarkAttributes } from '../mark-spellings.ts'
 import { sameMark } from '../../adf/editor-normal.ts'
-import { spellAttributes, spellStringAttribute } from '../directive-syntax.ts'
 import { spellDestination, spellTitle } from '../link-syntax.ts'
 import { spellInlineNodeAttributes } from './inline-directive-spelling.ts'
+import { spellTextDirective } from '../text-directive.ts'
 
 type EmittedLine = { line: string; segments: InlineSegment[] }
 
@@ -84,9 +84,6 @@ function attemptLine(segments: readonly InlineSegment[], container: LineContaine
   const assembled = assembleInlineLine(segments, container)
   if (assembled.unspellableRun !== undefined) return success({ carry: assembled.unspellableRun })
   for (const [index, single] of assembled.line.split('\n').entries()) {
-    if (/^[ \t\v\f]|[ \t\v\f]$/.test(single)) {
-      return failure('unspellable-whitespace', 'a line begins or ends with whitespace CommonMark strips', path)
-    }
     if (container === 'paragraph' && claimsLine(single, index === 0 ? 'first' : 'later')) {
       return failure('unspellable-line-start', `block parsing would claim the emitted line ${JSON.stringify(single)}`, path)
     }
@@ -121,7 +118,7 @@ function carryEdges(segment: InlineSegment, leading: boolean, trailing: boolean)
 }
 
 function carriedText(text: string): InlineSegment {
-  return syntax(spellLeafDirective('text', spellAttributes([['text', spellStringAttribute(text)]])))
+  return syntax(spellTextDirective(text))
 }
 
 function spellLeafDirective(name: string, attributes: string): string {

src/markdown/parse/directive-marks.ts

@@ -0,0 +1,29 @@
+import type { AdfMark } from '../../adf/document.ts'
+import type { DirectiveAttributes } from '../directive-syntax.ts'
+import type { MarkSpelling } from '../mark-spellings.ts'
+import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
+import { markSpelling } from '../mark-spellings.ts'
+import { readVocabulary } from './directive-nodes.ts'
+
+export function readDirectiveMark(name: string, attributes: DirectiveAttributes, path: ConvertErrorPath): Result<AdfMark> | undefined {
+  const spelling = markSpelling(name)
+  if (spelling === undefined) return undefined
+  const markdown = markdownForm(spelling)
+  if (markdown !== undefined) return failure('unsupported-node-shape', `${name} is spelled ${markdown}, never as a directive`, path)
+  const attrs = readVocabulary(name, attributes, spelling.attributes, undefined, path)
+  if (!attrs.ok) return attrs
+  return success(Object.keys(attrs.value).length === 0 ? { type: name } : { attrs: attrs.value, type: name })
+}
+
+function markdownForm(spelling: MarkSpelling): string | undefined {
+  switch (spelling.kind) {
+    case 'code':
+      return '`x`'
+    case 'directive':
+      return undefined
+    case 'emphasis':
+      return `${spelling.spelling}x${spelling.spelling}`
+    case 'link':
+      return '[x](url)'
+  }
+}

src/markdown/parse/directive-nodes.ts

@@ -1,7 +1,7 @@
 import type { AdfAttributes, AdfMark, AdfNode } from '../../adf/document.ts'
 import type { AttributeVocabulary } from '../../adf/attribute-vocabulary.ts'
 import type { BlockDirective } from '../../adf/block-directives.ts'
-import type { DirectiveAttributes, DirectiveSpan, DirectiveValue } from '../directive-syntax.ts'
+import type { DirectiveAttributes, DirectiveValue } from '../directive-syntax.ts'
 import { attributeValue, spellAttributeValue, unknownDirectiveFault } from '../directive-syntax.ts'
 import { blockArgument } from '../block-directive-arguments.ts'
 import { blockDirective } from '../../adf/block-directives.ts'
@@ -41,21 +41,36 @@ export function readBlockDirectiveNode(
   return success({ contentModel: directive.contentModel, node: namedNode(name, attrs.value, marks.value) })
 }
 
-export function readInlineDirectiveNode(span: DirectiveSpan, path: ConvertErrorPath): Result<AdfNode> {
-  const directive = inlineDirective(span.name)
-  if (directive === undefined) return faulted(unknownDirectiveFault(span.name), path)
+export function readInlineDirectiveNode(
+  name: string,
+  attributes: DirectiveAttributes,
+  content: readonly AdfNode[] | undefined,
+  path: ConvertErrorPath,
+): Result<AdfNode> {
+  const directive = inlineDirective(name)
+  if (directive === undefined) return faulted(unknownDirectiveFault(name), path)
   const slot = directive.textAttribute
-  if (span.content !== undefined) {
-    const message = slot === undefined ? `${span.name} takes no content` : `the content slot ${span.name} spells its ${slot} attribute in is unsupported`
-    return failure('unsupported-node-shape', message, path)
-  }
+  if (slot === undefined && content !== undefined) return failure('unsupported-node-shape', `${name} takes no content`, path)
   const elsewhere: Elsewhere | undefined = slot === undefined ? undefined : { key: slot, slot: 'content' }
-  const attrs = readVocabulary(span.name, span.attributes, directive.attributes, elsewhere, path)
+  const attrs = readVocabulary(name, attributes, directive.attributes, elsewhere, path)
   if (!attrs.ok) return attrs
-  return success(namedNode(span.name, attrs.value, undefined))
+  if (slot !== undefined && content !== undefined) {
+    const text = slotText(content)
+    if (text === undefined) return failure('unsupported-node-shape', `the ${name} content slot holds one unmarked text node`, path)
+    attrs.value[slot] = text
+  }
+  return success(namedNode(name, attrs.value, undefined))
 }
 
-function readVocabulary(
+// spec/flavour.md, Inline nodes: the slot is plain text, its adjacent nodes already merged.
+function slotText(content: readonly AdfNode[]): string | undefined {
+  if (content.length === 0) return ''
+  const only = content.length === 1 ? content[0] : undefined
+  if (only?.type !== 'text' || (only.marks ?? []).length > 0 || typeof only.text !== 'string') return undefined
+  return only.text
+}
+
+export function readVocabulary(
   type: string,
   attributes: DirectiveAttributes,
   vocabulary: AttributeVocabulary,

src/markdown/parse/inline-content.ts

@@ -1,4 +1,5 @@
 import type { AdfMark, AdfNode } from '../../adf/document.ts'
+import type { DirectiveSpan } from '../directive-syntax.ts'
 import type { EmphasisPairing } from '../emphasis-matching.ts'
 import type { LinkDefinition } from '../link-syntax.ts'
 import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct, readBracketedAutolink, readEmailAutolink, trimTrailingSpace } from '../commonmark-grammar.ts'
@@ -7,8 +8,10 @@ import { delimiterFlags, matchEmphasis, runLength } from '../emphasis-matching.t
 import { failure, faulted, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { mergeAdjacentText } from '../../adf/editor-normal.ts'
 import { normalizeLabel, readInlineTarget, readLabel } from '../link-syntax.ts'
+import { readDirectiveMark } from './directive-marks.ts'
 import { readInlineDirective } from '../directive-syntax.ts'
 import { readInlineDirectiveNode } from './directive-nodes.ts'
+import { readTextDirective } from '../text-directive.ts'
 
 export type InlineContent = { image: AdfNode; nodes?: undefined } | { image?: undefined; nodes: AdfNode[] }
 
@@ -31,6 +34,10 @@ type Scan = { definitions: LinkDefinitions; path: ConvertErrorPath; pending: str
 const imageAlone = 'an image fits only as a paragraph of its own'
 
 export function parseInlineContent(source: string, definitions: LinkDefinitions, path: ConvertErrorPath): Result<InlineContent> {
+  return parseInline(source, definitions, path, true)
+}
+
+function parseInline(source: string, definitions: LinkDefinitions, path: ConvertErrorPath, strip: boolean): Result<InlineContent> {
   const scan: Scan = { definitions, path, pending: '', pieces: [], source }
   let index = 0
   while (index < source.length) {
@@ -76,7 +83,7 @@ export function parseInlineContent(source: string, definitions: LinkDefinitions,
         index += 1
     }
   }
-  flush(scan, true)
+  flush(scan, strip)
   return assemble(scan)
 }
 
@@ -144,13 +151,40 @@ function readDirective(scan: Scan, index: number): Result<number> {
     return success(index + 1)
   }
   if (directive.fault !== undefined) return faulted(directive.fault, scan.path)
-  const node = readInlineDirectiveNode(directive.value, scan.path)
-  if (!node.ok) return node
+  const nodes = directiveNodes(scan, directive.value)
+  if (!nodes.ok) return nodes
   flush(scan, false)
-  pushNode(scan, node.value)
+  scan.pieces.push({ kind: 'nodes', nodes: nodes.value })
   return success(index + directive.value.length)
 }
 
+function directiveNodes(scan: Scan, span: DirectiveSpan): Result<AdfNode[]> {
+  const text = readTextDirective(span)
+  if (text?.fault !== undefined) return faulted(text.fault, scan.path)
+  if (text !== undefined) return success([{ text: text.value, type: 'text' }])
+  const slot = slotNodes(scan, span.content)
+  if (!slot.ok) return slot
+  const mark = readDirectiveMark(span.name, span.attributes, scan.path)
+  if (mark !== undefined) {
+    if (!mark.ok) return mark
+    if (slot.value === undefined || slot.value.length === 0) {
+      return failure('unsupported-node-shape', `the ${span.name} mark wraps the [content] it marks`, scan.path)
+    }
+    return success(applyMark(slot.value, mark.value))
+  }
+  const node = readInlineDirectiveNode(span.name, span.attributes, slot.value, scan.path)
+  if (!node.ok) return node
+  return success([node.value])
+}
+
+function slotNodes(scan: Scan, content: string | undefined): Result<AdfNode[] | undefined> {
+  if (content === undefined) return success(undefined)
+  const parsed = parseInline(content, scan.definitions, scan.path, false)
+  if (!parsed.ok) return parsed
+  if (parsed.value.image !== undefined) return failure('unmappable-image', imageAlone, scan.path)
+  return success(parsed.value.nodes)
+}
+
 function flush(scan: Scan, strip: boolean): void {
   const raw = strip ? trimTrailingSpace(scan.pending) : scan.pending
   scan.pending = ''

src/markdown/parse/markdown-to-adf.test.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 import test from 'node:test'
 
-import type { AdfDocument, AdfMark, AdfNode } from '../../adf/document.ts'
+import type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from '../../adf/document.ts'
 import type { Result } from '../../result.ts'
 import { largestNesting } from '../../nesting.ts'
 import { markdownToAdf } from './markdown-to-adf.ts'
@@ -9,6 +9,7 @@ import { markdownToAdf } from './markdown-to-adf.ts'
 const em: AdfMark = { type: 'em' }
 const strike: AdfMark = { type: 'strike' }
 const strong: AdfMark = { type: 'strong' }
+const underline: AdfMark = { type: 'underline' }
 
 function code(result: Result<AdfDocument>): string {
   return result.ok ? `built ${JSON.stringify(result.value)}` : result.error.code
@@ -311,10 +312,6 @@ test('names the argument and the body a node takes no reading for', () => {
   assert.equal(content(markdownToAdf(':::codeBlock {wrap=true}\nx\n:::\n')), 'unsupported-node-shape: codeBlock takes one code block as its body')
   assert.equal(content(markdownToAdf(':::paragraph\n![a](/u)\n:::\n')), 'unmappable-image: no ADF node carries an image inside a paragraph')
   assert.equal(content(markdownToAdf('Part :date[now]{timestamp=1}.\n')), 'unsupported-node-shape: date takes no content')
-  assert.equal(
-    content(markdownToAdf('Part :emoji[x]{shortName=":x:"}.\n')),
-    'unsupported-node-shape: the content slot emoji spells its text attribute in is unsupported',
-  )
 })
 
 test('leaves the colon that opens no directive the text it is', () => {
@@ -460,6 +457,9 @@ test('names the block the claim inside a container opens', () => {
 test('refuses input nested deeper than the parser carries', () => {
   assert.equal(code(markdownToAdf('> '.repeat(501))), 'unsupported-nesting-depth')
   assert.ok(markdownToAdf('> '.repeat(500)).ok)
+  const marks = (levels: number): string => `${':underline['.repeat(levels)}a${']'.repeat(levels)}\n`
+  assert.equal(code(markdownToAdf(marks(largestNesting + 1))), 'unsupported-nesting-depth')
+  assert.deepEqual(content(markdownToAdf(marks(largestNesting))), [{ content: [marked('a', underline)], type: 'paragraph' }])
 })
 
 test('decodes the backslash escapes CommonMark spells, and keeps the rest literal', () => {
@@ -677,3 +677,72 @@ test('carries the mark a spelling nested inside its own kind names once', () =>
     { content: [marked('a ', em), marked('b', em, strong), marked(' c', em)], type: 'paragraph' },
   ])
 })
+
+test('reads the content slot as the text attribute the node spells there', () => {
+  const status = (attrs: AdfAttributes): AdfNode[] => [{ content: [{ attrs, type: 'status' }], type: 'paragraph' }]
+  assert.deepEqual(content(markdownToAdf(':status[In review]{color=yellow}\n')), status({ color: 'yellow', text: 'In review' }))
+  assert.deepEqual(content(markdownToAdf(':status{color=neutral}\n')), status({ color: 'neutral' }))
+  assert.deepEqual(content(markdownToAdf(':status[]{color=neutral}\n')), status({ color: 'neutral', text: '' }))
+  assert.deepEqual(content(markdownToAdf(':status[ In review ]{color=yellow}\n')), status({ color: 'yellow', text: ' In review ' }))
+  assert.deepEqual(content(markdownToAdf(':status[In:text{text=" "}review]{color=yellow}\n')), status({ color: 'yellow', text: 'In review' }))
+  assert.deepEqual(content(markdownToAdf(':status[a\\]b]{color=yellow}\n')), status({ color: 'yellow', text: 'a]b' }))
+  assert.deepEqual(content(markdownToAdf('**:mention[@A]{id=b1c2}**\n')), [
+    { content: [{ attrs: { id: 'b1c2', text: '@A' }, marks: [strong], type: 'mention' }], type: 'paragraph' },
+  ])
+})
+
+test('names the content slot no one unmarked text node reads back from', () => {
+  assert.equal(content(markdownToAdf(':status[**A**]{color=yellow}\n')), 'unsupported-node-shape: the status content slot holds one unmarked text node')
+  assert.equal(code(markdownToAdf(':status[a`b`]{color=yellow}\n')), 'unsupported-node-shape')
+  assert.equal(code(markdownToAdf(':status[:date{timestamp=1}]{color=yellow}\n')), 'unsupported-node-shape')
+  assert.equal(content(markdownToAdf(':status[![a](/u)]{color=yellow}\n')), 'unmappable-image: an image fits only as a paragraph of its own')
+  assert.equal(code(markdownToAdf(':status[<div>]{color=yellow}\n')), 'unmappable-html')
+  assert.equal(content(markdownToAdf('Part :mention{id=b1c2 text=A}.\n')), 'unsupported-node-shape: mention spells its text attribute in the content slot')
+})
+
+test('reads the whitespace the reserved text directive carries', () => {
+  assert.deepEqual(content(markdownToAdf(':text{text="  "}a\n')), [paragraph('  a')])
+  assert.deepEqual(content(markdownToAdf('a:text{text="\\n"}b\n')), [paragraph('a\nb')])
+  assert.deepEqual(content(markdownToAdf('a:text{text="\\t"}\n')), [paragraph('a\t')])
+  assert.deepEqual(content(markdownToAdf('_:text{text=" "}a_\n')), [{ content: [marked(' a', em)], type: 'paragraph' }])
+})
+
+test('names the text directive spelling no whitespace run reads back from', () => {
+  const named = 'unsupported-node-shape: text spells one run of spaces and tabs, or one run of newlines'
+  assert.equal(content(markdownToAdf(':text{text=hi}\n')), named)
+  assert.equal(content(markdownToAdf(':text{text=" \\n"}\n')), named)
+  assert.equal(content(markdownToAdf(':text{text=""}\n')), named)
+  assert.equal(content(markdownToAdf(':text{}\n')), 'unsupported-node-shape: text holds one text attribute alone')
+  assert.equal(content(markdownToAdf(':text{localId=a text=" "}\n')), 'unsupported-node-shape: text holds one text attribute alone')
+  assert.equal(content(markdownToAdf(':text[a]{text=" "}\n')), 'unsupported-node-shape: text takes no content')
+  assert.equal(content(markdownToAdf(':text{text="\\u0020"}\n')), 'unsupported-node-shape: text spells its text attribute as text=" "')
+})
+
+test('reads the directive marks, the nesting outermost first', () => {
+  const wrapped = (...marks: AdfMark[]): AdfNode[] => [{ content: [marked('a', ...marks)], type: 'paragraph' }]
+  assert.deepEqual(content(markdownToAdf(':underline[a]\n')), wrapped(underline))
+  assert.deepEqual(content(markdownToAdf('_:underline[a]_\n')), wrapped(em, underline))
+  assert.deepEqual(content(markdownToAdf(':underline[_a_]\n')), wrapped(underline, em))
+  assert.deepEqual(content(markdownToAdf(':underline[:underline[a]]\n')), wrapped(underline))
+  assert.deepEqual(content(markdownToAdf(':textColor[a]{color="#ae2e24"}\n')), wrapped({ attrs: { color: '#ae2e24' }, type: 'textColor' }))
+  assert.deepEqual(content(markdownToAdf(':subsup[a]{type=sub}\n')), wrapped({ attrs: { type: 'sub' }, type: 'subsup' }))
+  assert.deepEqual(content(markdownToAdf(':border[a]{color="#091e42" size=2}\n')), wrapped({ attrs: { color: '#091e42', size: 2 }, type: 'border' }))
+  assert.deepEqual(content(markdownToAdf(':underline[a:date{timestamp=1}]\n')), [
+    { content: [marked('a', underline), { attrs: { timestamp: '1' }, marks: [underline], type: 'date' }], type: 'paragraph' },
+  ])
+  assert.equal(content(markdownToAdf(':border[a]{color="#091e42" size=x}\n')), 'unsupported-node-shape: the size attribute of border is no number')
+})
+
+test('names the mark markdown spells, never a directive', () => {
+  assert.equal(content(markdownToAdf(':em[a]\n')), 'unsupported-node-shape: em is spelled _x_, never as a directive')
+  assert.equal(content(markdownToAdf(':strong[a]\n')), 'unsupported-node-shape: strong is spelled **x**, never as a directive')
+  assert.equal(content(markdownToAdf(':strike[a]\n')), 'unsupported-node-shape: strike is spelled ~~x~~, never as a directive')
+  assert.equal(content(markdownToAdf(':code[a]\n')), 'unsupported-node-shape: code is spelled `x`, never as a directive')
+  assert.equal(content(markdownToAdf(':link[a]{href="/u"}\n')), 'unsupported-node-shape: link is spelled [x](url), never as a directive')
+})
+
+test('names the directive mark left without the content it wraps', () => {
+  const named = 'unsupported-node-shape: the underline mark wraps the [content] it marks'
+  assert.equal(content(markdownToAdf(':underline[]\n')), named)
+  assert.equal(content(markdownToAdf(':underline{}\n')), named)
+})

src/markdown/text-directive.ts

@@ -0,0 +1,25 @@
+import type { ConvertFault } from '../result.ts'
+import type { DirectiveSpan, Read } from './directive-syntax.ts'
+import { spellAttributes, spellStringAttribute } from './directive-syntax.ts'
+
+const name = 'text'
+const whitespaceRun = /^(?:[ \t]+|\n+)$/
+
+export function spellTextDirective(text: string): string {
+  return `:${name}${spellAttributes([[name, spellStringAttribute(text)]])}`
+}
+
+export function readTextDirective(span: DirectiveSpan): Read<string> | undefined {
+  if (span.name !== name) return undefined
+  if (span.content !== undefined) return { fault: unsupported(`${name} takes no content`) }
+  const spelled = span.attributes.get(name)
+  if (spelled === undefined || span.attributes.size !== 1) return { fault: unsupported(`${name} holds one ${name} attribute alone`) }
+  const spelling = spellStringAttribute(spelled.decoded)
+  if (spelling !== spelled.spelling) return { fault: unsupported(`${name} spells its ${name} attribute as ${name}=${spelling}`) }
+  if (!whitespaceRun.test(spelled.decoded)) return { fault: unsupported(`${name} spells one run of spaces and tabs, or one run of newlines`) }
+  return { value: spelled.decoded }
+}
+
+function unsupported(message: string): ConvertFault {
+  return { code: 'unsupported-node-shape', message }
+}