Read the inline text, and decode the escapes and references CommonMark spells (#33)

2026-08-30 23:32:57 +02:00
parent ddc55bc7c8
commit 0476d33b7b
25 changed files with 729 additions and 110 deletions
@@ -4,13 +4,14 @@ import {
  claimsDirectiveLine,
  claimsPipeLine,
  closingCodeFence,
+  decodeTextEscapes,
  isThematicBreak,
  listMarker,
  markerInterruptsParagraph,
  openingCodeFence,
+  openingHtmlBlock,
  setextHeadingLevel,
 } from '../commonmark-grammar.ts'
-import { openingHtmlBlock } from './html-blocks.ts'
 import { readLinkDefinitions } from './link-reference-definitions.ts'

 export type ClaimedConstruct = 'directive' | 'pipe-table'
@@ -177,7 +178,7 @@ function continuesLazily(walk: Walk, line: Line): boolean {
  if (leadingColumns(line) >= indentedCodeColumns) return true
  const opener = removeColumns(line, largestOpenerIndentation).text
  if (claimedConstruct(opener) !== undefined || isThematicBreak(opener)) return false
-  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, false) === undefined
+  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, true) === undefined
 }

 function readBlockLine(walk: Walk, line: Line): void {
@@ -281,7 +282,7 @@ function closeLeaf(walk: Walk): void {
  }
  walk.leaf = undefined
  if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html' })
-  else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? leaf.info : '', text: leaf.lines.join('\n') })
+  else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? decodeTextEscapes(leaf.info) : '', text: leaf.lines.join('\n') })
 }

 function takeParagraph(walk: Walk): string | undefined {
@@ -1,28 +0,0 @@
-export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
-
-type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
-
-// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
-const blockTagNames =
-  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
-const attributeSource = '(?:[ \\t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \\t]*=[ \\t]*(?:[^ \\t"\'=<>`]+|\'[^\']*\'|"[^"]*"))?)'
-const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \\t]*/?>|</[A-Za-z][A-Za-z0-9-]*[ \\t]*>)[ \\t]*$`)
-const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/
-
-const conditions: HtmlBlockCondition[] = [
-  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
-  { closer: /-->/, construct: 'an HTML comment', interrupts: true, start: /^<!--/ },
-  { closer: /\?>/, construct: 'an HTML processing instruction', interrupts: true, start: /^<\?/ },
-  { closer: />/, construct: 'an HTML declaration', interrupts: true, start: /^<![A-Za-z]/ },
-  { closer: /\]\]>/, construct: 'a CDATA section', interrupts: true, start: /^<!\[CDATA\[/ },
-  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
-  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
-]
-
-export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
-  for (const condition of conditions) {
-    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
-    return { closer: condition.closer, construct: condition.construct ?? line.replace(tagName, '<$1>') }
-  }
-  return undefined
-}
@@ -0,0 +1,80 @@
+import type { AdfNode } from '../../adf/document.ts'
+import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts'
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
+import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
+
+type Run = { nodes: AdfNode[]; text: string; undecodedFrom: number }
+
+const hardBreakSpaces = / {2,}$/
+const trailingSpace = /[ \t]+$/
+
+export function parseInlineContent(source: string, path: ConvertErrorPath): Result<AdfNode[]> {
+  const run: Run = { nodes: [], text: '', undecodedFrom: 0 }
+  let index = 0
+  while (index < source.length) {
+    const character = source.charAt(index)
+    if (character === '\\' && source.charAt(index + 1) === '\n') {
+      // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash.
+      takeRun(run, source, index, index + 2, false)
+      pushNode(run, { type: 'hardBreak' })
+      index += 2
+      continue
+    }
+    if (character === '\n') {
+      const hard = hardBreakSpaces.test(source.slice(run.undecodedFrom, index))
+      takeRun(run, source, index, index + 1, true)
+      if (hard) pushNode(run, { type: 'hardBreak' })
+      else run.text += ' '
+      index += 1
+      continue
+    }
+    if (character === '`') {
+      const span = readCodeSpan(source, index)
+      if (span === undefined) {
+        index += backtickRun(source, index)
+        continue
+      }
+      takeRun(run, source, index, span.end, false)
+      pushNode(run, { marks: [{ type: 'code' }], text: span.text, type: 'text' })
+      index = span.end
+      continue
+    }
+    if (character === '<') {
+      const construct = inlineHtmlConstruct(source, index)
+      if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, path)
+    }
+    index += backslashEscape(source, index) === undefined ? 1 : 2
+  }
+  takeRun(run, source, source.length, source.length, true)
+  pushText(run)
+  return success(run.nodes)
+}
+
+function takeRun(run: Run, source: string, end: number, resume: number, strip: boolean): void {
+  const raw = source.slice(run.undecodedFrom, end)
+  run.text += decodeTextEscapes(strip ? raw.replace(trailingSpace, '') : raw)
+  run.undecodedFrom = resume
+}
+
+function pushText(run: Run): void {
+  if (run.text !== '') run.nodes.push({ text: run.text, type: 'text' })
+  run.text = ''
+}
+
+function pushNode(run: Run, node: AdfNode): void {
+  pushText(run)
+  run.nodes.push(node)
+}
+
+function readCodeSpan(source: string, index: number): { end: number; text: string } | undefined {
+  const opener = backtickRun(source, index)
+  const closer = closingBacktickRun(source, index + opener, opener)
+  if (closer === undefined) return undefined
+  return { end: closer + opener, text: codeSpanText(source.slice(index + opener, closer)) }
+}
+
+function codeSpanText(content: string): string {
+  const text = content.replaceAll('\n', ' ')
+  const padded = text.startsWith(' ') && text.endsWith(' ') && /[^ ]/.test(text)
+  return padded ? text.slice(1, -1) : text
+}
@@ -25,6 +25,14 @@ function paragraph(value: string): AdfNode {
  return { content: [text(value)], type: 'paragraph' }
 }

+function codeSpan(value: string): AdfNode {
+  return { marks: [{ type: 'code' }], text: value, type: 'text' }
+}
+
+function hardBreak(): AdfNode {
+  return { type: 'hardBreak' }
+}
+
 function item(...content: AdfNode[]): AdfNode {
  return content.length === 0 ? { type: 'listItem' } : { content, type: 'listItem' }
 }
@@ -112,7 +120,7 @@ test('claims a block-level colon run with no directive to parse it', () => {

 test('claims a block-level pipe with no table to parse it', () => {
  assert.equal(code(markdownToAdf('| Part | Qty |\n')), 'malformed-pipe-table')
-  assert.deepEqual(content(markdownToAdf('\\| Part\n')), [paragraph('\\| Part')])
+  assert.deepEqual(content(markdownToAdf('\\| Part\n')), [paragraph('| Part')])
 })

 test('refuses the raw HTML no element mapping carries', () => {
@@ -123,6 +131,8 @@ test('refuses the raw HTML no element mapping carries', () => {
  assert.equal(code(markdownToAdf('<![CDATA[x]]>\n')), 'unmappable-html')
  assert.equal(code(markdownToAdf('<pre>\nx\n</pre>\n')), 'unmappable-html')
  assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<div\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<?php\n')), 'unmappable-html')
  assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1])
  assert.equal(code(markdownToAdf('<div>\nx\n\n:::\n')), 'unmappable-html')
  assert.equal(code(markdownToAdf('<div>\n- x\n</div>\n')), 'unmappable-html')
@@ -133,11 +143,6 @@ test('swallows an HTML block ahead of the claim a line inside it would make', ()
  assert.equal(code(markdownToAdf('<div>\n| x |\n</div>\n')), 'unmappable-html')
 })

-test('leaves a tag that opens no HTML block to the paragraph it sits in', () => {
-  assert.deepEqual(content(markdownToAdf('Part.\n<span>\n')), [paragraph('Part. <span>')])
-  assert.deepEqual(content(markdownToAdf('3 < 4\n')), [paragraph('3 < 4')])
-})
-
 test('gives up the link reference definitions a paragraph opens with', () => {
  assert.deepEqual(content(markdownToAdf('[a]: /url\n')), [])
  assert.deepEqual(content(markdownToAdf('[a]: /url\n[b]: /other\nPart.\n')), [paragraph('Part.')])
@@ -222,6 +227,8 @@ test('folds a lazy continuation into the paragraph the container holds', () => {
  assert.deepEqual(content(markdownToAdf('> One\n---\n')), [quote(paragraph('One')), { type: 'rule' }])
  assert.deepEqual(content(markdownToAdf('> One\n```\n')), [quote(paragraph('One')), { type: 'codeBlock' }])
  assert.equal(code(markdownToAdf('> One\n<div>\n')), 'unmappable-html')
+  assert.deepEqual(path(markdownToAdf('> One\n<div>\n')), ['content', 1])
+  assert.deepEqual(path(markdownToAdf('> One\n<span>\n')), ['content', 0, 'content', 0])
 })

 test('ends a lazy continuation at a claimed line', () => {
@@ -239,3 +246,80 @@ test('refuses input nested deeper than the parser carries', () => {
  assert.equal(code(markdownToAdf('> '.repeat(501))), 'unsupported-nesting-depth')
  assert.ok(markdownToAdf('> '.repeat(500)).ok)
 })
+
+test('decodes the backslash escapes CommonMark spells, and keeps the rest literal', () => {
+  assert.deepEqual(content(markdownToAdf('\\*not emphasis\\*\n')), [paragraph('*not emphasis*')])
+  assert.deepEqual(content(markdownToAdf('\\\\\n')), [paragraph('\\')])
+  assert.deepEqual(content(markdownToAdf('\\a \\\u00a0\n')), [paragraph('\\a \\\u00a0')])
+  assert.deepEqual(content(markdownToAdf('Part\\\n')), [paragraph('Part\\')])
+  assert.deepEqual(content(markdownToAdf('a\\`b`\n')), [paragraph('a`b`')])
+})
+
+test('decodes the entity references HTML5 names, and the numeric ones', () => {
+  assert.deepEqual(content(markdownToAdf('&amp; &copy; &ngE; &zwnj; &AElig;\n')), [paragraph('& \u00a9 \u2267\u0338 \u200c \u00c6')])
+  assert.deepEqual(content(markdownToAdf('&#35; &#X22; &#x2665;\n')), [paragraph('# " \u2665')])
+  assert.deepEqual(content(markdownToAdf('&#0; &#xd800; &#9999999;\n')), [paragraph('\ufffd \ufffd \ufffd')])
+  assert.deepEqual(content(markdownToAdf('&zzz; &amp &#; &\n')), [paragraph('&zzz; &amp &#; &')])
+  assert.deepEqual(content(markdownToAdf('&#96;not code&#96;\n')), [paragraph('`not code`')])
+  assert.deepEqual(content(markdownToAdf('a&Tab;b&NewLine;c&nbsp;d&Aopf;e&verbar;f\n')), [paragraph('a\tb\nc d\u{1d538}e|f')])
+})
+
+test('reads a code span, its content literal', () => {
+  assert.deepEqual(content(markdownToAdf('Run `npm test` now.\n')), [
+    { content: [text('Run '), codeSpan('npm test'), text(' now.')], type: 'paragraph' },
+  ])
+  assert.deepEqual(content(markdownToAdf('``a`b``\n')), [{ content: [codeSpan('a`b')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('` `` `\n')), [{ content: [codeSpan('``')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`  `\n')), [{ content: [codeSpan('  ')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`a\nb`\n')), [{ content: [codeSpan('a b')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`foo``bar`\n')), [{ content: [codeSpan('foo``bar')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`:::panel` `~~x~~` `\\*` `&amp;`\n')), [
+    {
+      content: [codeSpan(':::panel'), text(' '), codeSpan('~~x~~'), text(' '), codeSpan('\\*'), text(' '), codeSpan('&amp;')],
+      type: 'paragraph',
+    },
+  ])
+  assert.deepEqual(content(markdownToAdf('`foo\n')), [paragraph('`foo')])
+  assert.deepEqual(content(markdownToAdf('``foo`\n')), [paragraph('``foo`')])
+})
+
+test('reads a hard break from a trailing backslash and from two trailing spaces alike', () => {
+  assert.deepEqual(content(markdownToAdf('One\\\ntwo.\n')), [{ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One  \ntwo.\n')), [{ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One\\  \ntwo.\n')), [{ content: [text('One\\'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One \\\ntwo.\n')), [{ content: [text('One '), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One \ntwo.\n')), [paragraph('One two.')])
+  assert.deepEqual(content(markdownToAdf('One \t\ntwo.\n')), [paragraph('One two.')])
+  assert.deepEqual(content(markdownToAdf('One  \n')), [paragraph('One')])
+  assert.deepEqual(content(markdownToAdf('> One\\\n> two.\n')), [quote({ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' })])
+})
+
+test('decodes the fenced info string the block walk leaves raw', () => {
+  assert.deepEqual(content(markdownToAdf('```java&#8203;script\nx\n```\n')), [
+    { attrs: { language: 'java\u200bscript' }, content: [text('x')], type: 'codeBlock' },
+  ])
+  assert.deepEqual(content(markdownToAdf('```\\#c\nx\n```\n')), [{ attrs: { language: '#c' }, content: [text('x')], type: 'codeBlock' }])
+})
+
+test('refuses the raw inline HTML no element mapping carries, naming it', () => {
+  assert.equal(content(markdownToAdf('Part <span> here.\n')), 'unmappable-html: no ADF node carries <span>')
+  assert.equal(content(markdownToAdf('Part </div> here.\n')), 'unmappable-html: no ADF node carries <div>')
+  assert.equal(content(markdownToAdf('Part <!-- note --> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(content(markdownToAdf('Part <?php ?> here.\n')), 'unmappable-html: no ADF node carries an HTML processing instruction')
+  assert.equal(content(markdownToAdf('Part <!DOCTYPE html> here.\n')), 'unmappable-html: no ADF node carries an HTML declaration')
+  assert.equal(content(markdownToAdf('Part <![CDATA[x]]> here.\n')), 'unmappable-html: no ADF node carries a CDATA section')
+  assert.equal(content(markdownToAdf('Part <!--> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(content(markdownToAdf('Part <!---> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(code(markdownToAdf('A <a href="/x" disabled\nid=y> b\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('Part.\n<span>\n')), 'unmappable-html')
+  assert.deepEqual(path(markdownToAdf('Part.\n\nA <b>b</b>.\n')), ['content', 1])
+})
+
+test('leaves the angle bracket that opens no HTML construct to the text it sits in', () => {
+  assert.deepEqual(content(markdownToAdf('3 < 4 and 5 <b 6\n')), [paragraph('3 < 4 and 5 <b 6')])
+  assert.deepEqual(content(markdownToAdf('a <b"c> d\n')), [paragraph('a <b"c> d')])
+  assert.deepEqual(content(markdownToAdf('a <!-- b\n')), [paragraph('a <!-- b')])
+  assert.deepEqual(content(markdownToAdf('a </b c> d\n')), [paragraph('a </b c> d')])
+  assert.deepEqual(content(markdownToAdf('`<span>`\n')), [{ content: [codeSpan('<span>')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('\\<span>\n')), [paragraph('<span>')])
+})
@@ -3,7 +3,7 @@ import type { Block, ClaimedConstruct } from './blocks.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { largestNesting } from '../../nesting.ts'
 import { parseBlocks } from './blocks.ts'
-import { trimSpace } from '../commonmark-grammar.ts'
+import { parseInlineContent } from './inline-content.ts'

 export function markdownToAdf(markdown: string): Result<AdfDocument> {
  const content = blockNodes(parseBlocks(markdown).blocks, [], 0)
@@ -33,13 +33,13 @@ function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result<
    case 'code':
      return success(codeBlockNode(block.language, block.text))
    case 'heading':
-      return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text))
+      return contentNode({ attrs: { level: block.level }, type: 'heading' }, block.text, path)
    case 'html':
      return failure('unmappable-html', `no ADF node carries ${block.construct}`, path)
    case 'orderedList':
      return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth)
    case 'paragraph':
-      return success(withContent({ type: 'paragraph' }, block.text))
+      return contentNode({ type: 'paragraph' }, block.text, path)
    case 'rule':
      return success({ type: 'rule' })
  }
@@ -75,15 +75,8 @@ function codeBlockNode(language: string, text: string): AdfNode {
  return text === '' ? node : { ...node, content: [{ text, type: 'text' }] }
 }

-function withContent(node: AdfNode, text: string): AdfNode {
-  const content = inlineContent(text)
-  return content.length === 0 ? node : { ...node, content }
-}
-
-function inlineContent(text: string): AdfNode[] {
-  const line = text
-    .split('\n')
-    .map((part) => trimSpace(part))
-    .join(' ')
-  return line === '' ? [] : [{ text: line, type: 'text' }]
+function contentNode(node: AdfNode, text: string, path: ConvertErrorPath): Result<AdfNode> {
+  const content = parseInlineContent(text, path)
+  if (!content.ok) return content
+  return success(content.value.length === 0 ? node : { ...node, content: content.value })
 }