Read the inline text, and decode the escapes and references CommonMark spells (#33)

src/markdown/backtick-runs.ts

@@ -1,3 +1,20 @@
+export function backtickRun(text: string, index: number): number {
+  let length = 0
+  while (text.charAt(index + length) === '`') length += 1
+  return length
+}
+
+// Where the run of exactly `opener` backticks closing a code span begins, `undefined` where none does.
+export function closingBacktickRun(text: string, from: number, opener: number): number | undefined {
+  let cursor = from
+  while (cursor < text.length) {
+    const run = backtickRun(text, cursor)
+    if (run === opener) return cursor
+    cursor += run === 0 ? 1 : run
+  }
+  return undefined
+}
+
 export function fencedCodeBlock(info: string, body: string): string {
   const fence = '`'.repeat(Math.max(3, longestBacktickRun(body) + 1))
   return body === '' ? `${fence}${info}\n${fence}` : `${fence}${info}\n${body}\n${fence}`

src/markdown/commonmark-grammar.ts

@@ -1,16 +1,53 @@
+import { readEntityReference } from './entity-references.ts'
+
 export type LinePosition = 'first' | 'later'
 
+type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
+
+type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
+
+const htmlConstructNames = {
+  cdata: 'a CDATA section',
+  comment: 'an HTML comment',
+  declaration: 'an HTML declaration',
+  processingInstruction: 'an HTML processing instruction',
+}
+
 const controlCharacterRange = '\\u0000-\\u001f\\u007f'
 const autolinkSource = `[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\\s<>${controlCharacterRange}]*`
 const nullCharacterSource = '\\u0000'
-const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});'
+const tagNameSource = '[A-Za-z][A-Za-z0-9-]*'
+const htmlSpaceSource = '[ \\t\\n]'
+const attributeSource = `(?:${htmlSpaceSource}+[A-Za-z_:][A-Za-z0-9_.:-]*(?:${htmlSpaceSource}*=${htmlSpaceSource}*(?:[^ \\t\\n"'=<>\`]+|'[^']*'|"[^"]*"))?)`
+
+const htmlTagSource = `(?:<${tagNameSource}${attributeSource}*${htmlSpaceSource}*/?>|</${tagNameSource}${htmlSpaceSource}*>)`
 
-const anchoredEntityReference = new RegExp(`^(?:${entityReferenceSource})`)
 const autolink = new RegExp(`^(?:${autolinkSource})$`)
-const bracketedAutolink = new RegExp(`^<(?:${autolinkSource})>`)
+const bracketedAutolink = new RegExp(`<(?:${autolinkSource})>`, 'y')
 const controlCharacter = new RegExp(`[${controlCharacterRange}]`)
-const entityReference = new RegExp(entityReferenceSource)
+const htmlTag = new RegExp(htmlTagSource, 'y')
 const nullCharacter = new RegExp(nullCharacterSource)
+const tagName = new RegExp(`^</?(${tagNameSource})[\\s\\S]*$`)
+// The opener's own match ends with the terminator where the construct is complete on its own (`<!-->`).
+const inlineHtmlConstructs = [
+  { name: htmlConstructNames.cdata, opener: /<!\[CDATA\[/y, terminator: ']]>' },
+  { name: htmlConstructNames.comment, opener: /<!(?:--->|-->|--)/y, terminator: '-->' },
+  { name: htmlConstructNames.declaration, opener: /<![A-Za-z]/y, terminator: '>' },
+  { name: htmlConstructNames.processingInstruction, opener: /<\?/y, terminator: '?>' },
+]
+// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
+const blockTagNames =
+  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
+const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`)
+const htmlBlockConditions: HtmlBlockCondition[] = [
+  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
+  { closer: /-->/, construct: htmlConstructNames.comment, interrupts: true, start: /^<!--/ },
+  { closer: /\?>/, construct: htmlConstructNames.processingInstruction, interrupts: true, start: /^<\?/ },
+  { closer: />/, construct: htmlConstructNames.declaration, interrupts: true, start: /^<![A-Za-z]/ },
+  { closer: /\]\]>/, construct: htmlConstructNames.cdata, interrupts: true, start: /^<!\[CDATA\[/ },
+  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
+  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
+]
 const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
 const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/
 const codeFenceOpener = /^(`{3,}|~{3,})/
@@ -19,7 +56,7 @@ const pipeClaim = /^\|/
 const bulletListOpener = /^[*+-](?:[ \t]|$)/
 // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip.
 const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim]
-const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/]
+const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y
 const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/
 const setextUnderline = /^(=+|-+)[ \t]*$/
 const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
@@ -32,6 +69,13 @@ export function atxHeading(line: string): { level: number; text: string } | unde
   return { level: hashes.length, text: trimSpace(text.replace(/(?:^|(?<=[ \t]))#+$/, '')) }
 }
 
+// The character a backslash escapes at `index`, `undefined` where the backslash is literal text.
+export function backslashEscape(text: string, index: number): string | undefined {
+  if (text.charAt(index) !== '\\') return undefined
+  const escaped = text.charAt(index + 1)
+  return isAsciiPunctuation(escaped) ? escaped : undefined
+}
+
 export function claimsDirectiveLine(line: string): boolean {
   return directiveClaim.test(line)
 }
@@ -50,9 +94,32 @@ export function closingCodeFence(line: string, marker: string): boolean {
   return /^[ \t]*$/.test(line.slice(closing.length))
 }
 
+export function decodeTextEscapes(text: string): string {
+  let decoded = ''
+  let index = 0
+  while (index < text.length) {
+    const escaped = backslashEscape(text, index)
+    if (escaped !== undefined) {
+      decoded += escaped
+      index += 2
+      continue
+    }
+    const reference = readEntityReference(text, index)
+    if (reference !== undefined) {
+      decoded += reference.text
+      index += reference.length
+      continue
+    }
+    decoded += text.charAt(index)
+    index += 1
+  }
+  return decoded
+}
+
 export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean {
   if (offset === 0) {
     if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true
+    if (openingHtmlBlock(line, position === 'later') !== undefined) return true
     return position === 'later' && setextUnderline.test(line)
   }
   const digits = orderedListOpener.exec(line)?.[1]
@@ -63,14 +130,27 @@ export function holdsControlCharacter(text: string): boolean {
   return controlCharacter.test(text)
 }
 
-export function holdsEntityReference(text: string): boolean {
-  return entityReference.test(text)
-}
-
 export function holdsNullCharacter(text: string): boolean {
   return nullCharacter.test(text)
 }
 
+function htmlTagName(text: string): string {
+  return text.replace(tagName, '<$1>')
+}
+
+export function inlineHtmlConstruct(text: string, index: number): string | undefined {
+  for (const construct of inlineHtmlConstructs) {
+    construct.opener.lastIndex = index
+    const opened = construct.opener.exec(text)?.[0]
+    if (opened === undefined) continue
+    if (opened.endsWith(construct.terminator)) return construct.name
+    return text.includes(construct.terminator, index + opened.length) ? construct.name : undefined
+  }
+  htmlTag.lastIndex = index
+  const tag = htmlTag.exec(text)?.[0]
+  return tag === undefined ? undefined : htmlTagName(tag)
+}
+
 export function isAsciiPunctuation(character: string): boolean {
   return asciiPunctuation.test(character)
 }
@@ -108,12 +188,22 @@ export function openingCodeFence(line: string): { info: string; marker: string }
   return marker.startsWith('`') && info.includes('`') ? undefined : { info, marker }
 }
 
-export function opensBracketedAutolink(text: string): boolean {
+export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
+  for (const condition of htmlBlockConditions) {
+    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
+    return { closer: condition.closer, construct: condition.construct ?? htmlTagName(line) }
+  }
+  return undefined
+}
+
+export function opensBracketedAutolink(text: string, index: number): boolean {
+  bracketedAutolink.lastIndex = index
   return bracketedAutolink.test(text)
 }
 
-export function opensHtmlConstruct(text: string): boolean {
-  return htmlConstructs.some((construct) => construct.test(text))
+export function opensEmailAutolink(text: string, index: number): boolean {
+  emailAutolink.lastIndex = index
+  return emailAutolink.test(text)
 }
 
 export function setextHeadingLevel(line: string): number | undefined {
@@ -122,10 +212,6 @@ export function setextHeadingLevel(line: string): number | undefined {
   return underline.startsWith('=') ? 1 : 2
 }
 
-export function startsEntityReference(text: string): boolean {
-  return anchoredEntityReference.test(text)
-}
-
 export function trimSpace(text: string): string {
   return text.replace(/^[ \t]+|[ \t]+$/g, '')
 }

src/markdown/directive-attributes.ts

@@ -3,7 +3,7 @@ import type { VocabularyPair } from '../adf/attribute-vocabulary.ts'
 import { serializeCanonicalJson } from '../canonical-json.ts'
 
 const bareToken = /^[A-Za-z0-9_-]+$/
-const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/
+const inlineDirectiveOpener = /:[a-z][A-Za-z0-9]*[[{]/y
 
 // spec/flavour.md, Attributes.
 const quotedEscapes = /[&<`|]/g
@@ -12,7 +12,8 @@ export function isBareToken(text: string): boolean {
   return bareToken.test(text)
 }
 
-export function opensInlineDirective(text: string): boolean {
+export function opensInlineDirective(text: string, index: number): boolean {
+  inlineDirectiveOpener.lastIndex = index
   return inlineDirectiveOpener.test(text)
 }
 

src/markdown/emit/adf-to-markdown.test.ts

@@ -202,8 +202,25 @@ test('spells a heading level no ATX heading fits as a directive', () => {
 test('escapes only text that would otherwise open a construct', () => {
   const emitted = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text, type: 'text' }))))
   assert.equal(emitted('<div>'), '\\<div>\n')
+  assert.equal(emitted('<div'), '\\<div\n')
+  assert.equal(emitted('<div and more'), '\\<div and more\n')
+  assert.equal(emitted('<pre'), '\\<pre\n')
+  assert.equal(emitted('<!x'), '\\<!x\n')
+  assert.equal(emitted('<!-- x'), '\\<!-- x\n')
+  assert.equal(emitted('<?php'), '\\<?php\n')
+  assert.equal(emitted('<![CDATA[x'), '\\<![CDATA[x\n')
+  assert.equal(emitted('<span'), '<span\n')
   assert.equal(emitted('a < b'), 'a < b\n')
   assert.equal(emitted('&amp; & x'), '\\&amp; & x\n')
+  assert.equal(emitted('&notareference; x'), '&notareference; x\n')
+  assert.equal(emitted('a <b@c.d> e'), 'a \\<b@c.d> e\n')
+  assert.equal(emitted('a <b 2'), 'a <b 2\n')
+  assert.equal(emitted('a <div b'), 'a <div b\n')
+  assert.equal(emitted('a <!-- b'), 'a <!-- b\n')
+  assert.equal(emitted('a <!-- b --> c'), 'a \\<!-- b --> c\n')
+  const later = paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, { text: '<div', type: 'text' })
+  assert.equal(markdown(adfToMarkdown(document(later))), 'a\\\n\\<div\n')
+  assert.equal(markdown(adfToMarkdown(document({ content: [paragraph({ text: '<!-- x', type: 'text' })], type: 'blockquote' }))), '> \\<!-- x\n')
   assert.equal(emitted('| a | b |'), '\\| a | b |\n')
   assert.equal(emitted(':mention[@x]{id=1}'), '\\:mention[@x]{id=1}\n')
   assert.equal(emitted(':::panel info'), '\\:::panel info\n')

src/markdown/emit/adf-to-markdown.ts

@@ -7,7 +7,8 @@ import { carriesOnly, isAdfDocument } from '../../adf/document.ts'
 import { emitInlineLine } from './inline-line.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { fencedCodeBlock } from '../backtick-runs.ts'
-import { holdsControlCharacter, holdsEntityReference, holdsNullCharacter, isThematicBreak, markerInterruptsParagraph } from '../commonmark-grammar.ts'
+import { holdsControlCharacter, holdsNullCharacter, isThematicBreak, markerInterruptsParagraph } from '../commonmark-grammar.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 import { largestNesting } from '../../nesting.ts'
 import { spellDirectiveHeader } from './block-directive-spelling.ts'
 import { tryImage } from './image.ts'

src/markdown/emit/destination-spelling.ts

@@ -1,11 +1,12 @@
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
-import { holdsControlCharacter, holdsEntityReference } from '../commonmark-grammar.ts'
+import { holdsControlCharacter } from '../commonmark-grammar.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 
 export function spellDestination(href: string, path: ConvertErrorPath): Result<string> {
   if (holdsControlCharacter(href)) return failure('unspellable-link-destination', 'a link destination holds a control character', path)
   if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination', path)
   if (holdsEntityReference(href)) {
-    return failure('unspellable-link-destination', 'a link destination shaped like an entity reference decodes on the way back', path)
+    return failure('unspellable-link-destination', 'a link destination holds an entity reference that decodes on the way back', path)
   }
   if (href.includes(' ')) {
     if (/[<>]/.test(href)) {
@@ -22,7 +23,7 @@ export function spellTitle(title: string, path: ConvertErrorPath): Result<string
   if (/["\n\r\\]/.test(title)) {
     return failure('unspellable-link-title', 'no canonical escape spells a quote, backslash or newline in a link title', path)
   }
-  if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title shaped like an entity reference decodes on the way back', path)
+  if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title holds an entity reference that decodes on the way back', path)
   return success(` "${title}"`)
 }
 

src/markdown/emit/inline-line.ts

@@ -2,8 +2,9 @@ import type { AdfMark, AdfNode } from '../../adf/document.ts'
 import type { InlineDirective } from '../../adf/inline-directives.ts'
 import { assembleInlineLine, type InlineEscaping, type InlineSegment, type LineContainer, type NodeRange } from './line-escaping.ts'
 import { carriedInline } from '../opaque-carry.ts'
-import { claimsLine, holdsEntityReference, holdsNullCharacter, isAutolink } from '../commonmark-grammar.ts'
+import { claimsLine, holdsNullCharacter, isAutolink } from '../commonmark-grammar.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 import { inlineDirective } from '../../adf/inline-directives.ts'
 import { largestNesting } from '../../nesting.ts'
 import { longestBacktickRun } from '../backtick-runs.ts'

src/markdown/emit/line-escaping.ts

@@ -1,13 +1,8 @@
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { delimiterFlags, isWordCharacter, matchEmphasis } from '../emphasis-matching.ts'
-import {
-  escapesLineClaim,
-  isAsciiPunctuation,
-  opensBracketedAutolink,
-  opensHtmlConstruct,
-  startsEntityReference,
-  type LinePosition,
-} from '../commonmark-grammar.ts'
+import { backslashEscape, escapesLineClaim, inlineHtmlConstruct, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts'
 import { opensInlineDirective } from '../directive-attributes.ts'
+import { readEntityReference } from '../entity-references.ts'
 
 export type EmphasisRole = 'close' | 'open'
 
@@ -21,6 +16,8 @@ export type InlineSegment =
 
 export type AssembledLine = { line: string; unspellableRun: NodeRange | undefined }
 
+type ScanLine = { position: LinePosition; start: number; text: string }
+
 export type LineContainer = 'heading' | 'paragraph' | 'table-cell'
 
 type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number }
@@ -72,10 +69,18 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A
   const escaped = new Set<number>()
   const placements: number[] = []
   let output = ''
+  const linkClose = lastLinkClose(scan, escapings)
+  let line = scanLine(scan, 0)
   for (let index = 0; index < scan.length; index += 1) {
+    if (index > line.start + line.text.length) line = scanLine(scan, line.start + line.text.length + 1)
     const escaping = escapings[index]
     const escapable = escaping === 'backslash' || escaping === 'bracketed'
-    if (escapable && (mergesWithSyntax(scan, escapings, index) || opensConstruct(scan, escapings, index, escaping === 'bracketed', container, escaped))) {
+    if (
+      escapable &&
+      (claimsLineStart(line, index, container) ||
+        mergesWithSyntax(scan, escapings, index) ||
+        opensConstruct(scan, linkClose, index, escaping === 'bracketed', container, escaped))
+    ) {
       output += '\\'
       escaped.add(index)
     }
@@ -181,23 +186,23 @@ function isSyntax(escaping: InlineEscaping | undefined): boolean {
 
 function opensConstruct(
   scan: string,
-  escapings: readonly (InlineEscaping | undefined)[],
+  linkClose: number,
   index: number,
   inBrackets: boolean,
   container: LineContainer,
   escaped: ReadonlySet<number>,
 ): boolean {
   if (container === 'heading' && closesHeading(scan, index)) return true
-  if (container === 'paragraph' && claimsLineStart(scan, index)) return true
-  return claimsCharacter(scan, escapings, index, inBrackets, container, escaped)
+  return claimsCharacter(scan, linkClose, index, inBrackets, container, escaped)
 }
 
-function claimsLineStart(scan: string, index: number): boolean {
-  const start = scan.lastIndexOf('\n', index - 1) + 1
-  const end = scan.indexOf('\n', index)
-  const line = scan.slice(start, end === -1 ? undefined : end)
-  const position: LinePosition = start === 0 ? 'first' : 'later'
-  return escapesLineClaim(line, index - start, position)
+function claimsLineStart(line: ScanLine, index: number, container: LineContainer): boolean {
+  return container === 'paragraph' && escapesLineClaim(line.text, index - line.start, line.position)
+}
+
+function scanLine(scan: string, start: number): ScanLine {
+  const end = scan.indexOf('\n', start)
+  return { position: start === 0 ? 'first' : 'later', start, text: scan.slice(start, end === -1 ? undefined : end) }
 }
 
 function closesHeading(scan: string, index: number): boolean {
@@ -207,39 +212,38 @@ function closesHeading(scan: string, index: number): boolean {
 
 function claimsCharacter(
   scan: string,
-  escapings: readonly (InlineEscaping | undefined)[],
+  linkClose: number,
   index: number,
   inBrackets: boolean,
   container: LineContainer,
   escaped: ReadonlySet<number>,
 ): boolean {
   const character = scan.charAt(index)
-  const rest = scan.slice(index)
   if (inBrackets && (character === '[' || character === ']')) return true
   if (character === '|') return container === 'table-cell'
-  if (character === '\\') return isAsciiPunctuation(scan.charAt(index + 1))
-  if (character === '&') return startsEntityReference(rest)
-  if (character === '<') return opensBracketedAutolink(rest) || opensHtmlConstruct(rest)
-  if (character === ':') return opensInlineDirective(rest)
-  if (character === '[') return opensLink(scan, escapings, index)
+  if (character === '\\') return backslashEscape(scan, index) !== undefined
+  if (character === '&') return readEntityReference(scan, index) !== undefined
+  if (character === '<') return opensBracketedAutolink(scan, index) || opensEmailAutolink(scan, index) || inlineHtmlConstruct(scan, index) !== undefined
+  if (character === ':') return opensInlineDirective(scan, index)
+  if (character === '[') return index < linkClose
   if (character === '`') return opensCodeSpan(scan, index, escaped)
   if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped)
   return false
 }
 
 // A `]` the emitter spelled sits inside a construct that binds before link text does.
-function opensLink(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean {
-  for (let cursor = index + 1; cursor < scan.length; cursor += 1) {
+function lastLinkClose(scan: string, escapings: readonly (InlineEscaping | undefined)[]): number {
+  for (let cursor = scan.length - 1; cursor >= 0; cursor -= 1) {
     if (scan.charAt(cursor) !== ']' || isSyntax(escapings[cursor])) continue
-    if (followsLinkText.test(scan.charAt(cursor + 1))) return true
+    if (followsLinkText.test(scan.charAt(cursor + 1))) return cursor
   }
-  return false
+  return -1
 }
 
 function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
   if (!startsRun(scan, index, escaped)) return false
-  const length = runLength(scan, index)
-  return new RegExp('(?<!`)`{' + length + '}(?!`)').test(scan.slice(index + length))
+  const opener = backtickRun(scan, index)
+  return closingBacktickRun(scan, index + opener, opener) !== undefined
 }
 
 function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet<number>): boolean {

src/markdown/entity-references.test.ts

@@ -0,0 +1,21 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+import { readEntityReference } from './entity-references.ts'
+
+const source = readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'entity-references.ts'), 'utf8')
+
+// A value holding a space or a tilde mis-splits into a wrong name and a lost one, silently, at load.
+test('every packed reference parts into one name and one value the table decodes', () => {
+  const packed = /const packedReferences =\n {2}'([\s\S]*?)'\n/.exec(source)?.[1]
+  assert.ok(packed !== undefined)
+  const references = packed.split(/(?:\\\n)? +/)
+  assert.equal(references.length, 2125)
+  for (const reference of references) {
+    assert.match(reference, /^[A-Za-z][A-Za-z0-9]*~[^ ~]+$/)
+    assert.ok((readEntityReference(`&${reference.slice(0, reference.indexOf('~'))};`, 0)?.text ?? '') !== '')
+  }
+})

src/markdown/entity-references.ts

@@ -0,0 +1,238 @@
+type EntityReference = { length: number; text: string }
+
+const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});'
+const anchoredEntityReference = new RegExp(`(?:${entityReferenceSource})`, 'y')
+const decimalReference = /^&#(\d+);/
+const hexadecimalReference = /^&#[Xx]([A-Fa-f0-9]+);/
+const largestCodePoint = 0x10ffff
+const replacementCharacter = '\ufffd'
+const surrogates = { first: 0xd800, last: 0xdfff }
+
+// HTML5's named character references (https://html.spec.whatwg.org/entities.json), the semicolon-terminated half
+// CommonMark decodes. Name and characters part on a tilde, references on a space, neither of which any value holds.
+const packedReferences =
+  'AElig~Æ AMP~& Aacute~Á Abreve~Ă Acirc~Â Acy~А Afr~𝔄 Agrave~À Alpha~Α Amacr~Ā And~⩓ Aogon~Ą Aopf~𝔸\
+  ApplyFunction~⁡ Aring~Å Ascr~𝒜 Assign~≔ Atilde~Ã Auml~Ä Backslash~∖ Barv~⫧ Barwed~⌆ Bcy~Б Because~∵ Bernoullis~ℬ\
+  Beta~Β Bfr~𝔅 Bopf~𝔹 Breve~˘ Bscr~ℬ Bumpeq~≎ CHcy~Ч COPY~© Cacute~Ć Cap~⋒ CapitalDifferentialD~ⅅ Cayleys~ℭ\
+  Ccaron~Č Ccedil~Ç Ccirc~Ĉ Cconint~∰ Cdot~Ċ Cedilla~¸ CenterDot~· Cfr~ℭ Chi~Χ CircleDot~⊙ CircleMinus~⊖\
+  CirclePlus~⊕ CircleTimes~⊗ ClockwiseContourIntegral~∲ CloseCurlyDoubleQuote~” CloseCurlyQuote~’ Colon~∷ Colone~⩴\
+  Congruent~≡ Conint~∯ ContourIntegral~∮ Copf~ℂ Coproduct~∐ CounterClockwiseContourIntegral~∳ Cross~⨯ Cscr~𝒞 Cup~⋓\
+  CupCap~≍ DD~ⅅ DDotrahd~⤑ DJcy~Ђ DScy~Ѕ DZcy~Џ Dagger~‡ Darr~↡ Dashv~⫤ Dcaron~Ď Dcy~Д Del~∇ Delta~Δ Dfr~𝔇\
+  DiacriticalAcute~´ DiacriticalDot~˙ DiacriticalDoubleAcute~˝ DiacriticalGrave~` DiacriticalTilde~˜ Diamond~⋄\
+  DifferentialD~ⅆ Dopf~𝔻 Dot~¨ DotDot~⃜ DotEqual~≐ DoubleContourIntegral~∯ DoubleDot~¨ DoubleDownArrow~⇓\
+  DoubleLeftArrow~⇐ DoubleLeftRightArrow~⇔ DoubleLeftTee~⫤ DoubleLongLeftArrow~⟸ DoubleLongLeftRightArrow~⟺\
+  DoubleLongRightArrow~⟹ DoubleRightArrow~⇒ DoubleRightTee~⊨ DoubleUpArrow~⇑ DoubleUpDownArrow~⇕ DoubleVerticalBar~∥\
+  DownArrow~↓ DownArrowBar~⤓ DownArrowUpArrow~⇵ DownBreve~̑ DownLeftRightVector~⥐ DownLeftTeeVector~⥞\
+  DownLeftVector~↽ DownLeftVectorBar~⥖ DownRightTeeVector~⥟ DownRightVector~⇁ DownRightVectorBar~⥗ DownTee~⊤\
+  DownTeeArrow~↧ Downarrow~⇓ Dscr~𝒟 Dstrok~Đ ENG~Ŋ ETH~Ð Eacute~É Ecaron~Ě Ecirc~Ê Ecy~Э Edot~Ė Efr~𝔈 Egrave~È\
+  Element~∈ Emacr~Ē EmptySmallSquare~◻ EmptyVerySmallSquare~▫ Eogon~Ę Eopf~𝔼 Epsilon~Ε Equal~⩵ EqualTilde~≂\
+  Equilibrium~⇌ Escr~ℰ Esim~⩳ Eta~Η Euml~Ë Exists~∃ ExponentialE~ⅇ Fcy~Ф Ffr~𝔉 FilledSmallSquare~◼\
+  FilledVerySmallSquare~▪ Fopf~𝔽 ForAll~∀ Fouriertrf~ℱ Fscr~ℱ GJcy~Ѓ GT~> Gamma~Γ Gammad~Ϝ Gbreve~Ğ Gcedil~Ģ\
+  Gcirc~Ĝ Gcy~Г Gdot~Ġ Gfr~𝔊 Gg~⋙ Gopf~𝔾 GreaterEqual~≥ GreaterEqualLess~⋛ GreaterFullEqual~≧ GreaterGreater~⪢\
+  GreaterLess~≷ GreaterSlantEqual~⩾ GreaterTilde~≳ Gscr~𝒢 Gt~≫ HARDcy~Ъ Hacek~ˇ Hat~^ Hcirc~Ĥ Hfr~ℌ HilbertSpace~ℋ\
+  Hopf~ℍ HorizontalLine~─ Hscr~ℋ Hstrok~Ħ HumpDownHump~≎ HumpEqual~≏ IEcy~Е IJlig~IJ IOcy~Ё Iacute~Í Icirc~Î Icy~И\
+  Idot~İ Ifr~ℑ Igrave~Ì Im~ℑ Imacr~Ī ImaginaryI~ⅈ Implies~⇒ Int~∬ Integral~∫ Intersection~⋂ InvisibleComma~⁣\
+  InvisibleTimes~⁢ Iogon~Į Iopf~𝕀 Iota~Ι Iscr~ℐ Itilde~Ĩ Iukcy~І Iuml~Ï Jcirc~Ĵ Jcy~Й Jfr~𝔍 Jopf~𝕁 Jscr~𝒥\
+  Jsercy~Ј Jukcy~Є KHcy~Х KJcy~Ќ Kappa~Κ Kcedil~Ķ Kcy~К Kfr~𝔎 Kopf~𝕂 Kscr~𝒦 LJcy~Љ LT~< Lacute~Ĺ Lambda~Λ Lang~⟪\
+  Laplacetrf~ℒ Larr~↞ Lcaron~Ľ Lcedil~Ļ Lcy~Л LeftAngleBracket~⟨ LeftArrow~← LeftArrowBar~⇤ LeftArrowRightArrow~⇆\
+  LeftCeiling~⌈ LeftDoubleBracket~⟦ LeftDownTeeVector~⥡ LeftDownVector~⇃ LeftDownVectorBar~⥙ LeftFloor~⌊\
+  LeftRightArrow~↔ LeftRightVector~⥎ LeftTee~⊣ LeftTeeArrow~↤ LeftTeeVector~⥚ LeftTriangle~⊲ LeftTriangleBar~⧏\
+  LeftTriangleEqual~⊴ LeftUpDownVector~⥑ LeftUpTeeVector~⥠ LeftUpVector~↿ LeftUpVectorBar~⥘ LeftVector~↼\
+  LeftVectorBar~⥒ Leftarrow~⇐ Leftrightarrow~⇔ LessEqualGreater~⋚ LessFullEqual~≦ LessGreater~≶ LessLess~⪡\
+  LessSlantEqual~⩽ LessTilde~≲ Lfr~𝔏 Ll~⋘ Lleftarrow~⇚ Lmidot~Ŀ LongLeftArrow~⟵ LongLeftRightArrow~⟷\
+  LongRightArrow~⟶ Longleftarrow~⟸ Longleftrightarrow~⟺ Longrightarrow~⟹ Lopf~𝕃 LowerLeftArrow~↙ LowerRightArrow~↘\
+  Lscr~ℒ Lsh~↰ Lstrok~Ł Lt~≪ Map~⤅ Mcy~М MediumSpace~  Mellintrf~ℳ Mfr~𝔐 MinusPlus~∓ Mopf~𝕄 Mscr~ℳ Mu~Μ NJcy~Њ\
+  Nacute~Ń Ncaron~Ň Ncedil~Ņ Ncy~Н NegativeMediumSpace~​ NegativeThickSpace~​ NegativeThinSpace~​\
+  NegativeVeryThinSpace~​ NestedGreaterGreater~≫ NestedLessLess~≪ NewLine~\n Nfr~𝔑 NoBreak~⁠ NonBreakingSpace~ \
+  Nopf~ℕ Not~⫬ NotCongruent~≢ NotCupCap~≭ NotDoubleVerticalBar~∦ NotElement~∉ NotEqual~≠ NotEqualTilde~≂̸\
+  NotExists~∄ NotGreater~≯ NotGreaterEqual~≱ NotGreaterFullEqual~≧̸ NotGreaterGreater~≫̸ NotGreaterLess~≹\
+  NotGreaterSlantEqual~⩾̸ NotGreaterTilde~≵ NotHumpDownHump~≎̸ NotHumpEqual~≏̸ NotLeftTriangle~⋪\
+  NotLeftTriangleBar~⧏̸ NotLeftTriangleEqual~⋬ NotLess~≮ NotLessEqual~≰ NotLessGreater~≸ NotLessLess~≪̸\
+  NotLessSlantEqual~⩽̸ NotLessTilde~≴ NotNestedGreaterGreater~⪢̸ NotNestedLessLess~⪡̸ NotPrecedes~⊀\
+  NotPrecedesEqual~⪯̸ NotPrecedesSlantEqual~⋠ NotReverseElement~∌ NotRightTriangle~⋫ NotRightTriangleBar~⧐̸\
+  NotRightTriangleEqual~⋭ NotSquareSubset~⊏̸ NotSquareSubsetEqual~⋢ NotSquareSuperset~⊐̸ NotSquareSupersetEqual~⋣\
+  NotSubset~⊂⃒ NotSubsetEqual~⊈ NotSucceeds~⊁ NotSucceedsEqual~⪰̸ NotSucceedsSlantEqual~⋡ NotSucceedsTilde~≿̸\
+  NotSuperset~⊃⃒ NotSupersetEqual~⊉ NotTilde~≁ NotTildeEqual~≄ NotTildeFullEqual~≇ NotTildeTilde~≉ NotVerticalBar~∤\
+  Nscr~𝒩 Ntilde~Ñ Nu~Ν OElig~Œ Oacute~Ó Ocirc~Ô Ocy~О Odblac~Ő Ofr~𝔒 Ograve~Ò Omacr~Ō Omega~Ω Omicron~Ο Oopf~𝕆\
+  OpenCurlyDoubleQuote~“ OpenCurlyQuote~‘ Or~⩔ Oscr~𝒪 Oslash~Ø Otilde~Õ Otimes~⨷ Ouml~Ö OverBar~‾ OverBrace~⏞\
+  OverBracket~⎴ OverParenthesis~⏜ PartialD~∂ Pcy~П Pfr~𝔓 Phi~Φ Pi~Π PlusMinus~± Poincareplane~ℌ Popf~ℙ Pr~⪻\
+  Precedes~≺ PrecedesEqual~⪯ PrecedesSlantEqual~≼ PrecedesTilde~≾ Prime~″ Product~∏ Proportion~∷ Proportional~∝\
+  Pscr~𝒫 Psi~Ψ QUOT~" Qfr~𝔔 Qopf~ℚ Qscr~𝒬 RBarr~⤐ REG~® Racute~Ŕ Rang~⟫ Rarr~↠ Rarrtl~⤖ Rcaron~Ř Rcedil~Ŗ Rcy~Р\
+  Re~ℜ ReverseElement~∋ ReverseEquilibrium~⇋ ReverseUpEquilibrium~⥯ Rfr~ℜ Rho~Ρ RightAngleBracket~⟩ RightArrow~→\
+  RightArrowBar~⇥ RightArrowLeftArrow~⇄ RightCeiling~⌉ RightDoubleBracket~⟧ RightDownTeeVector~⥝ RightDownVector~⇂\
+  RightDownVectorBar~⥕ RightFloor~⌋ RightTee~⊢ RightTeeArrow~↦ RightTeeVector~⥛ RightTriangle~⊳ RightTriangleBar~⧐\
+  RightTriangleEqual~⊵ RightUpDownVector~⥏ RightUpTeeVector~⥜ RightUpVector~↾ RightUpVectorBar~⥔ RightVector~⇀\
+  RightVectorBar~⥓ Rightarrow~⇒ Ropf~ℝ RoundImplies~⥰ Rrightarrow~⇛ Rscr~ℛ Rsh~↱ RuleDelayed~⧴ SHCHcy~Щ SHcy~Ш\
+  SOFTcy~Ь Sacute~Ś Sc~⪼ Scaron~Š Scedil~Ş Scirc~Ŝ Scy~С Sfr~𝔖 ShortDownArrow~↓ ShortLeftArrow~← ShortRightArrow~→\
+  ShortUpArrow~↑ Sigma~Σ SmallCircle~∘ Sopf~𝕊 Sqrt~√ Square~□ SquareIntersection~⊓ SquareSubset~⊏\
+  SquareSubsetEqual~⊑ SquareSuperset~⊐ SquareSupersetEqual~⊒ SquareUnion~⊔ Sscr~𝒮 Star~⋆ Sub~⋐ Subset~⋐\
+  SubsetEqual~⊆ Succeeds~≻ SucceedsEqual~⪰ SucceedsSlantEqual~≽ SucceedsTilde~≿ SuchThat~∋ Sum~∑ Sup~⋑ Superset~⊃\
+  SupersetEqual~⊇ Supset~⋑ THORN~Þ TRADE~™ TSHcy~Ћ TScy~Ц Tab~\t Tau~Τ Tcaron~Ť Tcedil~Ţ Tcy~Т Tfr~𝔗 Therefore~∴\
+  Theta~Θ ThickSpace~   ThinSpace~  Tilde~∼ TildeEqual~≃ TildeFullEqual~≅ TildeTilde~≈ Topf~𝕋 TripleDot~⃛ Tscr~𝒯\
+  Tstrok~Ŧ Uacute~Ú Uarr~↟ Uarrocir~⥉ Ubrcy~Ў Ubreve~Ŭ Ucirc~Û Ucy~У Udblac~Ű Ufr~𝔘 Ugrave~Ù Umacr~Ū UnderBar~_\
+  UnderBrace~⏟ UnderBracket~⎵ UnderParenthesis~⏝ Union~⋃ UnionPlus~⊎ Uogon~Ų Uopf~𝕌 UpArrow~↑ UpArrowBar~⤒\
+  UpArrowDownArrow~⇅ UpDownArrow~↕ UpEquilibrium~⥮ UpTee~⊥ UpTeeArrow~↥ Uparrow~⇑ Updownarrow~⇕ UpperLeftArrow~↖\
+  UpperRightArrow~↗ Upsi~ϒ Upsilon~Υ Uring~Ů Uscr~𝒰 Utilde~Ũ Uuml~Ü VDash~⊫ Vbar~⫫ Vcy~В Vdash~⊩ Vdashl~⫦ Vee~⋁\
+  Verbar~‖ Vert~‖ VerticalBar~∣ VerticalLine~| VerticalSeparator~❘ VerticalTilde~≀ VeryThinSpace~  Vfr~𝔙 Vopf~𝕍\
+  Vscr~𝒱 Vvdash~⊪ Wcirc~Ŵ Wedge~⋀ Wfr~𝔚 Wopf~𝕎 Wscr~𝒲 Xfr~𝔛 Xi~Ξ Xopf~𝕏 Xscr~𝒳 YAcy~Я YIcy~Ї YUcy~Ю Yacute~Ý\
+  Ycirc~Ŷ Ycy~Ы Yfr~𝔜 Yopf~𝕐 Yscr~𝒴 Yuml~Ÿ ZHcy~Ж Zacute~Ź Zcaron~Ž Zcy~З Zdot~Ż ZeroWidthSpace~​ Zeta~Ζ Zfr~ℨ\
+  Zopf~ℤ Zscr~𝒵 aacute~á abreve~ă ac~∾ acE~∾̳ acd~∿ acirc~â acute~´ acy~а aelig~æ af~⁡ afr~𝔞 agrave~à alefsym~ℵ\
+  aleph~ℵ alpha~α amacr~ā amalg~⨿ amp~& and~∧ andand~⩕ andd~⩜ andslope~⩘ andv~⩚ ang~∠ ange~⦤ angle~∠ angmsd~∡\
+  angmsdaa~⦨ angmsdab~⦩ angmsdac~⦪ angmsdad~⦫ angmsdae~⦬ angmsdaf~⦭ angmsdag~⦮ angmsdah~⦯ angrt~∟ angrtvb~⊾\
+  angrtvbd~⦝ angsph~∢ angst~Å angzarr~⍼ aogon~ą aopf~𝕒 ap~≈ apE~⩰ apacir~⩯ ape~≊ apid~≋ apos~\' approx~≈ approxeq~≊\
+  aring~å ascr~𝒶 ast~* asymp~≈ asympeq~≍ atilde~ã auml~ä awconint~∳ awint~⨑ bNot~⫭ backcong~≌ backepsilon~϶\
+  backprime~‵ backsim~∽ backsimeq~⋍ barvee~⊽ barwed~⌅ barwedge~⌅ bbrk~⎵ bbrktbrk~⎶ bcong~≌ bcy~б bdquo~„ becaus~∵\
+  because~∵ bemptyv~⦰ bepsi~϶ bernou~ℬ beta~β beth~ℶ between~≬ bfr~𝔟 bigcap~⋂ bigcirc~◯ bigcup~⋃ bigodot~⨀\
+  bigoplus~⨁ bigotimes~⨂ bigsqcup~⨆ bigstar~★ bigtriangledown~▽ bigtriangleup~△ biguplus~⨄ bigvee~⋁ bigwedge~⋀\
+  bkarow~⤍ blacklozenge~⧫ blacksquare~▪ blacktriangle~▴ blacktriangledown~▾ blacktriangleleft~◂ blacktriangleright~▸\
+  blank~␣ blk12~▒ blk14~░ blk34~▓ block~█ bne~=⃥ bnequiv~≡⃥ bnot~⌐ bopf~𝕓 bot~⊥ bottom~⊥ bowtie~⋈ boxDL~╗ boxDR~╔\
+  boxDl~╖ boxDr~╓ boxH~═ boxHD~╦ boxHU~╩ boxHd~╤ boxHu~╧ boxUL~╝ boxUR~╚ boxUl~╜ boxUr~╙ boxV~║ boxVH~╬ boxVL~╣\
+  boxVR~╠ boxVh~╫ boxVl~╢ boxVr~╟ boxbox~⧉ boxdL~╕ boxdR~╒ boxdl~┐ boxdr~┌ boxh~─ boxhD~╥ boxhU~╨ boxhd~┬ boxhu~┴\
+  boxminus~⊟ boxplus~⊞ boxtimes~⊠ boxuL~╛ boxuR~╘ boxul~┘ boxur~└ boxv~│ boxvH~╪ boxvL~╡ boxvR~╞ boxvh~┼ boxvl~┤\
+  boxvr~├ bprime~‵ breve~˘ brvbar~¦ bscr~𝒷 bsemi~⁏ bsim~∽ bsime~⋍ bsol~\\ bsolb~⧅ bsolhsub~⟈ bull~• bullet~• bump~≎\
+  bumpE~⪮ bumpe~≏ bumpeq~≏ cacute~ć cap~∩ capand~⩄ capbrcup~⩉ capcap~⩋ capcup~⩇ capdot~⩀ caps~∩︀ caret~⁁ caron~ˇ\
+  ccaps~⩍ ccaron~č ccedil~ç ccirc~ĉ ccups~⩌ ccupssm~⩐ cdot~ċ cedil~¸ cemptyv~⦲ cent~¢ centerdot~· cfr~𝔠 chcy~ч\
+  check~✓ checkmark~✓ chi~χ cir~○ cirE~⧃ circ~ˆ circeq~≗ circlearrowleft~↺ circlearrowright~↻ circledR~® circledS~Ⓢ\
+  circledast~⊛ circledcirc~⊚ circleddash~⊝ cire~≗ cirfnint~⨐ cirmid~⫯ cirscir~⧂ clubs~♣ clubsuit~♣ colon~: colone~≔\
+  coloneq~≔ comma~, commat~@ comp~∁ compfn~∘ complement~∁ complexes~ℂ cong~≅ congdot~⩭ conint~∮ copf~𝕔 coprod~∐\
+  copy~© copysr~℗ crarr~↵ cross~✗ cscr~𝒸 csub~⫏ csube~⫑ csup~⫐ csupe~⫒ ctdot~⋯ cudarrl~⤸ cudarrr~⤵ cuepr~⋞ cuesc~⋟\
+  cularr~↶ cularrp~⤽ cup~∪ cupbrcap~⩈ cupcap~⩆ cupcup~⩊ cupdot~⊍ cupor~⩅ cups~∪︀ curarr~↷ curarrm~⤼ curlyeqprec~⋞\
+  curlyeqsucc~⋟ curlyvee~⋎ curlywedge~⋏ curren~¤ curvearrowleft~↶ curvearrowright~↷ cuvee~⋎ cuwed~⋏ cwconint~∲\
+  cwint~∱ cylcty~⌭ dArr~⇓ dHar~⥥ dagger~† daleth~ℸ darr~↓ dash~‐ dashv~⊣ dbkarow~⤏ dblac~˝ dcaron~ď dcy~д dd~ⅆ\
+  ddagger~‡ ddarr~⇊ ddotseq~⩷ deg~° delta~δ demptyv~⦱ dfisht~⥿ dfr~𝔡 dharl~⇃ dharr~⇂ diam~⋄ diamond~⋄ diamondsuit~♦\
+  diams~♦ die~¨ digamma~ϝ disin~⋲ div~÷ divide~÷ divideontimes~⋇ divonx~⋇ djcy~ђ dlcorn~⌞ dlcrop~⌍ dollar~$ dopf~𝕕\
+  dot~˙ doteq~≐ doteqdot~≑ dotminus~∸ dotplus~∔ dotsquare~⊡ doublebarwedge~⌆ downarrow~↓ downdownarrows~⇊\
+  downharpoonleft~⇃ downharpoonright~⇂ drbkarow~⤐ drcorn~⌟ drcrop~⌌ dscr~𝒹 dscy~ѕ dsol~⧶ dstrok~đ dtdot~⋱ dtri~▿\
+  dtrif~▾ duarr~⇵ duhar~⥯ dwangle~⦦ dzcy~џ dzigrarr~⟿ eDDot~⩷ eDot~≑ eacute~é easter~⩮ ecaron~ě ecir~≖ ecirc~ê\
+  ecolon~≕ ecy~э edot~ė ee~ⅇ efDot~≒ efr~𝔢 eg~⪚ egrave~è egs~⪖ egsdot~⪘ el~⪙ elinters~⏧ ell~ℓ els~⪕ elsdot~⪗\
+  emacr~ē empty~∅ emptyset~∅ emptyv~∅ emsp~  emsp13~  emsp14~  eng~ŋ ensp~  eogon~ę eopf~𝕖 epar~⋕ eparsl~⧣ eplus~⩱\
+  epsi~ε epsilon~ε epsiv~ϵ eqcirc~≖ eqcolon~≕ eqsim~≂ eqslantgtr~⪖ eqslantless~⪕ equals~= equest~≟ equiv~≡ equivDD~⩸\
+  eqvparsl~⧥ erDot~≓ erarr~⥱ escr~ℯ esdot~≐ esim~≂ eta~η eth~ð euml~ë euro~€ excl~! exist~∃ expectation~ℰ\
+  exponentiale~ⅇ fallingdotseq~≒ fcy~ф female~♀ ffilig~ffi fflig~ff ffllig~ffl ffr~𝔣 filig~fi fjlig~fj flat~♭ fllig~fl\
+  fltns~▱ fnof~ƒ fopf~𝕗 forall~∀ fork~⋔ forkv~⫙ fpartint~⨍ frac12~½ frac13~⅓ frac14~¼ frac15~⅕ frac16~⅙ frac18~⅛\
+  frac23~⅔ frac25~⅖ frac34~¾ frac35~⅗ frac38~⅜ frac45~⅘ frac56~⅚ frac58~⅝ frac78~⅞ frasl~⁄ frown~⌢ fscr~𝒻 gE~≧\
+  gEl~⪌ gacute~ǵ gamma~γ gammad~ϝ gap~⪆ gbreve~ğ gcirc~ĝ gcy~г gdot~ġ ge~≥ gel~⋛ geq~≥ geqq~≧ geqslant~⩾ ges~⩾\
+  gescc~⪩ gesdot~⪀ gesdoto~⪂ gesdotol~⪄ gesl~⋛︀ gesles~⪔ gfr~𝔤 gg~≫ ggg~⋙ gimel~ℷ gjcy~ѓ gl~≷ glE~⪒ gla~⪥ glj~⪤\
+  gnE~≩ gnap~⪊ gnapprox~⪊ gne~⪈ gneq~⪈ gneqq~≩ gnsim~⋧ gopf~𝕘 grave~` gscr~ℊ gsim~≳ gsime~⪎ gsiml~⪐ gt~> gtcc~⪧\
+  gtcir~⩺ gtdot~⋗ gtlPar~⦕ gtquest~⩼ gtrapprox~⪆ gtrarr~⥸ gtrdot~⋗ gtreqless~⋛ gtreqqless~⪌ gtrless~≷ gtrsim~≳\
+  gvertneqq~≩︀ gvnE~≩︀ hArr~⇔ hairsp~  half~½ hamilt~ℋ hardcy~ъ harr~↔ harrcir~⥈ harrw~↭ hbar~ℏ hcirc~ĥ hearts~♥\
+  heartsuit~♥ hellip~… hercon~⊹ hfr~𝔥 hksearow~⤥ hkswarow~⤦ hoarr~⇿ homtht~∻ hookleftarrow~↩ hookrightarrow~↪\
+  hopf~𝕙 horbar~― hscr~𝒽 hslash~ℏ hstrok~ħ hybull~⁃ hyphen~‐ iacute~í ic~⁣ icirc~î icy~и iecy~е iexcl~¡ iff~⇔\
+  ifr~𝔦 igrave~ì ii~ⅈ iiiint~⨌ iiint~∭ iinfin~⧜ iiota~℩ ijlig~ij imacr~ī image~ℑ imagline~ℐ imagpart~ℑ imath~ı\
+  imof~⊷ imped~Ƶ in~∈ incare~℅ infin~∞ infintie~⧝ inodot~ı int~∫ intcal~⊺ integers~ℤ intercal~⊺ intlarhk~⨗ intprod~⨼\
+  iocy~ё iogon~į iopf~𝕚 iota~ι iprod~⨼ iquest~¿ iscr~𝒾 isin~∈ isinE~⋹ isindot~⋵ isins~⋴ isinsv~⋳ isinv~∈ it~⁢\
+  itilde~ĩ iukcy~і iuml~ï jcirc~ĵ jcy~й jfr~𝔧 jmath~ȷ jopf~𝕛 jscr~𝒿 jsercy~ј jukcy~є kappa~κ kappav~ϰ kcedil~ķ\
+  kcy~к kfr~𝔨 kgreen~ĸ khcy~х kjcy~ќ kopf~𝕜 kscr~𝓀 lAarr~⇚ lArr~⇐ lAtail~⤛ lBarr~⤎ lE~≦ lEg~⪋ lHar~⥢ lacute~ĺ\
+  laemptyv~⦴ lagran~ℒ lambda~λ lang~⟨ langd~⦑ langle~⟨ lap~⪅ laquo~« larr~← larrb~⇤ larrbfs~⤟ larrfs~⤝ larrhk~↩\
+  larrlp~↫ larrpl~⤹ larrsim~⥳ larrtl~↢ lat~⪫ latail~⤙ late~⪭ lates~⪭︀ lbarr~⤌ lbbrk~❲ lbrace~{ lbrack~[ lbrke~⦋\
+  lbrksld~⦏ lbrkslu~⦍ lcaron~ľ lcedil~ļ lceil~⌈ lcub~{ lcy~л ldca~⤶ ldquo~“ ldquor~„ ldrdhar~⥧ ldrushar~⥋ ldsh~↲\
+  le~≤ leftarrow~← leftarrowtail~↢ leftharpoondown~↽ leftharpoonup~↼ leftleftarrows~⇇ leftrightarrow~↔\
+  leftrightarrows~⇆ leftrightharpoons~⇋ leftrightsquigarrow~↭ leftthreetimes~⋋ leg~⋚ leq~≤ leqq~≦ leqslant~⩽ les~⩽\
+  lescc~⪨ lesdot~⩿ lesdoto~⪁ lesdotor~⪃ lesg~⋚︀ lesges~⪓ lessapprox~⪅ lessdot~⋖ lesseqgtr~⋚ lesseqqgtr~⪋ lessgtr~≶\
+  lesssim~≲ lfisht~⥼ lfloor~⌊ lfr~𝔩 lg~≶ lgE~⪑ lhard~↽ lharu~↼ lharul~⥪ lhblk~▄ ljcy~љ ll~≪ llarr~⇇ llcorner~⌞\
+  llhard~⥫ lltri~◺ lmidot~ŀ lmoust~⎰ lmoustache~⎰ lnE~≨ lnap~⪉ lnapprox~⪉ lne~⪇ lneq~⪇ lneqq~≨ lnsim~⋦ loang~⟬\
+  loarr~⇽ lobrk~⟦ longleftarrow~⟵ longleftrightarrow~⟷ longmapsto~⟼ longrightarrow~⟶ looparrowleft~↫\
+  looparrowright~↬ lopar~⦅ lopf~𝕝 loplus~⨭ lotimes~⨴ lowast~∗ lowbar~_ loz~◊ lozenge~◊ lozf~⧫ lpar~( lparlt~⦓\
+  lrarr~⇆ lrcorner~⌟ lrhar~⇋ lrhard~⥭ lrm~‎ lrtri~⊿ lsaquo~‹ lscr~𝓁 lsh~↰ lsim~≲ lsime~⪍ lsimg~⪏ lsqb~[ lsquo~‘\
+  lsquor~‚ lstrok~ł lt~< ltcc~⪦ ltcir~⩹ ltdot~⋖ lthree~⋋ ltimes~⋉ ltlarr~⥶ ltquest~⩻ ltrPar~⦖ ltri~◃ ltrie~⊴ ltrif~◂\
+  lurdshar~⥊ luruhar~⥦ lvertneqq~≨︀ lvnE~≨︀ mDDot~∺ macr~¯ male~♂ malt~✠ maltese~✠ map~↦ mapsto~↦ mapstodown~↧\
+  mapstoleft~↤ mapstoup~↥ marker~▮ mcomma~⨩ mcy~м mdash~— measuredangle~∡ mfr~𝔪 mho~℧ micro~µ mid~∣ midast~*\
+  midcir~⫰ middot~· minus~− minusb~⊟ minusd~∸ minusdu~⨪ mlcp~⫛ mldr~… mnplus~∓ models~⊧ mopf~𝕞 mp~∓ mscr~𝓂\
+  mstpos~∾ mu~μ multimap~⊸ mumap~⊸ nGg~⋙̸ nGt~≫⃒ nGtv~≫̸ nLeftarrow~⇍ nLeftrightarrow~⇎ nLl~⋘̸ nLt~≪⃒ nLtv~≪̸\
+  nRightarrow~⇏ nVDash~⊯ nVdash~⊮ nabla~∇ nacute~ń nang~∠⃒ nap~≉ napE~⩰̸ napid~≋̸ napos~ʼn napprox~≉ natur~♮\
+  natural~♮ naturals~ℕ nbsp~  nbump~≎̸ nbumpe~≏̸ ncap~⩃ ncaron~ň ncedil~ņ ncong~≇ ncongdot~⩭̸ ncup~⩂ ncy~н ndash~–\
+  ne~≠ neArr~⇗ nearhk~⤤ nearr~↗ nearrow~↗ nedot~≐̸ nequiv~≢ nesear~⤨ nesim~≂̸ nexist~∄ nexists~∄ nfr~𝔫 ngE~≧̸ nge~≱\
+  ngeq~≱ ngeqq~≧̸ ngeqslant~⩾̸ nges~⩾̸ ngsim~≵ ngt~≯ ngtr~≯ nhArr~⇎ nharr~↮ nhpar~⫲ ni~∋ nis~⋼ nisd~⋺ niv~∋ njcy~њ\
+  nlArr~⇍ nlE~≦̸ nlarr~↚ nldr~‥ nle~≰ nleftarrow~↚ nleftrightarrow~↮ nleq~≰ nleqq~≦̸ nleqslant~⩽̸ nles~⩽̸ nless~≮\
+  nlsim~≴ nlt~≮ nltri~⋪ nltrie~⋬ nmid~∤ nopf~𝕟 not~¬ notin~∉ notinE~⋹̸ notindot~⋵̸ notinva~∉ notinvb~⋷ notinvc~⋶\
+  notni~∌ notniva~∌ notnivb~⋾ notnivc~⋽ npar~∦ nparallel~∦ nparsl~⫽⃥ npart~∂̸ npolint~⨔ npr~⊀ nprcue~⋠ npre~⪯̸\
+  nprec~⊀ npreceq~⪯̸ nrArr~⇏ nrarr~↛ nrarrc~⤳̸ nrarrw~↝̸ nrightarrow~↛ nrtri~⋫ nrtrie~⋭ nsc~⊁ nsccue~⋡ nsce~⪰̸\
+  nscr~𝓃 nshortmid~∤ nshortparallel~∦ nsim~≁ nsime~≄ nsimeq~≄ nsmid~∤ nspar~∦ nsqsube~⋢ nsqsupe~⋣ nsub~⊄ nsubE~⫅̸\
+  nsube~⊈ nsubset~⊂⃒ nsubseteq~⊈ nsubseteqq~⫅̸ nsucc~⊁ nsucceq~⪰̸ nsup~⊅ nsupE~⫆̸ nsupe~⊉ nsupset~⊃⃒ nsupseteq~⊉\
+  nsupseteqq~⫆̸ ntgl~≹ ntilde~ñ ntlg~≸ ntriangleleft~⋪ ntrianglelefteq~⋬ ntriangleright~⋫ ntrianglerighteq~⋭ nu~ν\
+  num~# numero~№ numsp~  nvDash~⊭ nvHarr~⤄ nvap~≍⃒ nvdash~⊬ nvge~≥⃒ nvgt~>⃒ nvinfin~⧞ nvlArr~⤂ nvle~≤⃒ nvlt~<⃒\
+  nvltrie~⊴⃒ nvrArr~⤃ nvrtrie~⊵⃒ nvsim~∼⃒ nwArr~⇖ nwarhk~⤣ nwarr~↖ nwarrow~↖ nwnear~⤧ oS~Ⓢ oacute~ó oast~⊛ ocir~⊚\
+  ocirc~ô ocy~о odash~⊝ odblac~ő odiv~⨸ odot~⊙ odsold~⦼ oelig~œ ofcir~⦿ ofr~𝔬 ogon~˛ ograve~ò ogt~⧁ ohbar~⦵ ohm~Ω\
+  oint~∮ olarr~↺ olcir~⦾ olcross~⦻ oline~‾ olt~⧀ omacr~ō omega~ω omicron~ο omid~⦶ ominus~⊖ oopf~𝕠 opar~⦷ operp~⦹\
+  oplus~⊕ or~∨ orarr~↻ ord~⩝ order~ℴ orderof~ℴ ordf~ª ordm~º origof~⊶ oror~⩖ orslope~⩗ orv~⩛ oscr~ℴ oslash~ø osol~⊘\
+  otilde~õ otimes~⊗ otimesas~⨶ ouml~ö ovbar~⌽ par~∥ para~¶ parallel~∥ parsim~⫳ parsl~⫽ part~∂ pcy~п percnt~%\
+  period~. permil~‰ perp~⊥ pertenk~‱ pfr~𝔭 phi~φ phiv~ϕ phmmat~ℳ phone~☎ pi~π pitchfork~⋔ piv~ϖ planck~ℏ planckh~ℎ\
+  plankv~ℏ plus~+ plusacir~⨣ plusb~⊞ pluscir~⨢ plusdo~∔ plusdu~⨥ pluse~⩲ plusmn~± plussim~⨦ plustwo~⨧ pm~±\
+  pointint~⨕ popf~𝕡 pound~£ pr~≺ prE~⪳ prap~⪷ prcue~≼ pre~⪯ prec~≺ precapprox~⪷ preccurlyeq~≼ preceq~⪯\
+  precnapprox~⪹ precneqq~⪵ precnsim~⋨ precsim~≾ prime~′ primes~ℙ prnE~⪵ prnap~⪹ prnsim~⋨ prod~∏ profalar~⌮\
+  profline~⌒ profsurf~⌓ prop~∝ propto~∝ prsim~≾ prurel~⊰ pscr~𝓅 psi~ψ puncsp~  qfr~𝔮 qint~⨌ qopf~𝕢 qprime~⁗\
+  qscr~𝓆 quaternions~ℍ quatint~⨖ quest~? questeq~≟ quot~" rAarr~⇛ rArr~⇒ rAtail~⤜ rBarr~⤏ rHar~⥤ race~∽̱ racute~ŕ\
+  radic~√ raemptyv~⦳ rang~⟩ rangd~⦒ range~⦥ rangle~⟩ raquo~» rarr~→ rarrap~⥵ rarrb~⇥ rarrbfs~⤠ rarrc~⤳ rarrfs~⤞\
+  rarrhk~↪ rarrlp~↬ rarrpl~⥅ rarrsim~⥴ rarrtl~↣ rarrw~↝ ratail~⤚ ratio~∶ rationals~ℚ rbarr~⤍ rbbrk~❳ rbrace~}\
+  rbrack~] rbrke~⦌ rbrksld~⦎ rbrkslu~⦐ rcaron~ř rcedil~ŗ rceil~⌉ rcub~} rcy~р rdca~⤷ rdldhar~⥩ rdquo~” rdquor~”\
+  rdsh~↳ real~ℜ realine~ℛ realpart~ℜ reals~ℝ rect~▭ reg~® rfisht~⥽ rfloor~⌋ rfr~𝔯 rhard~⇁ rharu~⇀ rharul~⥬ rho~ρ\
+  rhov~ϱ rightarrow~→ rightarrowtail~↣ rightharpoondown~⇁ rightharpoonup~⇀ rightleftarrows~⇄ rightleftharpoons~⇌\
+  rightrightarrows~⇉ rightsquigarrow~↝ rightthreetimes~⋌ ring~˚ risingdotseq~≓ rlarr~⇄ rlhar~⇌ rlm~‏ rmoust~⎱\
+  rmoustache~⎱ rnmid~⫮ roang~⟭ roarr~⇾ robrk~⟧ ropar~⦆ ropf~𝕣 roplus~⨮ rotimes~⨵ rpar~) rpargt~⦔ rppolint~⨒ rrarr~⇉\
+  rsaquo~› rscr~𝓇 rsh~↱ rsqb~] rsquo~’ rsquor~’ rthree~⋌ rtimes~⋊ rtri~▹ rtrie~⊵ rtrif~▸ rtriltri~⧎ ruluhar~⥨ rx~℞\
+  sacute~ś sbquo~‚ sc~≻ scE~⪴ scap~⪸ scaron~š sccue~≽ sce~⪰ scedil~ş scirc~ŝ scnE~⪶ scnap~⪺ scnsim~⋩ scpolint~⨓\
+  scsim~≿ scy~с sdot~⋅ sdotb~⊡ sdote~⩦ seArr~⇘ searhk~⤥ searr~↘ searrow~↘ sect~§ semi~; seswar~⤩ setminus~∖ setmn~∖\
+  sext~✶ sfr~𝔰 sfrown~⌢ sharp~♯ shchcy~щ shcy~ш shortmid~∣ shortparallel~∥ shy~­ sigma~σ sigmaf~ς sigmav~ς sim~∼\
+  simdot~⩪ sime~≃ simeq~≃ simg~⪞ simgE~⪠ siml~⪝ simlE~⪟ simne~≆ simplus~⨤ simrarr~⥲ slarr~← smallsetminus~∖ smashp~⨳\
+  smeparsl~⧤ smid~∣ smile~⌣ smt~⪪ smte~⪬ smtes~⪬︀ softcy~ь sol~/ solb~⧄ solbar~⌿ sopf~𝕤 spades~♠ spadesuit~♠ spar~∥\
+  sqcap~⊓ sqcaps~⊓︀ sqcup~⊔ sqcups~⊔︀ sqsub~⊏ sqsube~⊑ sqsubset~⊏ sqsubseteq~⊑ sqsup~⊐ sqsupe~⊒ sqsupset~⊐\
+  sqsupseteq~⊒ squ~□ square~□ squarf~▪ squf~▪ srarr~→ sscr~𝓈 ssetmn~∖ ssmile~⌣ sstarf~⋆ star~☆ starf~★\
+  straightepsilon~ϵ straightphi~ϕ strns~¯ sub~⊂ subE~⫅ subdot~⪽ sube~⊆ subedot~⫃ submult~⫁ subnE~⫋ subne~⊊ subplus~⪿\
+  subrarr~⥹ subset~⊂ subseteq~⊆ subseteqq~⫅ subsetneq~⊊ subsetneqq~⫋ subsim~⫇ subsub~⫕ subsup~⫓ succ~≻ succapprox~⪸\
+  succcurlyeq~≽ succeq~⪰ succnapprox~⪺ succneqq~⪶ succnsim~⋩ succsim~≿ sum~∑ sung~♪ sup~⊃ sup1~¹ sup2~² sup3~³\
+  supE~⫆ supdot~⪾ supdsub~⫘ supe~⊇ supedot~⫄ suphsol~⟉ suphsub~⫗ suplarr~⥻ supmult~⫂ supnE~⫌ supne~⊋ supplus~⫀\
+  supset~⊃ supseteq~⊇ supseteqq~⫆ supsetneq~⊋ supsetneqq~⫌ supsim~⫈ supsub~⫔ supsup~⫖ swArr~⇙ swarhk~⤦ swarr~↙\
+  swarrow~↙ swnwar~⤪ szlig~ß target~⌖ tau~τ tbrk~⎴ tcaron~ť tcedil~ţ tcy~т tdot~⃛ telrec~⌕ tfr~𝔱 there4~∴\
+  therefore~∴ theta~θ thetasym~ϑ thetav~ϑ thickapprox~≈ thicksim~∼ thinsp~  thkap~≈ thksim~∼ thorn~þ tilde~˜ times~×\
+  timesb~⊠ timesbar~⨱ timesd~⨰ tint~∭ toea~⤨ top~⊤ topbot~⌶ topcir~⫱ topf~𝕥 topfork~⫚ tosa~⤩ tprime~‴ trade~™\
+  triangle~▵ triangledown~▿ triangleleft~◃ trianglelefteq~⊴ triangleq~≜ triangleright~▹ trianglerighteq~⊵ tridot~◬\
+  trie~≜ triminus~⨺ triplus~⨹ trisb~⧍ tritime~⨻ trpezium~⏢ tscr~𝓉 tscy~ц tshcy~ћ tstrok~ŧ twixt~≬\
+  twoheadleftarrow~↞ twoheadrightarrow~↠ uArr~⇑ uHar~⥣ uacute~ú uarr~↑ ubrcy~ў ubreve~ŭ ucirc~û ucy~у udarr~⇅\
+  udblac~ű udhar~⥮ ufisht~⥾ ufr~𝔲 ugrave~ù uharl~↿ uharr~↾ uhblk~▀ ulcorn~⌜ ulcorner~⌜ ulcrop~⌏ ultri~◸ umacr~ū\
+  uml~¨ uogon~ų uopf~𝕦 uparrow~↑ updownarrow~↕ upharpoonleft~↿ upharpoonright~↾ uplus~⊎ upsi~υ upsih~ϒ upsilon~υ\
+  upuparrows~⇈ urcorn~⌝ urcorner~⌝ urcrop~⌎ uring~ů urtri~◹ uscr~𝓊 utdot~⋰ utilde~ũ utri~▵ utrif~▴ uuarr~⇈ uuml~ü\
+  uwangle~⦧ vArr~⇕ vBar~⫨ vBarv~⫩ vDash~⊨ vangrt~⦜ varepsilon~ϵ varkappa~ϰ varnothing~∅ varphi~ϕ varpi~ϖ varpropto~∝\
+  varr~↕ varrho~ϱ varsigma~ς varsubsetneq~⊊︀ varsubsetneqq~⫋︀ varsupsetneq~⊋︀ varsupsetneqq~⫌︀ vartheta~ϑ\
+  vartriangleleft~⊲ vartriangleright~⊳ vcy~в vdash~⊢ vee~∨ veebar~⊻ veeeq~≚ vellip~⋮ verbar~| vert~| vfr~𝔳 vltri~⊲\
+  vnsub~⊂⃒ vnsup~⊃⃒ vopf~𝕧 vprop~∝ vrtri~⊳ vscr~𝓋 vsubnE~⫋︀ vsubne~⊊︀ vsupnE~⫌︀ vsupne~⊋︀ vzigzag~⦚ wcirc~ŵ\
+  wedbar~⩟ wedge~∧ wedgeq~≙ weierp~℘ wfr~𝔴 wopf~𝕨 wp~℘ wr~≀ wreath~≀ wscr~𝓌 xcap~⋂ xcirc~◯ xcup~⋃ xdtri~▽ xfr~𝔵\
+  xhArr~⟺ xharr~⟷ xi~ξ xlArr~⟸ xlarr~⟵ xmap~⟼ xnis~⋻ xodot~⨀ xopf~𝕩 xoplus~⨁ xotime~⨂ xrArr~⟹ xrarr~⟶ xscr~𝓍\
+  xsqcup~⨆ xuplus~⨄ xutri~△ xvee~⋁ xwedge~⋀ yacute~ý yacy~я ycirc~ŷ ycy~ы yen~¥ yfr~𝔶 yicy~ї yopf~𝕪 yscr~𝓎 yucy~ю\
+  yuml~ÿ zacute~ź zcaron~ž zcy~з zdot~ż zeetrf~ℨ zeta~ζ zfr~𝔷 zhcy~ж zigrarr~⇝ zopf~𝕫 zscr~𝓏 zwj~‍ zwnj~‌'
+
+const namedReferences = new Map(packedReferences.split(/ +/).map(namedReference))
+
+export function holdsEntityReference(text: string): boolean {
+  for (let index = text.indexOf('&'); index !== -1; index = text.indexOf('&', index + 1)) {
+    if (readEntityReference(text, index) !== undefined) return true
+  }
+  return false
+}
+
+// The reference standing at `index`, `undefined` where none does.
+export function readEntityReference(text: string, index: number): EntityReference | undefined {
+  anchoredEntityReference.lastIndex = index
+  const reference = anchoredEntityReference.exec(text)?.[0]
+  if (reference === undefined) return undefined
+  const decoded = decodeReference(reference)
+  return decoded === undefined ? undefined : { length: reference.length, text: decoded }
+}
+
+function decodeReference(reference: string): string | undefined {
+  const decimal = decimalReference.exec(reference)?.[1]
+  if (decimal !== undefined) return characterOf(Number.parseInt(decimal, 10))
+  const hexadecimal = hexadecimalReference.exec(reference)?.[1]
+  if (hexadecimal !== undefined) return characterOf(Number.parseInt(hexadecimal, 16))
+  return namedReferences.get(reference.slice(1, -1))
+}
+
+function characterOf(codePoint: number): string {
+  if (codePoint === 0 || codePoint > largestCodePoint) return replacementCharacter
+  if (codePoint >= surrogates.first && codePoint <= surrogates.last) return replacementCharacter
+  return String.fromCodePoint(codePoint)
+}
+
+function namedReference(entry: string): [string, string] {
+  const separator = entry.indexOf('~')
+  return [entry.slice(0, separator), entry.slice(separator + 1)]
+}

src/markdown/parse/blocks.ts

@@ -4,13 +4,14 @@ import {
   claimsDirectiveLine,
   claimsPipeLine,
   closingCodeFence,
+  decodeTextEscapes,
   isThematicBreak,
   listMarker,
   markerInterruptsParagraph,
   openingCodeFence,
+  openingHtmlBlock,
   setextHeadingLevel,
 } from '../commonmark-grammar.ts'
-import { openingHtmlBlock } from './html-blocks.ts'
 import { readLinkDefinitions } from './link-reference-definitions.ts'
 
 export type ClaimedConstruct = 'directive' | 'pipe-table'
@@ -177,7 +178,7 @@ function continuesLazily(walk: Walk, line: Line): boolean {
   if (leadingColumns(line) >= indentedCodeColumns) return true
   const opener = removeColumns(line, largestOpenerIndentation).text
   if (claimedConstruct(opener) !== undefined || isThematicBreak(opener)) return false
-  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, false) === undefined
+  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, true) === undefined
 }
 
 function readBlockLine(walk: Walk, line: Line): void {
@@ -281,7 +282,7 @@ function closeLeaf(walk: Walk): void {
   }
   walk.leaf = undefined
   if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html' })
-  else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? leaf.info : '', text: leaf.lines.join('\n') })
+  else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? decodeTextEscapes(leaf.info) : '', text: leaf.lines.join('\n') })
 }
 
 function takeParagraph(walk: Walk): string | undefined {

src/markdown/parse/html-blocks.ts

@@ -1,28 +0,0 @@
-export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
-
-type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
-
-// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
-const blockTagNames =
-  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
-const attributeSource = '(?:[ \\t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \\t]*=[ \\t]*(?:[^ \\t"\'=<>`]+|\'[^\']*\'|"[^"]*"))?)'
-const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \\t]*/?>|</[A-Za-z][A-Za-z0-9-]*[ \\t]*>)[ \\t]*$`)
-const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/
-
-const conditions: HtmlBlockCondition[] = [
-  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
-  { closer: /-->/, construct: 'an HTML comment', interrupts: true, start: /^<!--/ },
-  { closer: /\?>/, construct: 'an HTML processing instruction', interrupts: true, start: /^<\?/ },
-  { closer: />/, construct: 'an HTML declaration', interrupts: true, start: /^<![A-Za-z]/ },
-  { closer: /\]\]>/, construct: 'a CDATA section', interrupts: true, start: /^<!\[CDATA\[/ },
-  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
-  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
-]
-
-export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
-  for (const condition of conditions) {
-    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
-    return { closer: condition.closer, construct: condition.construct ?? line.replace(tagName, '<$1>') }
-  }
-  return undefined
-}

src/markdown/parse/inline-content.ts

@@ -0,0 +1,80 @@
+import type { AdfNode } from '../../adf/document.ts'
+import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts'
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
+import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
+
+type Run = { nodes: AdfNode[]; text: string; undecodedFrom: number }
+
+const hardBreakSpaces = / {2,}$/
+const trailingSpace = /[ \t]+$/
+
+export function parseInlineContent(source: string, path: ConvertErrorPath): Result<AdfNode[]> {
+  const run: Run = { nodes: [], text: '', undecodedFrom: 0 }
+  let index = 0
+  while (index < source.length) {
+    const character = source.charAt(index)
+    if (character === '\\' && source.charAt(index + 1) === '\n') {
+      // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash.
+      takeRun(run, source, index, index + 2, false)
+      pushNode(run, { type: 'hardBreak' })
+      index += 2
+      continue
+    }
+    if (character === '\n') {
+      const hard = hardBreakSpaces.test(source.slice(run.undecodedFrom, index))
+      takeRun(run, source, index, index + 1, true)
+      if (hard) pushNode(run, { type: 'hardBreak' })
+      else run.text += ' '
+      index += 1
+      continue
+    }
+    if (character === '`') {
+      const span = readCodeSpan(source, index)
+      if (span === undefined) {
+        index += backtickRun(source, index)
+        continue
+      }
+      takeRun(run, source, index, span.end, false)
+      pushNode(run, { marks: [{ type: 'code' }], text: span.text, type: 'text' })
+      index = span.end
+      continue
+    }
+    if (character === '<') {
+      const construct = inlineHtmlConstruct(source, index)
+      if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, path)
+    }
+    index += backslashEscape(source, index) === undefined ? 1 : 2
+  }
+  takeRun(run, source, source.length, source.length, true)
+  pushText(run)
+  return success(run.nodes)
+}
+
+function takeRun(run: Run, source: string, end: number, resume: number, strip: boolean): void {
+  const raw = source.slice(run.undecodedFrom, end)
+  run.text += decodeTextEscapes(strip ? raw.replace(trailingSpace, '') : raw)
+  run.undecodedFrom = resume
+}
+
+function pushText(run: Run): void {
+  if (run.text !== '') run.nodes.push({ text: run.text, type: 'text' })
+  run.text = ''
+}
+
+function pushNode(run: Run, node: AdfNode): void {
+  pushText(run)
+  run.nodes.push(node)
+}
+
+function readCodeSpan(source: string, index: number): { end: number; text: string } | undefined {
+  const opener = backtickRun(source, index)
+  const closer = closingBacktickRun(source, index + opener, opener)
+  if (closer === undefined) return undefined
+  return { end: closer + opener, text: codeSpanText(source.slice(index + opener, closer)) }
+}
+
+function codeSpanText(content: string): string {
+  const text = content.replaceAll('\n', ' ')
+  const padded = text.startsWith(' ') && text.endsWith(' ') && /[^ ]/.test(text)
+  return padded ? text.slice(1, -1) : text
+}

src/markdown/parse/markdown-to-adf.test.ts

@@ -25,6 +25,14 @@ function paragraph(value: string): AdfNode {
   return { content: [text(value)], type: 'paragraph' }
 }
 
+function codeSpan(value: string): AdfNode {
+  return { marks: [{ type: 'code' }], text: value, type: 'text' }
+}
+
+function hardBreak(): AdfNode {
+  return { type: 'hardBreak' }
+}
+
 function item(...content: AdfNode[]): AdfNode {
   return content.length === 0 ? { type: 'listItem' } : { content, type: 'listItem' }
 }
@@ -112,7 +120,7 @@ test('claims a block-level colon run with no directive to parse it', () => {
 
 test('claims a block-level pipe with no table to parse it', () => {
   assert.equal(code(markdownToAdf('| Part | Qty |\n')), 'malformed-pipe-table')
-  assert.deepEqual(content(markdownToAdf('\\| Part\n')), [paragraph('\\| Part')])
+  assert.deepEqual(content(markdownToAdf('\\| Part\n')), [paragraph('| Part')])
 })
 
 test('refuses the raw HTML no element mapping carries', () => {
@@ -123,6 +131,8 @@ test('refuses the raw HTML no element mapping carries', () => {
   assert.equal(code(markdownToAdf('<![CDATA[x]]>\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<pre>\nx\n</pre>\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<div\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<?php\n')), 'unmappable-html')
   assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1])
   assert.equal(code(markdownToAdf('<div>\nx\n\n:::\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<div>\n- x\n</div>\n')), 'unmappable-html')
@@ -133,11 +143,6 @@ test('swallows an HTML block ahead of the claim a line inside it would make', ()
   assert.equal(code(markdownToAdf('<div>\n| x |\n</div>\n')), 'unmappable-html')
 })
 
-test('leaves a tag that opens no HTML block to the paragraph it sits in', () => {
-  assert.deepEqual(content(markdownToAdf('Part.\n<span>\n')), [paragraph('Part. <span>')])
-  assert.deepEqual(content(markdownToAdf('3 < 4\n')), [paragraph('3 < 4')])
-})
-
 test('gives up the link reference definitions a paragraph opens with', () => {
   assert.deepEqual(content(markdownToAdf('[a]: /url\n')), [])
   assert.deepEqual(content(markdownToAdf('[a]: /url\n[b]: /other\nPart.\n')), [paragraph('Part.')])
@@ -222,6 +227,8 @@ test('folds a lazy continuation into the paragraph the container holds', () => {
   assert.deepEqual(content(markdownToAdf('> One\n---\n')), [quote(paragraph('One')), { type: 'rule' }])
   assert.deepEqual(content(markdownToAdf('> One\n```\n')), [quote(paragraph('One')), { type: 'codeBlock' }])
   assert.equal(code(markdownToAdf('> One\n<div>\n')), 'unmappable-html')
+  assert.deepEqual(path(markdownToAdf('> One\n<div>\n')), ['content', 1])
+  assert.deepEqual(path(markdownToAdf('> One\n<span>\n')), ['content', 0, 'content', 0])
 })
 
 test('ends a lazy continuation at a claimed line', () => {
@@ -239,3 +246,80 @@ test('refuses input nested deeper than the parser carries', () => {
   assert.equal(code(markdownToAdf('> '.repeat(501))), 'unsupported-nesting-depth')
   assert.ok(markdownToAdf('> '.repeat(500)).ok)
 })
+
+test('decodes the backslash escapes CommonMark spells, and keeps the rest literal', () => {
+  assert.deepEqual(content(markdownToAdf('\\*not emphasis\\*\n')), [paragraph('*not emphasis*')])
+  assert.deepEqual(content(markdownToAdf('\\\\\n')), [paragraph('\\')])
+  assert.deepEqual(content(markdownToAdf('\\a \\\u00a0\n')), [paragraph('\\a \\\u00a0')])
+  assert.deepEqual(content(markdownToAdf('Part\\\n')), [paragraph('Part\\')])
+  assert.deepEqual(content(markdownToAdf('a\\`b`\n')), [paragraph('a`b`')])
+})
+
+test('decodes the entity references HTML5 names, and the numeric ones', () => {
+  assert.deepEqual(content(markdownToAdf('&amp; &copy; &ngE; &zwnj; &AElig;\n')), [paragraph('& \u00a9 \u2267\u0338 \u200c \u00c6')])
+  assert.deepEqual(content(markdownToAdf('&#35; &#X22; &#x2665;\n')), [paragraph('# " \u2665')])
+  assert.deepEqual(content(markdownToAdf('&#0; &#xd800; &#9999999;\n')), [paragraph('\ufffd \ufffd \ufffd')])
+  assert.deepEqual(content(markdownToAdf('&zzz; &amp &#; &\n')), [paragraph('&zzz; &amp &#; &')])
+  assert.deepEqual(content(markdownToAdf('&#96;not code&#96;\n')), [paragraph('`not code`')])
+  assert.deepEqual(content(markdownToAdf('a&Tab;b&NewLine;c&nbsp;d&Aopf;e&verbar;f\n')), [paragraph('a\tb\nc d\u{1d538}e|f')])
+})
+
+test('reads a code span, its content literal', () => {
+  assert.deepEqual(content(markdownToAdf('Run `npm test` now.\n')), [
+    { content: [text('Run '), codeSpan('npm test'), text(' now.')], type: 'paragraph' },
+  ])
+  assert.deepEqual(content(markdownToAdf('``a`b``\n')), [{ content: [codeSpan('a`b')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('` `` `\n')), [{ content: [codeSpan('``')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`  `\n')), [{ content: [codeSpan('  ')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`a\nb`\n')), [{ content: [codeSpan('a b')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`foo``bar`\n')), [{ content: [codeSpan('foo``bar')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('`:::panel` `~~x~~` `\\*` `&amp;`\n')), [
+    {
+      content: [codeSpan(':::panel'), text(' '), codeSpan('~~x~~'), text(' '), codeSpan('\\*'), text(' '), codeSpan('&amp;')],
+      type: 'paragraph',
+    },
+  ])
+  assert.deepEqual(content(markdownToAdf('`foo\n')), [paragraph('`foo')])
+  assert.deepEqual(content(markdownToAdf('``foo`\n')), [paragraph('``foo`')])
+})
+
+test('reads a hard break from a trailing backslash and from two trailing spaces alike', () => {
+  assert.deepEqual(content(markdownToAdf('One\\\ntwo.\n')), [{ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One  \ntwo.\n')), [{ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One\\  \ntwo.\n')), [{ content: [text('One\\'), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One \\\ntwo.\n')), [{ content: [text('One '), hardBreak(), text('two.')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('One \ntwo.\n')), [paragraph('One two.')])
+  assert.deepEqual(content(markdownToAdf('One \t\ntwo.\n')), [paragraph('One two.')])
+  assert.deepEqual(content(markdownToAdf('One  \n')), [paragraph('One')])
+  assert.deepEqual(content(markdownToAdf('> One\\\n> two.\n')), [quote({ content: [text('One'), hardBreak(), text('two.')], type: 'paragraph' })])
+})
+
+test('decodes the fenced info string the block walk leaves raw', () => {
+  assert.deepEqual(content(markdownToAdf('```java&#8203;script\nx\n```\n')), [
+    { attrs: { language: 'java\u200bscript' }, content: [text('x')], type: 'codeBlock' },
+  ])
+  assert.deepEqual(content(markdownToAdf('```\\#c\nx\n```\n')), [{ attrs: { language: '#c' }, content: [text('x')], type: 'codeBlock' }])
+})
+
+test('refuses the raw inline HTML no element mapping carries, naming it', () => {
+  assert.equal(content(markdownToAdf('Part <span> here.\n')), 'unmappable-html: no ADF node carries <span>')
+  assert.equal(content(markdownToAdf('Part </div> here.\n')), 'unmappable-html: no ADF node carries <div>')
+  assert.equal(content(markdownToAdf('Part <!-- note --> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(content(markdownToAdf('Part <?php ?> here.\n')), 'unmappable-html: no ADF node carries an HTML processing instruction')
+  assert.equal(content(markdownToAdf('Part <!DOCTYPE html> here.\n')), 'unmappable-html: no ADF node carries an HTML declaration')
+  assert.equal(content(markdownToAdf('Part <![CDATA[x]]> here.\n')), 'unmappable-html: no ADF node carries a CDATA section')
+  assert.equal(content(markdownToAdf('Part <!--> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(content(markdownToAdf('Part <!---> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(code(markdownToAdf('A <a href="/x" disabled\nid=y> b\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('Part.\n<span>\n')), 'unmappable-html')
+  assert.deepEqual(path(markdownToAdf('Part.\n\nA <b>b</b>.\n')), ['content', 1])
+})
+
+test('leaves the angle bracket that opens no HTML construct to the text it sits in', () => {
+  assert.deepEqual(content(markdownToAdf('3 < 4 and 5 <b 6\n')), [paragraph('3 < 4 and 5 <b 6')])
+  assert.deepEqual(content(markdownToAdf('a <b"c> d\n')), [paragraph('a <b"c> d')])
+  assert.deepEqual(content(markdownToAdf('a <!-- b\n')), [paragraph('a <!-- b')])
+  assert.deepEqual(content(markdownToAdf('a </b c> d\n')), [paragraph('a </b c> d')])
+  assert.deepEqual(content(markdownToAdf('`<span>`\n')), [{ content: [codeSpan('<span>')], type: 'paragraph' }])
+  assert.deepEqual(content(markdownToAdf('\\<span>\n')), [paragraph('<span>')])
+})

src/markdown/parse/markdown-to-adf.ts

@@ -3,7 +3,7 @@ import type { Block, ClaimedConstruct } from './blocks.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { largestNesting } from '../../nesting.ts'
 import { parseBlocks } from './blocks.ts'
-import { trimSpace } from '../commonmark-grammar.ts'
+import { parseInlineContent } from './inline-content.ts'
 
 export function markdownToAdf(markdown: string): Result<AdfDocument> {
   const content = blockNodes(parseBlocks(markdown).blocks, [], 0)
@@ -33,13 +33,13 @@ function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result<
     case 'code':
       return success(codeBlockNode(block.language, block.text))
     case 'heading':
-      return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text))
+      return contentNode({ attrs: { level: block.level }, type: 'heading' }, block.text, path)
     case 'html':
       return failure('unmappable-html', `no ADF node carries ${block.construct}`, path)
     case 'orderedList':
       return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth)
     case 'paragraph':
-      return success(withContent({ type: 'paragraph' }, block.text))
+      return contentNode({ type: 'paragraph' }, block.text, path)
     case 'rule':
       return success({ type: 'rule' })
   }
@@ -75,15 +75,8 @@ function codeBlockNode(language: string, text: string): AdfNode {
   return text === '' ? node : { ...node, content: [{ text, type: 'text' }] }
 }
 
-function withContent(node: AdfNode, text: string): AdfNode {
-  const content = inlineContent(text)
-  return content.length === 0 ? node : { ...node, content }
-}
-
-function inlineContent(text: string): AdfNode[] {
-  const line = text
-    .split('\n')
-    .map((part) => trimSpace(part))
-    .join(' ')
-  return line === '' ? [] : [{ text: line, type: 'text' }]
+function contentNode(node: AdfNode, text: string, path: ConvertErrorPath): Result<AdfNode> {
+  const content = parseInlineContent(text, path)
+  if (!content.ok) return content
+  return success(content.value.length === 0 ? node : { ...node, content: content.value })
 }