Give CommonMark's emphasis section one home, and split the collision property

src/corpus.test.ts

@@ -66,22 +66,33 @@ for (const directory of emittingDirectories) {
   }
 }
 
+function roundTripFixtures(): { name: string; path: string }[] {
+  return emittingDirectories.flatMap((directory) =>
+    fixtureNames(directory, '.json').map((name) => ({ name: `${directory}/${name}`, path: join(roundTripRoot, directory, `${name}.json`) })),
+  )
+}
+
 // One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it.
-test('no two round-trip documents share one spelling', () => {
-  const documents = new Map<string, string>()
+test('no two round-trip documents share one markdown spelling', () => {
   const spellings = new Map<string, string>()
-  for (const directory of emittingDirectories) {
-    for (const name of fixtureNames(directory, '.md')) {
-      const fixture = `${directory}/${name}`
-      const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
-      assert.ok(isJsonValue(parsed))
-      const document = serializeCanonicalJson(parsed, 'compact')
-      const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8')
-      assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`)
-      assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`)
-      documents.set(document, fixture)
-      spellings.set(markdown, fixture)
-    }
+  for (const fixture of roundTripFixtures()) {
+    const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8'))
+    assert.ok(isAdfDocument(parsed), `${fixture.name} is not an ADF document`)
+    const result = adfToMarkdown(parsed)
+    assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
+    assert.equal(spellings.get(result.value), undefined, `${fixture.name} and ${spellings.get(result.value)} share one markdown spelling`)
+    spellings.set(result.value, fixture.name)
+  }
+})
+
+test('no round-trip fixture repeats the document another holds', () => {
+  const documents = new Map<string, string>()
+  for (const fixture of roundTripFixtures()) {
+    const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8'))
+    assert.ok(isJsonValue(parsed))
+    const document = serializeCanonicalJson(parsed, 'compact')
+    assert.equal(documents.get(document), undefined, `${fixture.name} repeats the document ${documents.get(document)} holds`)
+    documents.set(document, fixture.name)
   }
 })
 

src/emphasis-matching.ts

@@ -1,40 +1,37 @@
-export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number }
+import { isUnicodeWhitespace } from './commonmark-grammar.ts'
 
-export type EmphasisRun = {
-  canClose: boolean
-  canOpen: boolean
-  character: string
-  delimiters: readonly EmphasisDelimiter[]
-  end: number
-  start: number
-}
+export type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number }
 
-type Candidate = {
+export type EmphasisPairing<Run> = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number }
+
+type Candidate<Run> = {
   head: number
-  next: Candidate | undefined
+  next: Candidate<Run> | undefined
   original: number
-  previous: Candidate | undefined
+  previous: Candidate<Run> | undefined
   remaining: number
-  run: EmphasisRun
+  run: Run
   tail: number
 }
 
-// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document.
-export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined {
-  const matched = matchDelimiters(runs)
-  // The last opener left unpaired is the innermost: the smallest carry that changes the line.
-  let innermost: number | undefined
-  for (const run of runs) {
-    for (const delimiter of run.delimiters) {
-      if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair
-    }
-  }
-  return innermost
+const unicodePunctuation = /[\p{P}\p{S}]/u
+
+// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too.
+export function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } {
+  const left = isLeftFlanking(before, after)
+  const right = isRightFlanking(before, after)
+  if (character !== '_') return { canClose: right, canOpen: left }
+  return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) }
 }
 
-function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet<number> {
-  const matched = new Set<number>()
-  const bottoms = new Map<string, Candidate | undefined>()
+export function isWordCharacter(character: string): boolean {
+  return character !== '' && !isWhitespace(character) && !isPunctuation(character)
+}
+
+// Flanking decides which delimiters may pair; this decides which ones do, and a pair it leaves out reads back as another document.
+export function matchEmphasis<Run extends DelimiterRun>(runs: readonly Run[]): EmphasisPairing<Run>[] {
+  const pairings: EmphasisPairing<Run>[] = []
+  const bottoms = new Map<string, Candidate<Run> | undefined>()
   let closer = candidates(runs)
   while (closer !== undefined) {
     if (!closer.run.canClose) {
@@ -53,9 +50,9 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet<number> {
       continue
     }
     const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1
-    record(matched, opener, closer, used)
     opener.remaining -= used
     opener.tail -= used
+    pairings.push({ closer: closer.run, closerOffset: closer.head, opener: opener.run, openerOffset: opener.tail, used })
     closer.head += used
     closer.remaining -= used
     opener.next = closer
@@ -66,15 +63,22 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet<number> {
     unlink(closer)
     closer = following
   }
-  return matched
+  return pairings
 }
 
-function candidates(runs: readonly EmphasisRun[]): Candidate | undefined {
-  let first: Candidate | undefined
-  let previous: Candidate | undefined
+function candidates<Run extends DelimiterRun>(runs: readonly Run[]): Candidate<Run> | undefined {
+  let first: Candidate<Run> | undefined
+  let previous: Candidate<Run> | undefined
   for (const run of runs) {
-    const length = run.end - run.start
-    const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end }
+    const candidate: Candidate<Run> = {
+      head: 0,
+      next: undefined,
+      original: run.length,
+      previous,
+      remaining: run.length,
+      run,
+      tail: run.length,
+    }
     if (previous === undefined) first = candidate
     else previous.next = candidate
     previous = candidate
@@ -82,19 +86,33 @@ function candidates(runs: readonly EmphasisRun[]): Candidate | undefined {
   return first
 }
 
-function pairs(opener: Candidate, closer: Candidate): boolean {
+function pairs<Run extends DelimiterRun>(opener: Candidate<Run>, closer: Candidate<Run>): boolean {
   if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false
   const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0
   return !odd
 }
 
-function record(matched: Set<number>, opener: Candidate, closer: Candidate, used: number): void {
-  const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail)
-  const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used)
-  if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair)
-}
-
-function unlink(candidate: Candidate): void {
+function unlink<Run>(candidate: Candidate<Run>): void {
   if (candidate.previous !== undefined) candidate.previous.next = candidate.next
   if (candidate.next !== undefined) candidate.next.previous = candidate.previous
 }
+
+function isLeftFlanking(before: string, after: string): boolean {
+  if (isWhitespace(after)) return false
+  if (!isPunctuation(after)) return true
+  return isWhitespace(before) || isPunctuation(before)
+}
+
+function isRightFlanking(before: string, after: string): boolean {
+  if (isWhitespace(before)) return false
+  if (!isPunctuation(before)) return true
+  return isWhitespace(after) || isPunctuation(after)
+}
+
+function isPunctuation(character: string): boolean {
+  return character !== '' && unicodePunctuation.test(character)
+}
+
+function isWhitespace(character: string): boolean {
+  return character === '' || isUnicodeWhitespace(character)
+}

src/markdown-escaping.ts

@@ -1,5 +1,5 @@
-import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
-import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts'
+import { escapesLineClaim, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
+import { delimiterFlags, isWordCharacter, matchEmphasis } from './emphasis-matching.ts'
 
 export type EmphasisRole = 'close' | 'open'
 
@@ -15,7 +15,9 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine
 
 export type LineContainer = 'heading' | 'paragraph' | 'table-cell'
 
-type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number }
+type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number }
+
+type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; delimiters: EmittedDelimiter[]; length: number; start: number }
 
 const delimiters = ['*', '_', '`', '~']
 
@@ -23,7 +25,6 @@ const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
 const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/]
 const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/
 const followsLinkText = /[([:]/
-const unicodePunctuation = /[\p{P}\p{S}]/u
 
 export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine {
   return escape(resolveEmphasis(segments), container)
@@ -79,18 +80,43 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A
 }
 
 function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined {
-  const { nodes, runs } = emphasisRuns(segments, placements, output)
-  for (const run of runs) {
-    for (const delimiter of run.delimiters) {
-      if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair]
-    }
-  }
-  const unmatched = unmatchedPair(runs)
-  return unmatched === undefined ? undefined : nodes[unmatched]
+  const { nodes, runs } = emittedRuns(segments, placements, output)
+  const pair = misflanked(runs) ?? unpaired(runs)
+  return pair === undefined ? undefined : nodes[pair]
 }
 
-function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } {
-  const groups: DelimiterGroup[] = []
+function misflanked(runs: readonly EmittedRun[]): number | undefined {
+  for (const run of runs) {
+    for (const delimiter of run.delimiters) {
+      if (!(delimiter.closes ? run.canClose : run.canOpen)) return delimiter.pair
+    }
+  }
+  return undefined
+}
+
+function unpaired(runs: readonly EmittedRun[]): number | undefined {
+  const matched = new Set<number>()
+  for (const pairing of matchEmphasis(runs)) {
+    const opened = delimiterAt(pairing.opener, false, pairing.openerOffset, pairing.used)
+    const closed = delimiterAt(pairing.closer, true, pairing.closerOffset, pairing.used)
+    if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair)
+  }
+  // The last opener left unpaired is the innermost: the smallest carry that changes the line.
+  let innermost: number | undefined
+  for (const run of runs) {
+    for (const delimiter of run.delimiters) {
+      if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair
+    }
+  }
+  return innermost
+}
+
+function delimiterAt(run: EmittedRun, closes: boolean, offset: number, width: number): EmittedDelimiter | undefined {
+  return run.delimiters.find((delimiter) => delimiter.closes === closes && delimiter.offset === offset && delimiter.width === width)
+}
+
+function emittedRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmittedRun[] } {
+  const runs: EmittedRun[] = []
   const nodes: NodeRange[] = []
   const open: number[] = []
   let cursor = 0
@@ -99,30 +125,33 @@ function emphasisRuns(segments: readonly InlineSegment[], placements: readonly n
     cursor += segment.text.length
     if (segment.emphasis === undefined) continue
     const closes = segment.emphasis === 'close'
-    const end = start + segment.text.length
     const pair = closes ? (open.pop() ?? nodes.length) : nodes.length
     if (!closes) {
       nodes.push(segment.nodes)
       open.push(pair)
     }
-    const delimiter = { closes, end, pair, start }
-    const previous = groups[groups.length - 1]
-    if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) {
-      previous.delimiters.push(delimiter)
-      previous.end = end
+    const width = segment.text.length
+    const previous = runs[runs.length - 1]
+    if (previous !== undefined && previous.start + previous.length === start && previous.character === segment.text.charAt(0)) {
+      previous.delimiters.push({ closes, offset: start - previous.start, pair, width })
+      previous.length += width
       continue
     }
-    groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start })
+    runs.push({
+      canClose: false,
+      canOpen: false,
+      character: segment.text.charAt(0),
+      delimiters: [{ closes, offset: 0, pair, width }],
+      length: width,
+      start,
+    })
   }
-  return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) }
-}
-
-// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too.
-function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } {
-  const left = isLeftFlanking(before, after)
-  const right = isRightFlanking(before, after)
-  if (character !== '_') return { canClose: right, canOpen: left }
-  return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) }
+  for (const run of runs) {
+    const flags = delimiterFlags(run.character, charAt(output, run.start - 1), output.charAt(run.start + run.length))
+    run.canClose = flags.canClose
+    run.canOpen = flags.canOpen
+  }
+  return { nodes, runs }
 }
 
 function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean {
@@ -228,29 +257,8 @@ function runLength(scan: string, index: number): number {
   return length
 }
 
-function isLeftFlanking(before: string, after: string): boolean {
-  if (isWhitespace(after)) return false
-  if (!isPunctuation(after)) return true
-  return isWhitespace(before) || isPunctuation(before)
-}
 
-function isRightFlanking(before: string, after: string): boolean {
-  if (isWhitespace(before)) return false
-  if (!isPunctuation(before)) return true
-  return isWhitespace(after) || isPunctuation(after)
-}
 
-function isPunctuation(character: string): boolean {
-  return character !== '' && unicodePunctuation.test(character)
-}
-
-function isWhitespace(character: string): boolean {
-  return character === '' || isUnicodeWhitespace(character)
-}
-
-function isWordCharacter(character: string): boolean {
-  return character !== '' && !isUnicodeWhitespace(character) && !unicodePunctuation.test(character)
-}
 
 function charAt(text: string, index: number): string {
   return index < 0 ? '' : text.charAt(index)

src/markdown-inline.ts

@@ -48,7 +48,7 @@ export function tryImageLine(alt: string | undefined, href: string, path: Conver
   return attempt.ok ? attempt.value.line : undefined
 }
 
-// A demand names a run no spelling holds, and a carried node joins no run, so every pass carries at least one more node.
+// A carried node joins no run, so every pass carries at least one more node.
 function emitLine(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath): Result<EmittedLine> {
   const carried = new Set<number>()
   for (;;) {