Model CommonMark's emphasis matching, and escape the delimiter that only closes

src/adf-to-markdown.test.ts

@@ -197,12 +197,12 @@ test('escapes only text that would otherwise open a construct', () => {
   assert.equal(emitted(':::panel info'), '\\:::panel info\n')
   assert.equal(emitted('10:30 tomorrow'), '10:30 tomorrow\n')
   assert.equal(emitted('[a](b)'), '\\[a](b)\n')
-  assert.equal(emitted('**bold**'), '\\*\\*bold**\n')
+  assert.equal(emitted('**bold**'), '\\*\\*bold\\*\\*\n')
   assert.equal(emitted('a `x` b'), 'a \\`x` b\n')
-  assert.equal(emitted('~~struck~~'), '\\~~struck~~\n')
-  assert.equal(emitted('a \\* b'), 'a \\\\* b\n')
+  assert.equal(emitted('~~struck~~'), '\\~~struck\\~~\n')
+  assert.equal(emitted('a \\* b'), 'a \\\\\\* b\n')
   assert.equal(emitted('1. not a list'), '1\\. not a list\n')
-  assert.equal(emitted('*"quoted"*'), '\\*"quoted"*\n')
+  assert.equal(emitted('*"quoted"*'), '\\*"quoted"\\*\n')
   assert.equal(emitted('x"_y"'), 'x"\\_y"\n')
 })
 
@@ -297,6 +297,38 @@ test('escapes a literal delimiter that would merge with an emitted one', () => {
   assert.equal(emitted(marked('x', { attrs: { href: 'https://example.com/' }, type: 'link' }), { text: '{}', type: 'text' }), '[x](https://example.com/){}\n')
 })
 
+test('escapes a literal delimiter run that flanks either way', () => {
+  const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' })
+  const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content))))
+  assert.equal(emitted({ text: 'un', type: 'text' }, marked('a* b', { type: 'em' }), { text: 'istic', type: 'text' }), 'un*a\\* b*istic\n')
+  assert.equal(emitted(marked('a~~ b', { type: 'strike' })), '~~a\\~~ b~~\n')
+  assert.equal(emitted(marked('a_ b', { type: 'em' })), '_a\\_ b_\n')
+  assert.equal(emitted({ text: 'a* b', type: 'text' }), 'a\\* b\n')
+  assert.equal(emitted({ text: '2 * 3', type: 'text' }), '2 * 3\n')
+  assert.equal(emitted({ text: 'snake_case_name', type: 'text' }), 'snake_case_name\n')
+})
+
+test('carries a mark run CommonMark matching pairs elsewhere', () => {
+  const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' })
+  const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content))))
+  const em: AdfMark = { type: 'em' }
+  const strong: AdfMark = { type: 'strong' }
+  assert.equal(
+    emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', strong), { text: 'istic', type: 'text' }),
+    'un:adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"}],\\"text\\":\\"a\\",\\"type\\":\\"text\\"}"}' +
+      ':adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"},{\\"type\\":\\"strong\\"}],\\"text\\":\\"b\\",\\"type\\":\\"text\\"}"}**c**istic\n',
+  )
+  assert.equal(
+    emitted(marked('a', strong, em), marked('b', strong), marked('c', strong, em)),
+    '***a*b**:adf{json="{\\"marks\\":[{\\"type\\":\\"strong\\"},{\\"type\\":\\"em\\"}],\\"text\\":\\"c\\",\\"type\\":\\"text\\"}"}\n',
+  )
+  assert.equal(emitted({ text: 'un', type: 'text' }, marked('a', em, strong), { text: 'istic', type: 'text' }), 'un***a***istic\n')
+  assert.equal(
+    emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', em), { text: 'istic', type: 'text' }),
+    'un*a**b**c*istic\n',
+  )
+})
+
 test('escapes a hyphen underline a hard break would expose', () => {
   const line = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text: 'foo', type: 'text' }, { type: 'hardBreak' }, { text, type: 'text' }))))
   assert.equal(line('--'), 'foo\\\n\\--\n')

src/corpus.test.ts

@@ -66,6 +66,25 @@ for (const directory of emittingDirectories) {
   }
 }
 
+// One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it.
+test('no two round-trip documents share one spelling', () => {
+  const documents = new Map<string, string>()
+  const spellings = new Map<string, string>()
+  for (const directory of emittingDirectories) {
+    for (const name of fixtureNames(directory, '.md')) {
+      const fixture = `${directory}/${name}`
+      const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
+      assert.ok(isJsonValue(parsed))
+      const document = serializeCanonicalJson(parsed, 'compact')
+      const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8')
+      assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`)
+      assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`)
+      documents.set(document, fixture)
+      spellings.set(markdown, fixture)
+    }
+  }
+})
+
 // spec/flavour.md, Directives: the container fence rule, checked against the emitted bytes.
 function fenceNestingFault(markdown: string): string | undefined {
   const open: number[] = []

src/emphasis-matching.ts

@@ -0,0 +1,100 @@
+export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number }
+
+export type EmphasisRun = {
+  canClose: boolean
+  canOpen: boolean
+  character: string
+  delimiters: readonly EmphasisDelimiter[]
+  end: number
+  start: number
+}
+
+type Candidate = {
+  head: number
+  next: Candidate | undefined
+  original: number
+  previous: Candidate | undefined
+  remaining: number
+  run: EmphasisRun
+  tail: number
+}
+
+// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document.
+export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined {
+  const matched = matchDelimiters(runs)
+  // The last opener left unpaired is the innermost: the smallest carry that changes the line.
+  let innermost: number | undefined
+  for (const run of runs) {
+    for (const delimiter of run.delimiters) {
+      if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair
+    }
+  }
+  return innermost
+}
+
+function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet<number> {
+  const matched = new Set<number>()
+  const bottoms = new Map<string, Candidate | undefined>()
+  let closer = candidates(runs)
+  while (closer !== undefined) {
+    if (!closer.run.canClose) {
+      closer = closer.next
+      continue
+    }
+    const key = `${closer.run.character}${closer.run.canOpen}${closer.original % 3}`
+    const bottom = bottoms.get(key)
+    let opener = closer.previous
+    while (opener !== undefined && opener !== bottom && !pairs(opener, closer)) opener = opener.previous
+    if (opener === undefined || opener === bottom) {
+      bottoms.set(key, closer.previous)
+      const following = closer.next
+      if (!closer.run.canOpen) unlink(closer)
+      closer = following
+      continue
+    }
+    const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1
+    record(matched, opener, closer, used)
+    opener.remaining -= used
+    opener.tail -= used
+    closer.head += used
+    closer.remaining -= used
+    opener.next = closer
+    closer.previous = opener
+    if (opener.remaining === 0) unlink(opener)
+    if (closer.remaining > 0) continue
+    const following = closer.next
+    unlink(closer)
+    closer = following
+  }
+  return matched
+}
+
+function candidates(runs: readonly EmphasisRun[]): Candidate | undefined {
+  let first: Candidate | undefined
+  let previous: Candidate | undefined
+  for (const run of runs) {
+    const length = run.end - run.start
+    const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end }
+    if (previous === undefined) first = candidate
+    else previous.next = candidate
+    previous = candidate
+  }
+  return first
+}
+
+function pairs(opener: Candidate, closer: Candidate): boolean {
+  if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false
+  const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0
+  return !odd
+}
+
+function record(matched: Set<number>, opener: Candidate, closer: Candidate, used: number): void {
+  const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail)
+  const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used)
+  if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair)
+}
+
+function unlink(candidate: Candidate): void {
+  if (candidate.previous !== undefined) candidate.previous.next = candidate.next
+  if (candidate.next !== undefined) candidate.next.previous = candidate.previous
+}

src/markdown-escaping.ts

@@ -1,4 +1,5 @@
 import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
+import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts'
 
 export type EmphasisRole = 'close' | 'open'
 
@@ -14,7 +15,7 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine
 
 export type LineContainer = 'heading' | 'paragraph' | 'table-cell'
 
-type DelimiterRun = { character: string; closeNodes: NodeRange | undefined; end: number; openNodes: NodeRange | undefined; start: number }
+type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number }
 
 const delimiters = ['*', '_', '`', '~']
 
@@ -78,17 +79,20 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A
 }
 
 function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined {
-  for (const run of delimiterRuns(segments, placements)) {
-    const before = charAt(output, run.start - 1)
-    const after = output.charAt(run.end)
-    if (run.openNodes !== undefined && !isLeftFlanking(before, after)) return run.openNodes
-    if (run.closeNodes !== undefined && !isRightFlanking(before, after)) return run.closeNodes
+  const { nodes, runs } = emphasisRuns(segments, placements, output)
+  for (const run of runs) {
+    for (const delimiter of run.delimiters) {
+      if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair]
+    }
   }
-  return undefined
+  const unmatched = unmatchedPair(runs)
+  return unmatched === undefined ? undefined : nodes[unmatched]
 }
 
-function delimiterRuns(segments: readonly InlineSegment[], placements: readonly number[]): DelimiterRun[] {
-  const runs: DelimiterRun[] = []
+function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } {
+  const groups: DelimiterGroup[] = []
+  const nodes: NodeRange[] = []
+  const open: number[] = []
   let cursor = 0
   for (const segment of segments) {
     const start = placements[cursor] ?? 0
@@ -96,22 +100,29 @@ function delimiterRuns(segments: readonly InlineSegment[], placements: readonly
     if (segment.emphasis === undefined) continue
     const closes = segment.emphasis === 'close'
     const end = start + segment.text.length
-    const previous = runs[runs.length - 1]
+    const pair = closes ? (open.pop() ?? nodes.length) : nodes.length
+    if (!closes) {
+      nodes.push(segment.nodes)
+      open.push(pair)
+    }
+    const delimiter = { closes, end, pair, start }
+    const previous = groups[groups.length - 1]
     if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) {
-      previous.closeNodes = previous.closeNodes ?? (closes ? segment.nodes : undefined)
+      previous.delimiters.push(delimiter)
       previous.end = end
-      previous.openNodes = previous.openNodes ?? (closes ? undefined : segment.nodes)
       continue
     }
-    runs.push({
-      character: segment.text.charAt(0),
-      closeNodes: closes ? segment.nodes : undefined,
-      end,
-      openNodes: closes ? undefined : segment.nodes,
-      start,
-    })
+    groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start })
   }
-  return runs
+  return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) }
+}
+
+// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too.
+function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } {
+  const left = isLeftFlanking(before, after)
+  const right = isRightFlanking(before, after)
+  if (character !== '_') return { canClose: right, canOpen: left }
+  return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) }
 }
 
 function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean {
@@ -177,7 +188,7 @@ function claimsCharacter(
   if (character === ':') return inlineDirectiveOpener.test(rest)
   if (character === '[') return opensLink(scan, escapings, index)
   if (character === '`') return opensCodeSpan(scan, index, escaped)
-  if (character === '*' || character === '_' || character === '~') return opensEmphasis(scan, index, escaped)
+  if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped)
   return false
 }
 
@@ -196,16 +207,13 @@ function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>
   return new RegExp('(?<!`)`{' + length + '}(?!`)').test(scan.slice(index + length))
 }
 
-function opensEmphasis(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
+function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
   if (!startsRun(scan, index, escaped)) return false
   const character = scan.charAt(index)
   const length = runLength(scan, index)
-  const before = index === 0 ? '' : scan.charAt(index - 1)
-  const after = scan.charAt(index + length)
-  if (character === '~') return length === 2 && isLeftFlanking(before, after)
-  if (!isLeftFlanking(before, after)) return false
-  if (character === '*') return true
-  return !isRightFlanking(before, after) || isPunctuation(before)
+  if (character === '~' && length !== 2) return false
+  const flags = delimiterFlags(character, index === 0 ? '' : scan.charAt(index - 1), scan.charAt(index + length))
+  return flags.canClose || flags.canOpen
 }
 
 function startsRun(scan: string, index: number, escaped: ReadonlySet<number>): boolean {