Read the inline text, and decode the escapes and references CommonMark spells (#33)

src/markdown/emit/adf-to-markdown.test.ts

@@ -202,8 +202,25 @@ test('spells a heading level no ATX heading fits as a directive', () => {
 test('escapes only text that would otherwise open a construct', () => {
   const emitted = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text, type: 'text' }))))
   assert.equal(emitted('<div>'), '\\<div>\n')
+  assert.equal(emitted('<div'), '\\<div\n')
+  assert.equal(emitted('<div and more'), '\\<div and more\n')
+  assert.equal(emitted('<pre'), '\\<pre\n')
+  assert.equal(emitted('<!x'), '\\<!x\n')
+  assert.equal(emitted('<!-- x'), '\\<!-- x\n')
+  assert.equal(emitted('<?php'), '\\<?php\n')
+  assert.equal(emitted('<![CDATA[x'), '\\<![CDATA[x\n')
+  assert.equal(emitted('<span'), '<span\n')
   assert.equal(emitted('a < b'), 'a < b\n')
   assert.equal(emitted('&amp; & x'), '\\&amp; & x\n')
+  assert.equal(emitted('&notareference; x'), '&notareference; x\n')
+  assert.equal(emitted('a <b@c.d> e'), 'a \\<b@c.d> e\n')
+  assert.equal(emitted('a <b 2'), 'a <b 2\n')
+  assert.equal(emitted('a <div b'), 'a <div b\n')
+  assert.equal(emitted('a <!-- b'), 'a <!-- b\n')
+  assert.equal(emitted('a <!-- b --> c'), 'a \\<!-- b --> c\n')
+  const later = paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, { text: '<div', type: 'text' })
+  assert.equal(markdown(adfToMarkdown(document(later))), 'a\\\n\\<div\n')
+  assert.equal(markdown(adfToMarkdown(document({ content: [paragraph({ text: '<!-- x', type: 'text' })], type: 'blockquote' }))), '> \\<!-- x\n')
   assert.equal(emitted('| a | b |'), '\\| a | b |\n')
   assert.equal(emitted(':mention[@x]{id=1}'), '\\:mention[@x]{id=1}\n')
   assert.equal(emitted(':::panel info'), '\\:::panel info\n')

src/markdown/emit/adf-to-markdown.ts

@@ -7,7 +7,8 @@ import { carriesOnly, isAdfDocument } from '../../adf/document.ts'
 import { emitInlineLine } from './inline-line.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { fencedCodeBlock } from '../backtick-runs.ts'
-import { holdsControlCharacter, holdsEntityReference, holdsNullCharacter, isThematicBreak, markerInterruptsParagraph } from '../commonmark-grammar.ts'
+import { holdsControlCharacter, holdsNullCharacter, isThematicBreak, markerInterruptsParagraph } from '../commonmark-grammar.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 import { largestNesting } from '../../nesting.ts'
 import { spellDirectiveHeader } from './block-directive-spelling.ts'
 import { tryImage } from './image.ts'

src/markdown/emit/destination-spelling.ts

@@ -1,11 +1,12 @@
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
-import { holdsControlCharacter, holdsEntityReference } from '../commonmark-grammar.ts'
+import { holdsControlCharacter } from '../commonmark-grammar.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 
 export function spellDestination(href: string, path: ConvertErrorPath): Result<string> {
   if (holdsControlCharacter(href)) return failure('unspellable-link-destination', 'a link destination holds a control character', path)
   if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination', path)
   if (holdsEntityReference(href)) {
-    return failure('unspellable-link-destination', 'a link destination shaped like an entity reference decodes on the way back', path)
+    return failure('unspellable-link-destination', 'a link destination holds an entity reference that decodes on the way back', path)
   }
   if (href.includes(' ')) {
     if (/[<>]/.test(href)) {
@@ -22,7 +23,7 @@ export function spellTitle(title: string, path: ConvertErrorPath): Result<string
   if (/["\n\r\\]/.test(title)) {
     return failure('unspellable-link-title', 'no canonical escape spells a quote, backslash or newline in a link title', path)
   }
-  if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title shaped like an entity reference decodes on the way back', path)
+  if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title holds an entity reference that decodes on the way back', path)
   return success(` "${title}"`)
 }
 

src/markdown/emit/inline-line.ts

@@ -2,8 +2,9 @@ import type { AdfMark, AdfNode } from '../../adf/document.ts'
 import type { InlineDirective } from '../../adf/inline-directives.ts'
 import { assembleInlineLine, type InlineEscaping, type InlineSegment, type LineContainer, type NodeRange } from './line-escaping.ts'
 import { carriedInline } from '../opaque-carry.ts'
-import { claimsLine, holdsEntityReference, holdsNullCharacter, isAutolink } from '../commonmark-grammar.ts'
+import { claimsLine, holdsNullCharacter, isAutolink } from '../commonmark-grammar.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
+import { holdsEntityReference } from '../entity-references.ts'
 import { inlineDirective } from '../../adf/inline-directives.ts'
 import { largestNesting } from '../../nesting.ts'
 import { longestBacktickRun } from '../backtick-runs.ts'

src/markdown/emit/line-escaping.ts

@@ -1,13 +1,8 @@
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { delimiterFlags, isWordCharacter, matchEmphasis } from '../emphasis-matching.ts'
-import {
-  escapesLineClaim,
-  isAsciiPunctuation,
-  opensBracketedAutolink,
-  opensHtmlConstruct,
-  startsEntityReference,
-  type LinePosition,
-} from '../commonmark-grammar.ts'
+import { backslashEscape, escapesLineClaim, inlineHtmlConstruct, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts'
 import { opensInlineDirective } from '../directive-attributes.ts'
+import { readEntityReference } from '../entity-references.ts'
 
 export type EmphasisRole = 'close' | 'open'
 
@@ -21,6 +16,8 @@ export type InlineSegment =
 
 export type AssembledLine = { line: string; unspellableRun: NodeRange | undefined }
 
+type ScanLine = { position: LinePosition; start: number; text: string }
+
 export type LineContainer = 'heading' | 'paragraph' | 'table-cell'
 
 type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number }
@@ -72,10 +69,18 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A
   const escaped = new Set<number>()
   const placements: number[] = []
   let output = ''
+  const linkClose = lastLinkClose(scan, escapings)
+  let line = scanLine(scan, 0)
   for (let index = 0; index < scan.length; index += 1) {
+    if (index > line.start + line.text.length) line = scanLine(scan, line.start + line.text.length + 1)
     const escaping = escapings[index]
     const escapable = escaping === 'backslash' || escaping === 'bracketed'
-    if (escapable && (mergesWithSyntax(scan, escapings, index) || opensConstruct(scan, escapings, index, escaping === 'bracketed', container, escaped))) {
+    if (
+      escapable &&
+      (claimsLineStart(line, index, container) ||
+        mergesWithSyntax(scan, escapings, index) ||
+        opensConstruct(scan, linkClose, index, escaping === 'bracketed', container, escaped))
+    ) {
       output += '\\'
       escaped.add(index)
     }
@@ -181,23 +186,23 @@ function isSyntax(escaping: InlineEscaping | undefined): boolean {
 
 function opensConstruct(
   scan: string,
-  escapings: readonly (InlineEscaping | undefined)[],
+  linkClose: number,
   index: number,
   inBrackets: boolean,
   container: LineContainer,
   escaped: ReadonlySet<number>,
 ): boolean {
   if (container === 'heading' && closesHeading(scan, index)) return true
-  if (container === 'paragraph' && claimsLineStart(scan, index)) return true
-  return claimsCharacter(scan, escapings, index, inBrackets, container, escaped)
+  return claimsCharacter(scan, linkClose, index, inBrackets, container, escaped)
 }
 
-function claimsLineStart(scan: string, index: number): boolean {
-  const start = scan.lastIndexOf('\n', index - 1) + 1
-  const end = scan.indexOf('\n', index)
-  const line = scan.slice(start, end === -1 ? undefined : end)
-  const position: LinePosition = start === 0 ? 'first' : 'later'
-  return escapesLineClaim(line, index - start, position)
+function claimsLineStart(line: ScanLine, index: number, container: LineContainer): boolean {
+  return container === 'paragraph' && escapesLineClaim(line.text, index - line.start, line.position)
+}
+
+function scanLine(scan: string, start: number): ScanLine {
+  const end = scan.indexOf('\n', start)
+  return { position: start === 0 ? 'first' : 'later', start, text: scan.slice(start, end === -1 ? undefined : end) }
 }
 
 function closesHeading(scan: string, index: number): boolean {
@@ -207,39 +212,38 @@ function closesHeading(scan: string, index: number): boolean {
 
 function claimsCharacter(
   scan: string,
-  escapings: readonly (InlineEscaping | undefined)[],
+  linkClose: number,
   index: number,
   inBrackets: boolean,
   container: LineContainer,
   escaped: ReadonlySet<number>,
 ): boolean {
   const character = scan.charAt(index)
-  const rest = scan.slice(index)
   if (inBrackets && (character === '[' || character === ']')) return true
   if (character === '|') return container === 'table-cell'
-  if (character === '\\') return isAsciiPunctuation(scan.charAt(index + 1))
-  if (character === '&') return startsEntityReference(rest)
-  if (character === '<') return opensBracketedAutolink(rest) || opensHtmlConstruct(rest)
-  if (character === ':') return opensInlineDirective(rest)
-  if (character === '[') return opensLink(scan, escapings, index)
+  if (character === '\\') return backslashEscape(scan, index) !== undefined
+  if (character === '&') return readEntityReference(scan, index) !== undefined
+  if (character === '<') return opensBracketedAutolink(scan, index) || opensEmailAutolink(scan, index) || inlineHtmlConstruct(scan, index) !== undefined
+  if (character === ':') return opensInlineDirective(scan, index)
+  if (character === '[') return index < linkClose
   if (character === '`') return opensCodeSpan(scan, index, escaped)
   if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped)
   return false
 }
 
 // A `]` the emitter spelled sits inside a construct that binds before link text does.
-function opensLink(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean {
-  for (let cursor = index + 1; cursor < scan.length; cursor += 1) {
+function lastLinkClose(scan: string, escapings: readonly (InlineEscaping | undefined)[]): number {
+  for (let cursor = scan.length - 1; cursor >= 0; cursor -= 1) {
     if (scan.charAt(cursor) !== ']' || isSyntax(escapings[cursor])) continue
-    if (followsLinkText.test(scan.charAt(cursor + 1))) return true
+    if (followsLinkText.test(scan.charAt(cursor + 1))) return cursor
   }
-  return false
+  return -1
 }
 
 function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
   if (!startsRun(scan, index, escaped)) return false
-  const length = runLength(scan, index)
-  return new RegExp('(?<!`)`{' + length + '}(?!`)').test(scan.slice(index + length))
+  const opener = backtickRun(scan, index)
+  return closingBacktickRun(scan, index + opener, opener) !== undefined
 }
 
 function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet<number>): boolean {