Read the inline text, and decode the escapes and references CommonMark spells (#33)

src/markdown/commonmark-grammar.ts

@@ -1,16 +1,53 @@
+import { readEntityReference } from './entity-references.ts'
+
 export type LinePosition = 'first' | 'later'
 
+type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
+
+type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
+
+const htmlConstructNames = {
+  cdata: 'a CDATA section',
+  comment: 'an HTML comment',
+  declaration: 'an HTML declaration',
+  processingInstruction: 'an HTML processing instruction',
+}
+
 const controlCharacterRange = '\\u0000-\\u001f\\u007f'
 const autolinkSource = `[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\\s<>${controlCharacterRange}]*`
 const nullCharacterSource = '\\u0000'
-const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});'
+const tagNameSource = '[A-Za-z][A-Za-z0-9-]*'
+const htmlSpaceSource = '[ \\t\\n]'
+const attributeSource = `(?:${htmlSpaceSource}+[A-Za-z_:][A-Za-z0-9_.:-]*(?:${htmlSpaceSource}*=${htmlSpaceSource}*(?:[^ \\t\\n"'=<>\`]+|'[^']*'|"[^"]*"))?)`
+
+const htmlTagSource = `(?:<${tagNameSource}${attributeSource}*${htmlSpaceSource}*/?>|</${tagNameSource}${htmlSpaceSource}*>)`
 
-const anchoredEntityReference = new RegExp(`^(?:${entityReferenceSource})`)
 const autolink = new RegExp(`^(?:${autolinkSource})$`)
-const bracketedAutolink = new RegExp(`^<(?:${autolinkSource})>`)
+const bracketedAutolink = new RegExp(`<(?:${autolinkSource})>`, 'y')
 const controlCharacter = new RegExp(`[${controlCharacterRange}]`)
-const entityReference = new RegExp(entityReferenceSource)
+const htmlTag = new RegExp(htmlTagSource, 'y')
 const nullCharacter = new RegExp(nullCharacterSource)
+const tagName = new RegExp(`^</?(${tagNameSource})[\\s\\S]*$`)
+// The opener's own match ends with the terminator where the construct is complete on its own (`<!-->`).
+const inlineHtmlConstructs = [
+  { name: htmlConstructNames.cdata, opener: /<!\[CDATA\[/y, terminator: ']]>' },
+  { name: htmlConstructNames.comment, opener: /<!(?:--->|-->|--)/y, terminator: '-->' },
+  { name: htmlConstructNames.declaration, opener: /<![A-Za-z]/y, terminator: '>' },
+  { name: htmlConstructNames.processingInstruction, opener: /<\?/y, terminator: '?>' },
+]
+// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
+const blockTagNames =
+  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
+const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`)
+const htmlBlockConditions: HtmlBlockCondition[] = [
+  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
+  { closer: /-->/, construct: htmlConstructNames.comment, interrupts: true, start: /^<!--/ },
+  { closer: /\?>/, construct: htmlConstructNames.processingInstruction, interrupts: true, start: /^<\?/ },
+  { closer: />/, construct: htmlConstructNames.declaration, interrupts: true, start: /^<![A-Za-z]/ },
+  { closer: /\]\]>/, construct: htmlConstructNames.cdata, interrupts: true, start: /^<!\[CDATA\[/ },
+  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
+  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
+]
 const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
 const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/
 const codeFenceOpener = /^(`{3,}|~{3,})/
@@ -19,7 +56,7 @@ const pipeClaim = /^\|/
 const bulletListOpener = /^[*+-](?:[ \t]|$)/
 // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip.
 const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim]
-const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/]
+const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y
 const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/
 const setextUnderline = /^(=+|-+)[ \t]*$/
 const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
@@ -32,6 +69,13 @@ export function atxHeading(line: string): { level: number; text: string } | unde
   return { level: hashes.length, text: trimSpace(text.replace(/(?:^|(?<=[ \t]))#+$/, '')) }
 }
 
+// The character a backslash escapes at `index`, `undefined` where the backslash is literal text.
+export function backslashEscape(text: string, index: number): string | undefined {
+  if (text.charAt(index) !== '\\') return undefined
+  const escaped = text.charAt(index + 1)
+  return isAsciiPunctuation(escaped) ? escaped : undefined
+}
+
 export function claimsDirectiveLine(line: string): boolean {
   return directiveClaim.test(line)
 }
@@ -50,9 +94,32 @@ export function closingCodeFence(line: string, marker: string): boolean {
   return /^[ \t]*$/.test(line.slice(closing.length))
 }
 
+export function decodeTextEscapes(text: string): string {
+  let decoded = ''
+  let index = 0
+  while (index < text.length) {
+    const escaped = backslashEscape(text, index)
+    if (escaped !== undefined) {
+      decoded += escaped
+      index += 2
+      continue
+    }
+    const reference = readEntityReference(text, index)
+    if (reference !== undefined) {
+      decoded += reference.text
+      index += reference.length
+      continue
+    }
+    decoded += text.charAt(index)
+    index += 1
+  }
+  return decoded
+}
+
 export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean {
   if (offset === 0) {
     if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true
+    if (openingHtmlBlock(line, position === 'later') !== undefined) return true
     return position === 'later' && setextUnderline.test(line)
   }
   const digits = orderedListOpener.exec(line)?.[1]
@@ -63,14 +130,27 @@ export function holdsControlCharacter(text: string): boolean {
   return controlCharacter.test(text)
 }
 
-export function holdsEntityReference(text: string): boolean {
-  return entityReference.test(text)
-}
-
 export function holdsNullCharacter(text: string): boolean {
   return nullCharacter.test(text)
 }
 
+function htmlTagName(text: string): string {
+  return text.replace(tagName, '<$1>')
+}
+
+export function inlineHtmlConstruct(text: string, index: number): string | undefined {
+  for (const construct of inlineHtmlConstructs) {
+    construct.opener.lastIndex = index
+    const opened = construct.opener.exec(text)?.[0]
+    if (opened === undefined) continue
+    if (opened.endsWith(construct.terminator)) return construct.name
+    return text.includes(construct.terminator, index + opened.length) ? construct.name : undefined
+  }
+  htmlTag.lastIndex = index
+  const tag = htmlTag.exec(text)?.[0]
+  return tag === undefined ? undefined : htmlTagName(tag)
+}
+
 export function isAsciiPunctuation(character: string): boolean {
   return asciiPunctuation.test(character)
 }
@@ -108,12 +188,22 @@ export function openingCodeFence(line: string): { info: string; marker: string }
   return marker.startsWith('`') && info.includes('`') ? undefined : { info, marker }
 }
 
-export function opensBracketedAutolink(text: string): boolean {
+export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
+  for (const condition of htmlBlockConditions) {
+    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
+    return { closer: condition.closer, construct: condition.construct ?? htmlTagName(line) }
+  }
+  return undefined
+}
+
+export function opensBracketedAutolink(text: string, index: number): boolean {
+  bracketedAutolink.lastIndex = index
   return bracketedAutolink.test(text)
 }
 
-export function opensHtmlConstruct(text: string): boolean {
-  return htmlConstructs.some((construct) => construct.test(text))
+export function opensEmailAutolink(text: string, index: number): boolean {
+  emailAutolink.lastIndex = index
+  return emailAutolink.test(text)
 }
 
 export function setextHeadingLevel(line: string): number | undefined {
@@ -122,10 +212,6 @@ export function setextHeadingLevel(line: string): number | undefined {
   return underline.startsWith('=') ? 1 : 2
 }
 
-export function startsEntityReference(text: string): boolean {
-  return anchoredEntityReference.test(text)
-}
-
 export function trimSpace(text: string): string {
   return text.replace(/^[ \t]+|[ \t]+$/g, '')
 }