Answer the architecture review: one table for the HTML block start, and one code-span reader

AGENTS.md

@@ -54,9 +54,9 @@ Round-trip equality is a property tested over a corpus, not a claim made in pros
 `dependencies` is empty. A runtime dependency enters only through a decision entry here stating
 why ~20 lines of own code cannot do the job, who maintains it, and what auditing it costs. So the
 CommonMark and HTML parsers are written in this repo. A table a standard fixes is data rather than
-a dependency: HTML5's 2125 semicolon-terminated character references ship packed in
-`markdown/entity-references.ts`, so entity decoding is complete without one. `devDependencies`:
-few, each earning its keep; they never reach a consumer.
+a dependency: HTML5's 2125 semicolon-terminated character references ship packed in their own
+module, so entity decoding is complete without one. `devDependencies`: few, each earning its keep;
+they never reach a consumer.
 
 ## 6. The package contract
 

corpus/round-trip/commonmark-subset/text-escaping.json

@@ -44,6 +44,24 @@
         }
       ],
       "type": "paragraph"
+    },
+    {
+      "content": [
+        {
+          "text": "<div opens no HTML block",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "content": [
+        {
+          "text": "<?php opens none either",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
     }
   ],
   "type": "doc",

corpus/round-trip/commonmark-subset/text-escaping.md

@@ -7,3 +7,7 @@
 snake_case_name
 
 \*not emphasis\*
+
+\<div opens no HTML block
+
+\<?php opens none either

src/markdown/backtick-runs.ts

@@ -1,3 +1,20 @@
+export function backtickRun(text: string, index: number): number {
+  let length = 0
+  while (text.charAt(index + length) === '`') length += 1
+  return length
+}
+
+// Where the run of exactly `opener` backticks closing a code span begins, `undefined` where none does.
+export function closingBacktickRun(text: string, from: number, opener: number): number | undefined {
+  let cursor = from
+  while (cursor < text.length) {
+    const run = backtickRun(text, cursor)
+    if (run === opener) return cursor
+    cursor += run === 0 ? 1 : run
+  }
+  return undefined
+}
+
 export function fencedCodeBlock(info: string, body: string): string {
   const fence = '`'.repeat(Math.max(3, longestBacktickRun(body) + 1))
   return body === '' ? `${fence}${info}\n${fence}` : `${fence}${info}\n${body}\n${fence}`

src/markdown/commonmark-grammar.ts

@@ -4,6 +4,10 @@ export type HtmlConstruct = { length: number; name: string }
 
 export type LinePosition = 'first' | 'later'
 
+export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
+
+type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
+
 export const htmlConstructNames = {
   cdata: 'a CDATA section',
   comment: 'an HTML comment',
@@ -32,6 +36,19 @@ const inlineHtmlConstructs = [
   { name: htmlConstructNames.declaration, pattern: /^<![A-Za-z][^>]*>/ },
   { name: htmlConstructNames.processingInstruction, pattern: /^<\?[\s\S]*?\?>/ },
 ]
+// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
+const blockTagNames =
+  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
+const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`)
+const htmlBlockConditions: HtmlBlockCondition[] = [
+  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
+  { closer: /-->/, construct: htmlConstructNames.comment, interrupts: true, start: /^<!--/ },
+  { closer: /\?>/, construct: htmlConstructNames.processingInstruction, interrupts: true, start: /^<\?/ },
+  { closer: />/, construct: htmlConstructNames.declaration, interrupts: true, start: /^<![A-Za-z]/ },
+  { closer: /\]\]>/, construct: htmlConstructNames.cdata, interrupts: true, start: /^<!\[CDATA\[/ },
+  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
+  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
+]
 const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
 const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/
 const codeFenceOpener = /^(`{3,}|~{3,})/
@@ -53,6 +70,13 @@ export function atxHeading(line: string): { level: number; text: string } | unde
   return { level: hashes.length, text: trimSpace(text.replace(/(?:^|(?<=[ \t]))#+$/, '')) }
 }
 
+// The character a backslash escapes at `index`, `undefined` where the backslash is literal text.
+export function backslashEscape(text: string, index: number): string | undefined {
+  if (text.charAt(index) !== '\\') return undefined
+  const escaped = text.charAt(index + 1)
+  return isAsciiPunctuation(escaped) ? escaped : undefined
+}
+
 export function claimsDirectiveLine(line: string): boolean {
   return directiveClaim.test(line)
 }
@@ -71,7 +95,6 @@ export function closingCodeFence(line: string, marker: string): boolean {
   return /^[ \t]*$/.test(line.slice(closing.length))
 }
 
-// Text where only backslash escapes and entity references are processed: an info string, a link title.
 export function decodeTextEscapes(text: string): string {
   let decoded = ''
   let index = 0
@@ -97,6 +120,7 @@ export function decodeTextEscapes(text: string): string {
 export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean {
   if (offset === 0) {
     if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true
+    if (openingHtmlBlock(line, position === 'later') !== undefined) return true
     return position === 'later' && setextUnderline.test(line)
   }
   const digits = orderedListOpener.exec(line)?.[1]
@@ -161,6 +185,14 @@ export function openingCodeFence(line: string): { info: string; marker: string }
   return marker.startsWith('`') && info.includes('`') ? undefined : { info, marker }
 }
 
+export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
+  for (const condition of htmlBlockConditions) {
+    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
+    return { closer: condition.closer, construct: condition.construct ?? htmlTagName(line) }
+  }
+  return undefined
+}
+
 export function opensBracketedAutolink(text: string): boolean {
   return bracketedAutolink.test(text)
 }
@@ -178,9 +210,3 @@ export function setextHeadingLevel(line: string): number | undefined {
 export function trimSpace(text: string): string {
   return text.replace(/^[ \t]+|[ \t]+$/g, '')
 }
-
-function backslashEscape(text: string, index: number): string | undefined {
-  if (text.charAt(index) !== '\\') return undefined
-  const escaped = text.charAt(index + 1)
-  return isAsciiPunctuation(escaped) ? escaped : undefined
-}

src/markdown/emit/adf-to-markdown.test.ts

@@ -202,11 +202,23 @@ test('spells a heading level no ATX heading fits as a directive', () => {
 test('escapes only text that would otherwise open a construct', () => {
   const emitted = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text, type: 'text' }))))
   assert.equal(emitted('<div>'), '\\<div>\n')
+  assert.equal(emitted('<div'), '\\<div\n')
+  assert.equal(emitted('<div and more'), '\\<div and more\n')
+  assert.equal(emitted('<pre'), '\\<pre\n')
+  assert.equal(emitted('<!x'), '\\<!x\n')
+  assert.equal(emitted('<!-- x'), '\\<!-- x\n')
+  assert.equal(emitted('<?php'), '\\<?php\n')
+  assert.equal(emitted('<![CDATA[x'), '\\<![CDATA[x\n')
+  assert.equal(emitted('<span'), '<span\n')
   assert.equal(emitted('a < b'), 'a < b\n')
   assert.equal(emitted('&amp; & x'), '\\&amp; & x\n')
   assert.equal(emitted('&notareference; x'), '&notareference; x\n')
   assert.equal(emitted('a <b@c.d> e'), 'a \\<b@c.d> e\n')
   assert.equal(emitted('a <b 2'), 'a <b 2\n')
+  assert.equal(emitted('a <div b'), 'a <div b\n')
+  const later = paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, { text: '<div', type: 'text' })
+  assert.equal(markdown(adfToMarkdown(document(later))), 'a\\\n\\<div\n')
+  assert.equal(markdown(adfToMarkdown(document({ content: [paragraph({ text: '<!-- x', type: 'text' })], type: 'blockquote' }))), '> \\<!-- x\n')
   assert.equal(emitted('| a | b |'), '\\| a | b |\n')
   assert.equal(emitted(':mention[@x]{id=1}'), '\\:mention[@x]{id=1}\n')
   assert.equal(emitted(':::panel info'), '\\:::panel info\n')

src/markdown/emit/line-escaping.ts

@@ -1,3 +1,4 @@
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { delimiterFlags, isWordCharacter, matchEmphasis } from '../emphasis-matching.ts'
 import { escapesLineClaim, inlineHtmlConstruct, isAsciiPunctuation, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts'
 import { opensInlineDirective } from '../directive-attributes.ts'
@@ -232,8 +233,8 @@ function opensLink(scan: string, escapings: readonly (InlineEscaping | undefined
 
 function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
   if (!startsRun(scan, index, escaped)) return false
-  const length = runLength(scan, index)
-  return new RegExp('(?<!`)`{' + length + '}(?!`)').test(scan.slice(index + length))
+  const opener = backtickRun(scan, index)
+  return closingBacktickRun(scan, index + opener, opener) !== undefined
 }
 
 function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet<number>): boolean {

src/markdown/parse/blocks.ts

@@ -9,9 +9,9 @@ import {
   listMarker,
   markerInterruptsParagraph,
   openingCodeFence,
+  openingHtmlBlock,
   setextHeadingLevel,
 } from '../commonmark-grammar.ts'
-import { openingHtmlBlock } from './html-blocks.ts'
 import { readLinkDefinitions } from './link-reference-definitions.ts'
 
 export type ClaimedConstruct = 'directive' | 'pipe-table'

src/markdown/parse/html-blocks.ts

@@ -1,28 +0,0 @@
-import { htmlConstructNames, htmlTagName, htmlTagSource } from '../commonmark-grammar.ts'
-
-export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
-
-type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
-
-// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
-const blockTagNames =
-  'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul'
-const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`)
-
-const conditions: HtmlBlockCondition[] = [
-  { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
-  { closer: /-->/, construct: htmlConstructNames.comment, interrupts: true, start: /^<!--/ },
-  { closer: /\?>/, construct: htmlConstructNames.processingInstruction, interrupts: true, start: /^<\?/ },
-  { closer: />/, construct: htmlConstructNames.declaration, interrupts: true, start: /^<![A-Za-z]/ },
-  { closer: /\]\]>/, construct: htmlConstructNames.cdata, interrupts: true, start: /^<!\[CDATA\[/ },
-  { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
-  { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
-]
-
-export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
-  for (const condition of conditions) {
-    if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
-    return { closer: condition.closer, construct: condition.construct ?? htmlTagName(line) }
-  }
-  return undefined
-}

src/markdown/parse/inline-content.ts

@@ -1,49 +1,44 @@
 import type { AdfNode } from '../../adf/document.ts'
-import { decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts'
+import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts'
+import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
 import { readEntityReference } from '../entity-references.ts'
 
-type CodeSpan = { end: number; text: string }
+// `text` holds the decoded content of the text node being built; `start` where its undecoded tail begins.
+type Run = { nodes: AdfNode[]; start: number; text: string }
 
 const hardBreakSpaces = / {2,}$/
 const trailingSpace = /[ \t]+$/
 
 export function parseInlineContent(source: string, path: ConvertErrorPath): Result<AdfNode[]> {
-  const nodes: AdfNode[] = []
-  let text = ''
-  let runStart = 0
+  const run: Run = { nodes: [], start: 0, text: '' }
   let index = 0
   while (index < source.length) {
     const character = source.charAt(index)
     if (character === '\\' && source.charAt(index + 1) === '\n') {
       // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash.
-      text = pushText(nodes, text + decodeTextEscapes(source.slice(runStart, index)))
-      nodes.push({ type: 'hardBreak' })
+      takeRun(run, source, index, index + 2, false)
+      pushNode(run, { type: 'hardBreak' })
       index += 2
-      runStart = index
       continue
     }
     if (character === '\n') {
-      const run = source.slice(runStart, index)
-      text += decodeTextEscapes(run.replace(trailingSpace, ''))
-      if (hardBreakSpaces.test(run)) {
-        text = pushText(nodes, text)
-        nodes.push({ type: 'hardBreak' })
-      } else text += ' '
+      const hard = hardBreakSpaces.test(source.slice(run.start, index))
+      takeRun(run, source, index, index + 1, true)
+      if (hard) pushNode(run, { type: 'hardBreak' })
+      else run.text += ' '
       index += 1
-      runStart = index
       continue
     }
     if (character === '`') {
       const span = readCodeSpan(source, index)
-      if (span !== undefined) {
-        text = pushText(nodes, text + decodeTextEscapes(source.slice(runStart, index)))
-        nodes.push({ marks: [{ type: 'code' }], text: span.text, type: 'text' })
-        index = span.end
-        runStart = index
+      if (span === undefined) {
+        index += backtickRun(source, index)
         continue
       }
-      index += backtickRun(source, index)
+      takeRun(run, source, index, span.end, false)
+      pushNode(run, { marks: [{ type: 'code' }], text: span.text, type: 'text' })
+      index = span.end
       continue
     }
     if (character === '<') {
@@ -54,36 +49,34 @@ export function parseInlineContent(source: string, path: ConvertErrorPath): Resu
       index += readEntityReference(source.slice(index))?.length ?? 1
       continue
     }
-    index += character === '\\' ? 2 : 1
+    index += backslashEscape(source, index) === undefined ? 1 : 2
   }
-  pushText(nodes, text + decodeTextEscapes(source.slice(runStart).replace(trailingSpace, '')))
-  return success(nodes)
+  takeRun(run, source, source.length, source.length, true)
+  pushText(run)
+  return success(run.nodes)
 }
 
-function pushText(nodes: AdfNode[], text: string): string {
-  if (text !== '') nodes.push({ text, type: 'text' })
-  return ''
+function takeRun(run: Run, source: string, end: number, resume: number, strip: boolean): void {
+  const raw = source.slice(run.start, end)
+  run.text += decodeTextEscapes(strip ? raw.replace(trailingSpace, '') : raw)
+  run.start = resume
 }
 
-function backtickRun(source: string, index: number): number {
-  let length = 0
-  while (source.charAt(index + length) === '`') length += 1
-  return length
+function pushText(run: Run): void {
+  if (run.text !== '') run.nodes.push({ text: run.text, type: 'text' })
+  run.text = ''
 }
 
-function readCodeSpan(source: string, index: number): CodeSpan | undefined {
+function pushNode(run: Run, node: AdfNode): void {
+  pushText(run)
+  run.nodes.push(node)
+}
+
+function readCodeSpan(source: string, index: number): { end: number; text: string } | undefined {
   const opener = backtickRun(source, index)
-  let cursor = index + opener
-  while (cursor < source.length) {
-    if (source.charAt(cursor) !== '`') {
-      cursor += 1
-      continue
-    }
-    const closer = backtickRun(source, cursor)
-    if (closer === opener) return { end: cursor + closer, text: codeSpanText(source.slice(index + opener, cursor)) }
-    cursor += closer
-  }
-  return undefined
+  const closer = closingBacktickRun(source, index + opener, opener)
+  if (closer === undefined) return undefined
+  return { end: closer + opener, text: codeSpanText(source.slice(index + opener, closer)) }
 }
 
 function codeSpanText(content: string): string {

src/markdown/parse/markdown-to-adf.test.ts

@@ -131,6 +131,8 @@ test('refuses the raw HTML no element mapping carries', () => {
   assert.equal(code(markdownToAdf('<![CDATA[x]]>\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<pre>\nx\n</pre>\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<div\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<?php\n')), 'unmappable-html')
   assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1])
   assert.equal(code(markdownToAdf('<div>\nx\n\n:::\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('<div>\n- x\n</div>\n')), 'unmappable-html')
@@ -257,6 +259,7 @@ test('decodes the entity references HTML5 names, and the numeric ones', () => {
   assert.deepEqual(content(markdownToAdf('&#0; &#xd800; &#9999999;\n')), [paragraph('\ufffd \ufffd \ufffd')])
   assert.deepEqual(content(markdownToAdf('&zzz; &amp &#; &\n')), [paragraph('&zzz; &amp &#; &')])
   assert.deepEqual(content(markdownToAdf('&#96;not code&#96;\n')), [paragraph('`not code`')])
+  assert.deepEqual(content(markdownToAdf('a&Tab;b&NewLine;c&nbsp;d&Aopf;e&verbar;f\n')), [paragraph('a\tb\nc d\u{1d538}e|f')])
 })
 
 test('reads a code span, its content literal', () => {