Check in the CommonMark spec suite and pin its exception list

AGENTS.md

@@ -57,8 +57,12 @@ Round-trip equality is a property tested over a corpus, not a claim made in pros
 why ~20 lines of own code cannot do the job, who maintains it, and what auditing it costs. So the
 CommonMark and HTML parsers are written in this repo. A table a standard fixes is data rather than
 a dependency: HTML5's 2125 semicolon-terminated character references ship packed in their own
-module, so entity decoding is complete without one. `devDependencies`: few, each earning its keep;
+module, so entity decoding is complete without one. The CommonMark spec suite is the same shape of
-they never reach a consumer.
+data and ships vendored at `corpus/commonmark-spec/` rather than as the `commonmark-spec` dev
+dependency — that package is CommonJS-only, and Renovate auto-bumping a spec version would silently
+point the vendored exception list's example numbers at a renumbered suite. A spec bump is a
+deliberate re-pin, exceptions re-derived by hand beside it. `devDependencies`: few, each earning its
+keep; they never reach a consumer.
 
 ## 6. The package contract
 

browser-tests/run.js

@@ -72,6 +72,7 @@ assert.deepEqual(
   readdirSync(corpusRoot, { withFileTypes: true })
     .filter((entry) => entry.isDirectory())
     .map((entry) => entry.name)
+    .filter((name) => name !== 'commonmark-spec')
     .sort(),
   ['errors', 'normalization', 'round-trip'],
   'a corpus kind the browser leg does not convert',

corpus/README.md

@@ -12,5 +12,13 @@ One directory per contract kind, each landing with its milestone:
   pins which error.
 - `real-payloads/` — `<name>.json`: sanitized live ADF, round-tripped ADF→markdown→ADF. No
   expected markdown.
+- `commonmark-spec/` — `spec.json` (the pinned CommonMark suite, read as corpus data per
+  AGENTS.md §1) and `exceptions.json` (examples a documented divergence exempts, keyed by
+  `check` and `example` with a `reason`). Three checks an example must pass: the §2 canonical
+  fixpoint — a named error, or markdown that parses and emits back to itself; the reference HTML's
+  text, tags stripped and entities decoded, against the parsed document's concatenated text; and a
+  count of the CommonMark subset's elements against the nodes and marks they map to.
 
-JSON is editor-normal (AGENTS.md §2), two-space indent, keys sorted.
+JSON is editor-normal (AGENTS.md §2), two-space indent, keys sorted. `spec.json` is the vendored,
+upstream machine-readable suite (CommonMark 0.31.2, CC-BY-SA-4.0, © John MacFarlane) and is not
+re-serialized by the corpus gate.

corpus/commonmark-spec/exceptions.json

@@ -0,0 +1,322 @@
+[
+  {
+    "check": "text",
+    "example": 39,
+    "reason": "`
` decodes to a newline kept literal in text; CommonMark reads it a soft break."
+  },
+  {
+    "check": "fixpoint",
+    "example": 196,
+    "reason": "The link title holds literal newlines no escape spells."
+  },
+  {
+    "check": "fixpoint",
+    "example": 202,
+    "reason": "The link destination holds a backslash the flavour cannot escape."
+  },
+  {
+    "check": "text",
+    "example": 300,
+    "reason": "A heading and the block after it join with no space where CommonMark breaks them."
+  },
+  {
+    "check": "count",
+    "example": 301,
+    "reason": "A list continuing past a marker change renders as two lists, the parser opens one."
+  },
+  {
+    "check": "count",
+    "example": 302,
+    "reason": "A list continuing past a marker change renders as two lists, the parser opens one."
+  },
+  {
+    "check": "fixpoint",
+    "example": 330,
+    "reason": "A paragraph opens with a code span whose backticks read back as a fence."
+  },
+  {
+    "check": "fixpoint",
+    "example": 331,
+    "reason": "A paragraph opens with a code span whose backticks read back as a fence."
+  },
+  {
+    "check": "fixpoint",
+    "example": 340,
+    "reason": "A paragraph opens with a code span whose backticks read back as a fence."
+  },
+  {
+    "check": "count",
+    "example": 369,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 373,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 389,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 393,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 394,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 395,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 399,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 404,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 406,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 407,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 408,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 409,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 410,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 411,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 413,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 414,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 415,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 417,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 418,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 422,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 424,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 425,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 426,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 427,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 428,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 429,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 430,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 431,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 432,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 433,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 461,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 463,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 464,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 465,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 466,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 468,
+    "reason": "CommonMark nests same-kind elements; the single mark collapses them to one."
+  },
+  {
+    "check": "count",
+    "example": 470,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 478,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 479,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 484,
+    "reason": "The empty link text stays literal text; CommonMark renders an empty link."
+  },
+  {
+    "check": "text",
+    "example": 484,
+    "reason": "An empty link text stays literal text; CommonMark renders an empty link."
+  },
+  {
+    "check": "count",
+    "example": 487,
+    "reason": "The empty link text stays literal text; CommonMark renders an empty link."
+  },
+  {
+    "check": "text",
+    "example": 487,
+    "reason": "An empty link text stays literal text; CommonMark renders an empty link."
+  },
+  {
+    "check": "fixpoint",
+    "example": 502,
+    "reason": "The link destination holds a backslash the flavour cannot escape."
+  },
+  {
+    "check": "count",
+    "example": 516,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 519,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 530,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 533,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 540,
+    "reason": "The case-folding shortcut reference is unresolved; CommonMark folds case and links."
+  },
+  {
+    "check": "text",
+    "example": 540,
+    "reason": "A case-folding shortcut reference is unresolved text; CommonMark folds case and links."
+  },
+  {
+    "check": "count",
+    "example": 554,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 558,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 559,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 638,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  },
+  {
+    "check": "count",
+    "example": 639,
+    "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
+  }
+]

src/commonmark-spec.test.ts

@@ -0,0 +1,288 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+import type { AdfNode } from './adf/document.ts'
+import { readEntityReference } from './markdown/entity-references.ts'
+import { adfToMarkdown } from './markdown/emit/adf-to-markdown.ts'
+import { markdownToAdf } from './markdown/parse/markdown-to-adf.ts'
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus', 'commonmark-spec')
+
+type Check = 'count' | 'fixpoint' | 'text'
+
+type SpecExample = { example: number; html: string; markdown: string; section: string }
+
+type Exception = { check: Check; example: number; reason: string }
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function isCheck(value: unknown): value is Check {
+  return value === 'count' || value === 'fixpoint' || value === 'text'
+}
+
+function isSpecExample(value: unknown): value is SpecExample {
+  if (!isRecord(value)) return false
+  return typeof value['example'] === 'number' && typeof value['html'] === 'string' && typeof value['markdown'] === 'string' && typeof value['section'] === 'string'
+}
+
+function isException(value: unknown): value is Exception {
+  return isRecord(value) && isCheck(value['check']) && typeof value['example'] === 'number' && typeof value['reason'] === 'string'
+}
+
+function readSpec(): SpecExample[] {
+  const parsed: unknown = JSON.parse(readFileSync(join(root, 'spec.json'), 'utf8'))
+  assert.ok(Array.isArray(parsed), 'spec.json is not an array of examples')
+  return parsed.map((value) => {
+    assert.ok(isSpecExample(value), 'spec.json holds an example with the wrong shape')
+    return value
+  })
+}
+
+function readExceptions(): Exception[] {
+  const parsed: unknown = JSON.parse(readFileSync(join(root, 'exceptions.json'), 'utf8'))
+  assert.ok(Array.isArray(parsed), 'exceptions.json is not an array')
+  return parsed.map((value, index) => {
+    assert.ok(isException(value), `exception ${index} is not an exception`)
+    return value
+  })
+}
+
+const spec = readSpec()
+const exceptions = readExceptions()
+const exceptionIndex = new Map(exceptions.map((entry) => [`${entry.example}:${entry.check}`, entry.reason]))
+
+test('the CommonMark spec suite is the version the parser names', () => {
+  assert.ok(spec.length > 600, `spec.json holds ${spec.length} examples, not a full suite`)
+})
+
+test('the exception list is unique per example and check', () => {
+  assert.equal(exceptionIndex.size, exceptions.length, 'one exception repeats an example and check another holds')
+})
+
+// A mark is counted once per text node it touches: 3e's collapse makes `*(*a*)*` one `em` against two `<em>`.
+const countKeys = ['a', 'blockquote', 'br', 'code', 'em', 'heading', 'hr', 'img', 'li', 'ol', 'pre', 'strong', 'ul']
+const nodeElement: Record<string, string> = {
+  blockquote: 'blockquote',
+  bulletList: 'ul',
+  codeBlock: 'pre',
+  hardBreak: 'br',
+  heading: 'heading',
+  listItem: 'li',
+  media: 'img',
+  mediaInline: 'img',
+  orderedList: 'ol',
+  rule: 'hr',
+}
+const markElement: Record<string, string> = { code: 'code', em: 'em', link: 'a', strong: 'strong' }
+const blockTags = new Set(['blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'li', 'ol', 'p', 'pre', 'ul'])
+
+function tagName(tag: string): string {
+  return tag.slice(1).replace(/^\//, '').split(/[\s/>]/)[0] ?? ''
+}
+
+function emptyCounts(): Record<string, number> {
+  return Object.fromEntries(countKeys.map((key) => [key, 0]))
+}
+
+function referenceCounts(html: string): Record<string, number> {
+  const counts = emptyCounts()
+  let inPre = false
+  for (let index = 0; index < html.length; index += 1) {
+    if (html[index] !== '<') continue
+    const close = html.indexOf('>', index)
+    if (close === -1) break
+    const tag = html.slice(index, close + 1)
+    if (tag.startsWith('</')) {
+      if (tagName(tag) === 'pre') inPre = false
+      index = close
+      continue
+    }
+    const name = tagName(tag)
+    if (name === 'pre') {
+      inPre = true
+      counts['pre'] = (counts['pre'] ?? 0) + 1
+    } else if (name === 'code' && inPre) {
+      // A code block's `<code>` is the `<pre>`'s body, already counted.
+    } else if (/^h[1-6]$/.test(name)) {
+      counts['heading'] = (counts['heading'] ?? 0) + 1
+    } else if (countKeys.includes(name)) {
+      counts[name] = (counts[name] ?? 0) + 1
+    }
+    index = close
+  }
+  return counts
+}
+
+function nodeCounts(document: AdfNode): Record<string, number> {
+  const counts = emptyCounts()
+  const pending: AdfNode[] = [document]
+  while (pending.length > 0) {
+    const node = pending.pop()
+    if (node === undefined) continue
+    if (node.text !== undefined) {
+      const seen = new Set<string>()
+      for (const mark of node.marks ?? []) {
+        const element = markElement[mark.type]
+        if (element !== undefined) seen.add(element)
+      }
+      for (const element of seen) counts[element] = (counts[element] ?? 0) + 1
+      continue
+    }
+    const element = nodeElement[node.type]
+    if (element !== undefined) counts[element] = (counts[element] ?? 0) + 1
+    pending.push(...(node.content ?? []))
+  }
+  return counts
+}
+
+function referenceText(html: string): string {
+  let out = ''
+  let preDepth = 0
+  let skipNewline = false
+  for (let index = 0; index < html.length; index += 1) {
+    const character = html[index]
+    if (character === '<') {
+      const close = html.indexOf('>', index)
+      if (close === -1) break
+      const tag = html.slice(index, close + 1)
+      const name = tagName(tag)
+      if (name === 'br') {
+        out += '\n'
+        skipNewline = true
+        index = close
+        continue
+      }
+      if (name === 'pre') {
+        if (tag.startsWith('</')) {
+          preDepth -= 1
+          out = out.replace(/\n$/, '')
+        } else {
+          preDepth += 1
+        }
+      }
+      index = close
+      continue
+    }
+    if (character === '\n') {
+      if (skipNewline) {
+        skipNewline = false
+        continue
+      }
+      if (preDepth > 0) {
+        out += '\n'
+        continue
+      }
+      let next = index + 1
+      while (next < html.length && (html[next] === '\n' || html[next] === ' ' || html[next] === '\t')) next += 1
+      if (next >= html.length) continue
+      if (html[next] === '<') {
+        const close = html.indexOf('>', next)
+        if (close === -1) continue
+        if (!blockTags.has(tagName(html.slice(next, close + 1)))) out += ' '
+      } else {
+        out += ' '
+      }
+      continue
+    }
+    skipNewline = false
+    const reference = readEntityReference(html, index)
+    if (reference !== undefined) {
+      out += reference.text
+      index += reference.length - 1
+      continue
+    }
+    out += character
+  }
+  return out
+}
+
+function concatenatedText(document: AdfNode): string {
+  const parts: string[] = []
+  const pending: AdfNode[] = [document]
+  while (pending.length > 0) {
+    const node = pending.pop()
+    if (node === undefined) continue
+    if (node.text !== undefined) {
+      parts.push(node.text)
+      continue
+    }
+    if (node.type === 'hardBreak') {
+      parts.push('\n')
+      continue
+    }
+    const content = node.content ?? []
+    for (let index = content.length - 1; index >= 0; index -= 1) {
+      const child = content[index]
+      if (child !== undefined) pending.push(child)
+    }
+  }
+  return parts.join('')
+}
+
+function fixpointRefused(example: SpecExample): string | undefined {
+  const parsed = markdownToAdf(example.markdown)
+  assert.ok(parsed.ok, `example ${example.example} parsed to no document, not an error`)
+  const emitted = adfToMarkdown(parsed.value)
+  if (!emitted.ok) return emitted.error.code
+  const again = markdownToAdf(emitted.value)
+  assert.ok(again.ok, `example ${example.example} emits markdown it cannot read back`)
+  assert.deepStrictEqual(again.value, parsed.value, `example ${example.example} does not hold its own round-trip`)
+  return undefined
+}
+
+function textMismatch(example: SpecExample): string | undefined {
+  const parsed = markdownToAdf(example.markdown)
+  assert.ok(parsed.ok)
+  const expected = referenceText(example.html)
+  const actual = concatenatedText(parsed.value)
+  return expected === actual ? undefined : `${JSON.stringify(expected)} against ${JSON.stringify(actual)}`
+}
+
+function countMismatch(example: SpecExample): string | undefined {
+  const parsed = markdownToAdf(example.markdown)
+  assert.ok(parsed.ok)
+  const expected = referenceCounts(example.html)
+  const actual = nodeCounts(parsed.value)
+  const names = countKeys.filter((key) => expected[key] !== actual[key])
+  return names.length === 0 ? undefined : names.map((name) => `${name} ${expected[name]}/${actual[name]}`).join(' ')
+}
+
+for (const example of spec) {
+  test(`CommonMark example ${example.example} => ${example.section}`, () => {
+    const parse = markdownToAdf(example.markdown)
+    if (!parse.ok) {
+      assert.equal(exceptionIndex.get(`${example.example}:fixpoint`), undefined, `example ${example.example} is a named error but files a fixpoint exception`)
+      return
+    }
+
+    const failures: Record<Check, string | undefined> = {
+      fixpoint: fixpointRefused(example),
+      text: textMismatch(example),
+      count: countMismatch(example),
+    }
+
+    for (const check of ['fixpoint', 'text', 'count'] as const) {
+      const filed = exceptionIndex.get(`${example.example}:${check}`)
+      const result = failures[check]
+      if (result === undefined) {
+        assert.equal(filed, undefined, `example ${example.example} passes its ${check} check but files an exception`)
+      } else {
+        assert.equal(typeof filed, 'string', `example ${example.example} ${check} check fails: ${result}`)
+      }
+    }
+  })
+}
+
+for (const entry of exceptions) {
+  test(`exception ${entry.example} ${entry.check} still diverges`, () => {
+    const example = spec.find((candidate) => candidate.example === entry.example)
+    assert.ok(example !== undefined, `exception ${entry.example} names no example in the suite`)
+    const failure = entry.check === 'fixpoint' ? fixpointRefused(example) : entry.check === 'text' ? textMismatch(example) : countMismatch(example)
+    assert.equal(typeof failure, 'string', `exception ${entry.example} ${entry.check} no longer diverges: ${entry.reason}`)
+  })
+}

src/corpus.test.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict'
 import { readFileSync, readdirSync } from 'node:fs'
-import { basename, dirname, join } from 'node:path'
+import { basename, dirname, join, sep } from 'node:path'
 import test from 'node:test'
 import { fileURLToPath } from 'node:url'
 
@@ -53,12 +53,13 @@ function pairedNames(root: string, first: string, second: string): string[] {
 function corpusJsonPaths(): string[] {
   return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true })
     .filter((name) => name.endsWith('.json'))
+    .filter((name) => !name.startsWith(`commonmark-spec${sep}`))
     .map((name) => join(corpusRoot, name))
     .sort()
 }
 
 test('every corpus directory is a kind the runner reads', () => {
-  assert.deepEqual(directoryNames(corpusRoot), ['errors', 'normalization', 'round-trip'])
+  assert.deepEqual(directoryNames(corpusRoot), ['commonmark-spec', 'errors', 'normalization', 'round-trip'])
 })
 
 test('every round-trip directory is a kind the runner reads', () => {