Read the leaf blocks, ahead of any inline parsing #31

Merged
lilleman merged 5 commits from tick-3b into main 2026-08-28 12:11:35 +02:00
8 changed files with 95 additions and 83 deletions
Showing only changes of commit 05f53a2024 - Show all commits
+23 -28
View File
@@ -1,6 +1,6 @@
import assert from 'node:assert/strict' import assert from 'node:assert/strict'
import { readFileSync, readdirSync } from 'node:fs' import { readFileSync, readdirSync } from 'node:fs'
import { dirname, join } from 'node:path' import { basename, dirname, join } from 'node:path'
import test from 'node:test' import test from 'node:test'
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
@@ -36,6 +36,21 @@ function names(root: string, extension: string): string[] {
.sort() .sort()
} }
// One kind's fixture pairs, its two tests declared with them.
function pairedNames(root: string, first: string, second: string): string[] {
const kind = basename(root)
test(`${kind} pairs every ${first} with a ${second}`, () => {
assert.deepEqual(names(root, first), names(root, second))
})
test(`${kind} holds fixtures`, () => {
assert.ok(names(root, first).length > 0)
})
return names(root, first)
}
function corpusJsonPaths(): string[] { function corpusJsonPaths(): string[] {
return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true }) return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true })
.filter((name) => name.endsWith('.json')) .filter((name) => name.endsWith('.json'))
@@ -43,6 +58,10 @@ function corpusJsonPaths(): string[] {
.sort() .sort()
} }
test('every corpus directory is a kind the runner reads', () => {
assert.deepEqual(directoryNames(corpusRoot), ['errors', 'normalization', 'round-trip', 'unspellable'])
})
test('every round-trip directory emits', () => { test('every round-trip directory emits', () => {
assert.deepEqual(directoryNames(roundTripRoot), [...emittingDirectories].sort()) assert.deepEqual(directoryNames(roundTripRoot), [...emittingDirectories].sort())
}) })
@@ -147,15 +166,7 @@ for (const directory of emittingDirectories) {
} }
} }
test('unspellable pairs every .json with an .error', () => { for (const name of pairedNames(unspellableRoot, '.json', '.error')) {
assert.deepEqual(names(unspellableRoot, '.json'), names(unspellableRoot, '.error'))
})
test('unspellable holds fixtures', () => {
assert.ok(names(unspellableRoot, '.json').length > 0)
})
for (const name of names(unspellableRoot, '.json')) {
test(`unspellable/${name} is refused with the error it names`, () => { test(`unspellable/${name} is refused with the error it names`, () => {
const parsed: unknown = JSON.parse(readFileSync(join(unspellableRoot, `${name}.json`), 'utf8')) const parsed: unknown = JSON.parse(readFileSync(join(unspellableRoot, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`) assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`)
@@ -165,15 +176,7 @@ for (const name of names(unspellableRoot, '.json')) {
}) })
} }
test('normalization pairs every .md with a .json', () => { for (const name of pairedNames(normalizationRoot, '.md', '.json')) {
assert.deepEqual(names(normalizationRoot, '.md'), names(normalizationRoot, '.json'))
})
test('normalization holds fixtures', () => {
assert.ok(names(normalizationRoot, '.md').length > 0)
})
for (const name of names(normalizationRoot, '.md')) {
test(`normalization/${name} parses to the document beside it`, () => { test(`normalization/${name} parses to the document beside it`, () => {
const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8')) const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`) assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`)
@@ -183,15 +186,7 @@ for (const name of names(normalizationRoot, '.md')) {
}) })
} }
test('errors pairs every .md with an .error', () => { for (const name of pairedNames(errorsRoot, '.md', '.error')) {
assert.deepEqual(names(errorsRoot, '.md'), names(errorsRoot, '.error'))
})
test('errors holds fixtures', () => {
assert.ok(names(errorsRoot, '.md').length > 0)
})
for (const name of names(errorsRoot, '.md')) {
test(`errors/${name} is refused with the error it names`, () => { test(`errors/${name} is refused with the error it names`, () => {
const result = markdownToAdf(readFileSync(join(errorsRoot, `${name}.md`), 'utf8')) const result = markdownToAdf(readFileSync(join(errorsRoot, `${name}.md`), 'utf8'))
assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '') assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '')
+1
View File
@@ -29,6 +29,7 @@ type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; deli
const delimiters = ['*', '_', '`', '~'] const delimiters = ['*', '_', '`', '~']
// The `:` keeps a `[label]: url` line escaped: unescaped, the parser swallows it as a link reference definition.
const followsLinkText = /[([:]/ const followsLinkText = /[([:]/
export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine {
+7 -9
View File
@@ -2,21 +2,14 @@ import assert from 'node:assert/strict'
import test from 'node:test' import test from 'node:test'
import type { LinkDefinition } from './link-reference-definitions.ts' import type { LinkDefinition } from './link-reference-definitions.ts'
import type { ParsedBlocks } from './blocks.ts'
import { parseBlocks } from './blocks.ts' import { parseBlocks } from './blocks.ts'
function walk(markdown: string): ParsedBlocks {
const result = parseBlocks(markdown)
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
return result.value
}
function definitions(markdown: string): [string, LinkDefinition][] { function definitions(markdown: string): [string, LinkDefinition][] {
return [...walk(markdown).definitions] return [...parseBlocks(markdown).definitions]
} }
function kinds(markdown: string): string[] { function kinds(markdown: string): string[] {
return walk(markdown).blocks.map((block) => block.kind) return parseBlocks(markdown).blocks.map((block) => block.kind)
} }
test('keeps the link reference definitions a paragraph gives up, the first of a label winning', () => { test('keeps the link reference definitions a paragraph gives up, the first of a label winning', () => {
@@ -51,3 +44,8 @@ test('swallows an HTML block to the end condition its start sets', () => {
assert.deepEqual(kinds('<div>\nx\n'), ['html']) assert.deepEqual(kinds('<div>\nx\n'), ['html'])
assert.deepEqual(kinds('Part.\n<div>\n'), ['paragraph', 'html']) assert.deepEqual(kinds('Part.\n<div>\n'), ['paragraph', 'html'])
}) })
test('carries a claimed line as the block it opens, the refusal the node layer builds', () => {
assert.deepEqual(kinds(':::\nPart.\n'), ['claim', 'paragraph'])
assert.deepEqual(kinds('Part.\n| x |\n'), ['paragraph', 'claim'])
})
+36 -28
View File
@@ -1,14 +1,16 @@
import type { LinkDefinition } from './link-reference-definitions.ts' import type { LinkDefinition } from './link-reference-definitions.ts'
import type { OpenHtmlBlock } from './html-blocks.ts' import type { OpenHtmlBlock } from './html-blocks.ts'
import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts' import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts'
import { failure, success, type Result } from '../../result.ts'
import { openingHtmlBlock } from './html-blocks.ts' import { openingHtmlBlock } from './html-blocks.ts'
import { readLinkDefinitions } from './link-reference-definitions.ts' import { readLinkDefinitions } from './link-reference-definitions.ts'
export type ClaimedConstruct = 'directive' | 'pipe-table'
export type LeafBlock = export type LeafBlock =
| { construct: ClaimedConstruct; kind: 'claim' }
| { construct: string; kind: 'html' }
| { kind: 'code'; language: string; text: string } | { kind: 'code'; language: string; text: string }
| { kind: 'heading'; level: number; text: string } | { kind: 'heading'; level: number; text: string }
| { kind: 'html'; name: string }
| { kind: 'paragraph'; text: string } | { kind: 'paragraph'; text: string }
| { kind: 'rule' } | { kind: 'rule' }
@@ -21,41 +23,51 @@ const indentedCodeColumns = 4
const largestOpenerIndentation = 3 const largestOpenerIndentation = 3
const tabStop = 4 const tabStop = 4
export function parseBlocks(markdown: string): Result<ParsedBlocks> { export function parseBlocks(markdown: string): ParsedBlocks {
const lines = normalizeInput(markdown).split('\n') const lines = normalizeInput(markdown).split('\n')
const walk: Walk = { blocks: [], definitions: new Map(), paragraph: [] } const walk: Walk = { blocks: [], definitions: new Map(), paragraph: [] }
let index = 0 let index = 0
while (index < lines.length) { while (index < lines.length) {
const line = lines[index] ?? '' const line = lines[index] ?? ''
index += 1
if (blankLine.test(line)) { if (blankLine.test(line)) {
closeParagraph(walk) closeParagraph(walk)
index += 1
continue continue
} }
if (leadingColumns(line) >= indentedCodeColumns && walk.paragraph.length === 0) { if (leadingColumns(line) >= indentedCodeColumns && walk.paragraph.length === 0) {
index = readIndentedCode(walk, lines, index - 1) index = readIndentedCode(walk, lines, index)
continue continue
} }
const opener = removeColumns(line, largestOpenerIndentation) const opened = openBlock(walk, lines, index, line)
const claim = claimedLine(walk, opener) if (opened !== undefined) {
if (claim !== undefined) return claim index = opened
if (readLineBlock(walk, opener)) continue
const fence = openingCodeFence(opener)
if (fence !== undefined) {
closeParagraph(walk)
index = readFencedCode(walk, lines, index, fence, leadingColumns(line))
continue
}
const html = openingHtmlBlock(opener, walk.paragraph.length > 0)
if (html !== undefined) {
closeParagraph(walk)
index = readHtmlBlock(walk, lines, index - 1, html)
continue continue
} }
walk.paragraph.push(line.replace(/^[ \t]+/, '')) walk.paragraph.push(line.replace(/^[ \t]+/, ''))
index += 1
} }
closeParagraph(walk) closeParagraph(walk)
return success({ blocks: walk.blocks, definitions: walk.definitions }) return { blocks: walk.blocks, definitions: walk.definitions }
}
function openBlock(walk: Walk, lines: readonly string[], index: number, line: string): number | undefined {
const opener = removeColumns(line, largestOpenerIndentation)
const claimed = claimedConstruct(opener)
if (claimed !== undefined) {
closeParagraph(walk)
walk.blocks.push({ construct: claimed, kind: 'claim' })
return index + 1
}
if (readLineBlock(walk, opener)) return index + 1
const fence = openingCodeFence(opener)
if (fence !== undefined) {
closeParagraph(walk)
return readFencedCode(walk, lines, index + 1, fence, leadingColumns(line))
}
const html = openingHtmlBlock(opener, walk.paragraph.length > 0)
if (html === undefined) return undefined
closeParagraph(walk)
return readHtmlBlock(walk, lines, index, html)
} }
// The document's last line ending closes its line rather than opening an empty one. // The document's last line ending closes its line rather than opening an empty one.
@@ -66,13 +78,9 @@ function normalizeInput(markdown: string): string {
.replace(/\n$/, '') .replace(/\n$/, '')
} }
function claimedLine(walk: Walk, opener: string): Result<ParsedBlocks> | undefined { function claimedConstruct(opener: string): ClaimedConstruct | undefined {
const directive = claimsDirectiveLine(opener) if (claimsDirectiveLine(opener)) return 'directive'
if (!directive && !claimsPipeLine(opener)) return undefined return claimsPipeLine(opener) ? 'pipe-table' : undefined
closeParagraph(walk)
const path = ['content', walk.blocks.length]
if (directive) return failure('malformed-directive', 'the line claims a directive and parses as none', path)
return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path)
} }
// A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. // A setext underline over a paragraph the definitions emptied is no heading: it opens the next block.
@@ -139,7 +147,7 @@ function readHtmlBlock(walk: Walk, lines: readonly string[], start: number, html
index += 1 index += 1
if (html.closer !== undefined && html.closer.test(line)) break if (html.closer !== undefined && html.closer.test(line)) break
} }
walk.blocks.push({ kind: 'html', name: html.name }) walk.blocks.push({ construct: html.construct, kind: 'html' })
return index return index
} }
+10 -10
View File
@@ -1,6 +1,6 @@
export type OpenHtmlBlock = { closer: RegExp | undefined; name: string } export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
type HtmlBlockCondition = { closer: RegExp | undefined; interrupts: boolean; name: string | undefined; start: RegExp } type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. // CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
const blockTagNames = const blockTagNames =
@@ -10,19 +10,19 @@ const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \
const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/ const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/
const conditions: HtmlBlockCondition[] = [ const conditions: HtmlBlockCondition[] = [
{ closer: /<\/(?:pre|script|style|textarea)>/i, interrupts: true, name: undefined, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i }, { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i },
{ closer: /-->/, interrupts: true, name: 'an HTML comment', start: /^<!--/ }, { closer: /-->/, construct: 'an HTML comment', interrupts: true, start: /^<!--/ },
{ closer: /\?>/, interrupts: true, name: 'an HTML processing instruction', start: /^<\?/ }, { closer: /\?>/, construct: 'an HTML processing instruction', interrupts: true, start: /^<\?/ },
{ closer: />/, interrupts: true, name: 'an HTML declaration', start: /^<![A-Za-z]/ }, { closer: />/, construct: 'an HTML declaration', interrupts: true, start: /^<![A-Za-z]/ },
{ closer: /\]\]>/, interrupts: true, name: 'a CDATA section', start: /^<!\[CDATA\[/ }, { closer: /\]\]>/, construct: 'a CDATA section', interrupts: true, start: /^<!\[CDATA\[/ },
{ closer: undefined, interrupts: true, name: undefined, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') }, { closer: undefined, construct: undefined, interrupts: true, start: new RegExp(`^</?(?:${blockTagNames})(?:[ \\t>]|/>|$)`, 'i') },
{ closer: undefined, interrupts: false, name: undefined, start: completeTag }, { closer: undefined, construct: undefined, interrupts: false, start: completeTag },
] ]
export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined { export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlBlock | undefined {
for (const condition of conditions) { for (const condition of conditions) {
if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue if ((interrupting && !condition.interrupts) || !condition.start.test(line)) continue
return { closer: condition.closer, name: condition.name ?? line.replace(tagName, '<$1>') } return { closer: condition.closer, construct: condition.construct ?? line.replace(tagName, '<$1>') }
} }
return undefined return undefined
} }
@@ -107,6 +107,7 @@ test('refuses the raw HTML no element mapping carries', () => {
assert.equal(code(markdownToAdf('<pre>\nx\n</pre>\n')), 'unmappable-html') assert.equal(code(markdownToAdf('<pre>\nx\n</pre>\n')), 'unmappable-html')
assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html') assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html')
assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1]) assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1])
assert.equal(code(markdownToAdf('<div>\nx\n\n:::\n')), 'unmappable-html')
}) })
test('swallows an HTML block ahead of the claim a line inside it would make', () => { test('swallows an HTML block ahead of the claim a line inside it would make', () => {
+9 -5
View File
@@ -1,13 +1,11 @@
import type { AdfDocument, AdfNode } from '../../adf/document.ts' import type { AdfDocument, AdfNode } from '../../adf/document.ts'
import type { LeafBlock } from './blocks.ts' import type { ClaimedConstruct, LeafBlock } from './blocks.ts'
import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
import { parseBlocks } from './blocks.ts' import { parseBlocks } from './blocks.ts'
export function markdownToAdf(markdown: string): Result<AdfDocument> { export function markdownToAdf(markdown: string): Result<AdfDocument> {
const parsed = parseBlocks(markdown)
if (!parsed.ok) return parsed
const content: AdfNode[] = [] const content: AdfNode[] = []
for (const [index, block] of parsed.value.blocks.entries()) { for (const [index, block] of parseBlocks(markdown).blocks.entries()) {
const node = blockNode(block, ['content', index]) const node = blockNode(block, ['content', index])
if (!node.ok) return node if (!node.ok) return node
content.push(node.value) content.push(node.value)
@@ -16,13 +14,19 @@ export function markdownToAdf(markdown: string): Result<AdfDocument> {
} }
function blockNode(block: LeafBlock, path: ConvertErrorPath): Result<AdfNode> { function blockNode(block: LeafBlock, path: ConvertErrorPath): Result<AdfNode> {
if (block.kind === 'claim') return claimFailure(block.construct, path)
if (block.kind === 'code') return success(codeBlockNode(block.language, block.text)) if (block.kind === 'code') return success(codeBlockNode(block.language, block.text))
if (block.kind === 'heading') return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) if (block.kind === 'heading') return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text))
if (block.kind === 'html') return failure('unmappable-html', `no element mapping carries ${block.name}`, path) if (block.kind === 'html') return failure('unmappable-html', `no ADF node carries ${block.construct}`, path)
if (block.kind === 'paragraph') return success(withContent({ type: 'paragraph' }, block.text)) if (block.kind === 'paragraph') return success(withContent({ type: 'paragraph' }, block.text))
return success({ type: 'rule' }) return success({ type: 'rule' })
} }
function claimFailure(construct: ClaimedConstruct, path: ConvertErrorPath): Result<AdfNode> {
if (construct === 'directive') return failure('malformed-directive', 'the line claims a directive and parses as none', path)
return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path)
}
function codeBlockNode(language: string, text: string): AdfNode { function codeBlockNode(language: string, text: string): AdfNode {
const node: AdfNode = language === '' ? { type: 'codeBlock' } : { attrs: { language }, type: 'codeBlock' } const node: AdfNode = language === '' ? { type: 'codeBlock' } : { attrs: { language }, type: 'codeBlock' }
return text === '' ? node : { ...node, content: [{ text, type: 'text' }] } return text === '' ? node : { ...node, content: [{ text, type: 'text' }] }
+8 -3
View File
@@ -210,7 +210,10 @@ detail is settled at its own milestone.
rather than editing them. rather than editing them.
- [ ] **3c — The container blocks.** Blockquote, bullet and ordered list: the continuation a - [ ] **3c — The container blocks.** Blockquote, bullet and ordered list: the continuation a
marker's width sets, lazy continuation, and the tightness ADF does not record — `> ` marker's width sets, lazy continuation, and the tightness ADF does not record — `> `
repeated being two bytes a level, so this is the cheapest way to reach §11's 500. repeated being two bytes a level, so this is the cheapest way to reach §11's 500. 3b's leaf
readers scan the physical line themselves, so a container re-cuts the walk rather than adding
to it: the open containers' prefix comes off the line first and the readers take one line at a
time, `LeafBlock` renamed with the union they join.
**Settled** (the maintainer, 2026-08-27): a claimed line ends lazy continuation, so a **Settled** (the maintainer, 2026-08-27): a claimed line ends lazy continuation, so a
closing fence on the line after a blockquote's open paragraph closes its container instead closing fence on the line after a blockquote's open paragraph closes its container instead
of continuing the paragraph CommonMark would fold it into. Claiming at block level is of continuing the paragraph CommonMark would fold it into. Claiming at block level is
@@ -244,7 +247,8 @@ detail is settled at its own milestone.
- [ ] **3f — The directive grammar.** The three forms — inline `:name[content]{attrs}`, - [ ] **3f — The directive grammar.** The three forms — inline `:name[content]{attrs}`,
container `:::name arg {attrs}`, leaf `::name arg {attrs}` — the attribute grammar with container `:::name arg {attrs}`, leaf `::name arg {attrs}` — the attribute grammar with
its quoting and escapes, the fence-length and nesting rules, and the malformed list its quoting and escapes, the fence-length and nesting rules, and the malformed list
`spec/flavour.md` spells, each a named error. `spec/flavour.md` spells, each a named error. `corpus.test.ts`'s `fenceNestingFault` stays a
second reading of the fence rule over emitted bytes: the double entry is the check.
**Settled** (the maintainer, 2026-08-27): the code span, the entity and raw HTML bind **Settled** (the maintainer, 2026-08-27): the code span, the entity and raw HTML bind
first in input, as 2e3 already assumed of the emitted side — a raw `` ` ``, `&`, `<` or first in input, as 2e3 already assumed of the emitted side — a raw `` ` ``, `&`, `<` or
`|` inside `{attrs}` breaks the directive and is a named error, the author writing the `|` inside `{attrs}` breaks the directive and is a named error, the author writing the
@@ -281,7 +285,8 @@ detail is settled at its own milestone.
named error, a carry inside a mark spelling another, and the three carve-outs' escapes named error, a carry inside a mark spelling another, and the three carve-outs' escapes
reading as the literal text they hold. reading as the literal text they hold.
- [ ] **3k — The CommonMark spec suite.** Checked in at `corpus/commonmark-spec/`, pinned to - [ ] **3k — The CommonMark spec suite.** Checked in at `corpus/commonmark-spec/`, pinned to
the version it ships, `corpus/README.md` gaining the kind. the version it ships — the one `html-blocks.ts` names for its start conditions —
`corpus/README.md` gaining the kind.
**Settled** (the maintainer, 2026-08-27): three checks an example must pass, the reference **Settled** (the maintainer, 2026-08-27): three checks an example must pass, the reference
HTML each ships read as corpus data — which adds no format and no direction (§1). §2's HTML each ships read as corpus data — which adds no format and no direction (§1). §2's
canonical fixpoint: a named error, or markdown that parses and emits to itself byte for canonical fixpoint: a named error, or markdown that parses and emits to itself byte for