diff --git a/src/corpus.test.ts b/src/corpus.test.ts index 38d67cd..9a38e98 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import { readFileSync, readdirSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import test from 'node:test' import { fileURLToPath } from 'node:url' @@ -36,6 +36,21 @@ function names(root: string, extension: string): string[] { .sort() } +// One kind's fixture pairs, its two tests declared with them. +function pairedNames(root: string, first: string, second: string): string[] { + const kind = basename(root) + + test(`${kind} pairs every ${first} with a ${second}`, () => { + assert.deepEqual(names(root, first), names(root, second)) + }) + + test(`${kind} holds fixtures`, () => { + assert.ok(names(root, first).length > 0) + }) + + return names(root, first) +} + function corpusJsonPaths(): string[] { return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true }) .filter((name) => name.endsWith('.json')) @@ -43,6 +58,10 @@ function corpusJsonPaths(): string[] { .sort() } +test('every corpus directory is a kind the runner reads', () => { + assert.deepEqual(directoryNames(corpusRoot), ['errors', 'normalization', 'round-trip', 'unspellable']) +}) + test('every round-trip directory emits', () => { assert.deepEqual(directoryNames(roundTripRoot), [...emittingDirectories].sort()) }) @@ -147,15 +166,7 @@ for (const directory of emittingDirectories) { } } -test('unspellable pairs every .json with an .error', () => { - assert.deepEqual(names(unspellableRoot, '.json'), names(unspellableRoot, '.error')) -}) - -test('unspellable holds fixtures', () => { - assert.ok(names(unspellableRoot, '.json').length > 0) -}) - -for (const name of names(unspellableRoot, '.json')) { +for (const name of pairedNames(unspellableRoot, '.json', '.error')) { test(`unspellable/${name} is refused with the error it names`, () => { const parsed: unknown = JSON.parse(readFileSync(join(unspellableRoot, `${name}.json`), 'utf8')) assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`) @@ -165,15 +176,7 @@ for (const name of names(unspellableRoot, '.json')) { }) } -test('normalization pairs every .md with a .json', () => { - assert.deepEqual(names(normalizationRoot, '.md'), names(normalizationRoot, '.json')) -}) - -test('normalization holds fixtures', () => { - assert.ok(names(normalizationRoot, '.md').length > 0) -}) - -for (const name of names(normalizationRoot, '.md')) { +for (const name of pairedNames(normalizationRoot, '.md', '.json')) { test(`normalization/${name} parses to the document beside it`, () => { const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8')) assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`) @@ -183,15 +186,7 @@ for (const name of names(normalizationRoot, '.md')) { }) } -test('errors pairs every .md with an .error', () => { - assert.deepEqual(names(errorsRoot, '.md'), names(errorsRoot, '.error')) -}) - -test('errors holds fixtures', () => { - assert.ok(names(errorsRoot, '.md').length > 0) -}) - -for (const name of names(errorsRoot, '.md')) { +for (const name of pairedNames(errorsRoot, '.md', '.error')) { test(`errors/${name} is refused with the error it names`, () => { const result = markdownToAdf(readFileSync(join(errorsRoot, `${name}.md`), 'utf8')) assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '') diff --git a/src/markdown/emit/line-escaping.ts b/src/markdown/emit/line-escaping.ts index 5591e4b..c02fc64 100644 --- a/src/markdown/emit/line-escaping.ts +++ b/src/markdown/emit/line-escaping.ts @@ -29,6 +29,7 @@ type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; deli const delimiters = ['*', '_', '`', '~'] +// The `:` keeps a `[label]: url` line escaped: unescaped, the parser swallows it as a link reference definition. const followsLinkText = /[([:]/ export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { diff --git a/src/markdown/parse/blocks.test.ts b/src/markdown/parse/blocks.test.ts index 96966ee..1bfdb61 100644 --- a/src/markdown/parse/blocks.test.ts +++ b/src/markdown/parse/blocks.test.ts @@ -2,21 +2,14 @@ import assert from 'node:assert/strict' import test from 'node:test' import type { LinkDefinition } from './link-reference-definitions.ts' -import type { ParsedBlocks } from './blocks.ts' import { parseBlocks } from './blocks.ts' -function walk(markdown: string): ParsedBlocks { - const result = parseBlocks(markdown) - assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`) - return result.value -} - function definitions(markdown: string): [string, LinkDefinition][] { - return [...walk(markdown).definitions] + return [...parseBlocks(markdown).definitions] } function kinds(markdown: string): string[] { - return walk(markdown).blocks.map((block) => block.kind) + return parseBlocks(markdown).blocks.map((block) => block.kind) } test('keeps the link reference definitions a paragraph gives up, the first of a label winning', () => { @@ -51,3 +44,8 @@ test('swallows an HTML block to the end condition its start sets', () => { assert.deepEqual(kinds('
\nx\n'), ['html']) assert.deepEqual(kinds('Part.\n
\n'), ['paragraph', 'html']) }) + +test('carries a claimed line as the block it opens, the refusal the node layer builds', () => { + assert.deepEqual(kinds(':::\nPart.\n'), ['claim', 'paragraph']) + assert.deepEqual(kinds('Part.\n| x |\n'), ['paragraph', 'claim']) +}) diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index 2af8cf5..01a8ce6 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -1,14 +1,16 @@ import type { LinkDefinition } from './link-reference-definitions.ts' import type { OpenHtmlBlock } from './html-blocks.ts' import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts' -import { failure, success, type Result } from '../../result.ts' import { openingHtmlBlock } from './html-blocks.ts' import { readLinkDefinitions } from './link-reference-definitions.ts' +export type ClaimedConstruct = 'directive' | 'pipe-table' + export type LeafBlock = + | { construct: ClaimedConstruct; kind: 'claim' } + | { construct: string; kind: 'html' } | { kind: 'code'; language: string; text: string } | { kind: 'heading'; level: number; text: string } - | { kind: 'html'; name: string } | { kind: 'paragraph'; text: string } | { kind: 'rule' } @@ -21,41 +23,51 @@ const indentedCodeColumns = 4 const largestOpenerIndentation = 3 const tabStop = 4 -export function parseBlocks(markdown: string): Result { +export function parseBlocks(markdown: string): ParsedBlocks { const lines = normalizeInput(markdown).split('\n') const walk: Walk = { blocks: [], definitions: new Map(), paragraph: [] } let index = 0 while (index < lines.length) { const line = lines[index] ?? '' - index += 1 if (blankLine.test(line)) { closeParagraph(walk) + index += 1 continue } if (leadingColumns(line) >= indentedCodeColumns && walk.paragraph.length === 0) { - index = readIndentedCode(walk, lines, index - 1) + index = readIndentedCode(walk, lines, index) continue } - const opener = removeColumns(line, largestOpenerIndentation) - const claim = claimedLine(walk, opener) - if (claim !== undefined) return claim - if (readLineBlock(walk, opener)) continue - const fence = openingCodeFence(opener) - if (fence !== undefined) { - closeParagraph(walk) - index = readFencedCode(walk, lines, index, fence, leadingColumns(line)) - continue - } - const html = openingHtmlBlock(opener, walk.paragraph.length > 0) - if (html !== undefined) { - closeParagraph(walk) - index = readHtmlBlock(walk, lines, index - 1, html) + const opened = openBlock(walk, lines, index, line) + if (opened !== undefined) { + index = opened continue } walk.paragraph.push(line.replace(/^[ \t]+/, '')) + index += 1 } closeParagraph(walk) - return success({ blocks: walk.blocks, definitions: walk.definitions }) + return { blocks: walk.blocks, definitions: walk.definitions } +} + +function openBlock(walk: Walk, lines: readonly string[], index: number, line: string): number | undefined { + const opener = removeColumns(line, largestOpenerIndentation) + const claimed = claimedConstruct(opener) + if (claimed !== undefined) { + closeParagraph(walk) + walk.blocks.push({ construct: claimed, kind: 'claim' }) + return index + 1 + } + if (readLineBlock(walk, opener)) return index + 1 + const fence = openingCodeFence(opener) + if (fence !== undefined) { + closeParagraph(walk) + return readFencedCode(walk, lines, index + 1, fence, leadingColumns(line)) + } + const html = openingHtmlBlock(opener, walk.paragraph.length > 0) + if (html === undefined) return undefined + closeParagraph(walk) + return readHtmlBlock(walk, lines, index, html) } // The document's last line ending closes its line rather than opening an empty one. @@ -66,13 +78,9 @@ function normalizeInput(markdown: string): string { .replace(/\n$/, '') } -function claimedLine(walk: Walk, opener: string): Result | undefined { - const directive = claimsDirectiveLine(opener) - if (!directive && !claimsPipeLine(opener)) return undefined - closeParagraph(walk) - const path = ['content', walk.blocks.length] - if (directive) return failure('malformed-directive', 'the line claims a directive and parses as none', path) - return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path) +function claimedConstruct(opener: string): ClaimedConstruct | undefined { + if (claimsDirectiveLine(opener)) return 'directive' + return claimsPipeLine(opener) ? 'pipe-table' : undefined } // A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. @@ -139,7 +147,7 @@ function readHtmlBlock(walk: Walk, lines: readonly string[], start: number, html index += 1 if (html.closer !== undefined && html.closer.test(line)) break } - walk.blocks.push({ kind: 'html', name: html.name }) + walk.blocks.push({ construct: html.construct, kind: 'html' }) return index } diff --git a/src/markdown/parse/html-blocks.ts b/src/markdown/parse/html-blocks.ts index 76a1e91..ad5ef61 100644 --- a/src/markdown/parse/html-blocks.ts +++ b/src/markdown/parse/html-blocks.ts @@ -1,6 +1,6 @@ -export type OpenHtmlBlock = { closer: RegExp | undefined; name: string } +export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string } -type HtmlBlockCondition = { closer: RegExp | undefined; interrupts: boolean; name: string | undefined; start: RegExp } +type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp } // CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. const blockTagNames = @@ -10,19 +10,19 @@ const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \ const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/ const conditions: HtmlBlockCondition[] = [ - { closer: /<\/(?:pre|script|style|textarea)>/i, interrupts: true, name: undefined, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i }, - { closer: /-->/, interrupts: true, name: 'an HTML comment', start: /^/, construct: 'an HTML comment', interrupts: true, start: /^