diff --git a/AGENTS.md b/AGENTS.md index 9d97f5a..721f3b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,8 +82,9 @@ closed list a consumer may switch exhaustively, the message free text, the path from the document root. Adding, removing or renaming a code is breaking, so a milestone meeting a new failure cause reuses a code where one fits; the list is complete at `0.1.0`. A code names the cause; where one cause recurs across node types, one code covers them all and `path` and `message` -say which. A cause the carry answers gets no code: a mark no spelling writes rides the carry with -its node. +say which. A claim code names the spelling claimed, never the node that spelling would have built: +a malformed `:::table` is a `malformed-directive`. A cause the carry answers gets no code: a mark no +spelling writes rides the carry with its node. ## 9. Release automation @@ -139,7 +140,8 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga unrepresentable. - `src/adf/` holds ADF's own knowledge and imports no format. Each format directory (`markdown/`, `html/`) parts into `emit/` (ADF→format) and `parse/` (format→ADF), its root holding what both - directions read. + directions read. A construct's reader lives in that root beside the regex the emitter escapes + against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`. - The attribute vocabulary is ADF's: `adf/` walks it and narrows each value to its kind, and a format spells the narrowed value. A spelling that re-checks the type is the check's second copy. - Explicit over implicit; descriptive names; no catch-all files (`utils`, `helpers`, `misc`); a diff --git a/corpus/errors/colon-run-alone.error b/corpus/errors/colon-run-alone.error new file mode 100644 index 0000000..4f35411 --- /dev/null +++ b/corpus/errors/colon-run-alone.error @@ -0,0 +1 @@ +malformed-directive diff --git a/corpus/errors/colon-run-alone.md b/corpus/errors/colon-run-alone.md new file mode 100644 index 0000000..2d2bfd3 --- /dev/null +++ b/corpus/errors/colon-run-alone.md @@ -0,0 +1 @@ +::: diff --git a/corpus/errors/html-comment.error b/corpus/errors/html-comment.error new file mode 100644 index 0000000..847e0f9 --- /dev/null +++ b/corpus/errors/html-comment.error @@ -0,0 +1 @@ +unmappable-html diff --git a/corpus/errors/html-comment.md b/corpus/errors/html-comment.md new file mode 100644 index 0000000..2069895 --- /dev/null +++ b/corpus/errors/html-comment.md @@ -0,0 +1 @@ + diff --git a/corpus/errors/pipe-row-alone.error b/corpus/errors/pipe-row-alone.error new file mode 100644 index 0000000..5bf92ee --- /dev/null +++ b/corpus/errors/pipe-row-alone.error @@ -0,0 +1 @@ +malformed-pipe-table diff --git a/corpus/errors/pipe-row-alone.md b/corpus/errors/pipe-row-alone.md new file mode 100644 index 0000000..87f79c6 --- /dev/null +++ b/corpus/errors/pipe-row-alone.md @@ -0,0 +1 @@ +| Part | Qty | diff --git a/corpus/normalization/indented-code.json b/corpus/normalization/indented-code.json new file mode 100644 index 0000000..76b0c09 --- /dev/null +++ b/corpus/normalization/indented-code.json @@ -0,0 +1,15 @@ +{ + "content": [ + { + "content": [ + { + "text": "SELECT id\nFROM part", + "type": "text" + } + ], + "type": "codeBlock" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/indented-code.md b/corpus/normalization/indented-code.md new file mode 100644 index 0000000..0fc6398 --- /dev/null +++ b/corpus/normalization/indented-code.md @@ -0,0 +1,2 @@ + SELECT id + FROM part diff --git a/corpus/normalization/setext-headings.json b/corpus/normalization/setext-headings.json new file mode 100644 index 0000000..784a149 --- /dev/null +++ b/corpus/normalization/setext-headings.json @@ -0,0 +1,30 @@ +{ + "content": [ + { + "attrs": { + "level": 1 + }, + "content": [ + { + "text": "Assembly", + "type": "text" + } + ], + "type": "heading" + }, + { + "attrs": { + "level": 2 + }, + "content": [ + { + "text": "Parts", + "type": "text" + } + ], + "type": "heading" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/setext-headings.md b/corpus/normalization/setext-headings.md new file mode 100644 index 0000000..18ddd28 --- /dev/null +++ b/corpus/normalization/setext-headings.md @@ -0,0 +1,5 @@ +Assembly +======== + +Parts +----- diff --git a/src/corpus.test.ts b/src/corpus.test.ts index a3cafbe..9a38e98 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -1,15 +1,18 @@ import assert from 'node:assert/strict' import { readFileSync, readdirSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import test from 'node:test' import { fileURLToPath } from 'node:url' import { adfToMarkdown } from './markdown/emit/adf-to-markdown.ts' import { isAdfDocument } from './adf/document.ts' import { isJsonValue } from './json-value.ts' +import { markdownToAdf } from './markdown/parse/markdown-to-adf.ts' import { serializeCanonicalJson } from './canonical-json.ts' const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus') +const errorsRoot = join(corpusRoot, 'errors') +const normalizationRoot = join(corpusRoot, 'normalization') const roundTripRoot = join(corpusRoot, 'round-trip') const unspellableRoot = join(corpusRoot, 'unspellable') @@ -33,6 +36,21 @@ function names(root: string, extension: string): string[] { .sort() } +// One kind's fixture pairs, its two tests declared with them. +function pairedNames(root: string, first: string, second: string): string[] { + const kind = basename(root) + + test(`${kind} pairs every ${first} with a ${second}`, () => { + assert.deepEqual(names(root, first), names(root, second)) + }) + + test(`${kind} holds fixtures`, () => { + assert.ok(names(root, first).length > 0) + }) + + return names(root, first) +} + function corpusJsonPaths(): string[] { return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true }) .filter((name) => name.endsWith('.json')) @@ -40,6 +58,10 @@ function corpusJsonPaths(): string[] { .sort() } +test('every corpus directory is a kind the runner reads', () => { + assert.deepEqual(directoryNames(corpusRoot), ['errors', 'normalization', 'round-trip', 'unspellable']) +}) + test('every round-trip directory emits', () => { assert.deepEqual(directoryNames(roundTripRoot), [...emittingDirectories].sort()) }) @@ -144,15 +166,7 @@ for (const directory of emittingDirectories) { } } -test('unspellable pairs every .json with an .error', () => { - assert.deepEqual(names(unspellableRoot, '.json'), names(unspellableRoot, '.error')) -}) - -test('unspellable holds fixtures', () => { - assert.ok(names(unspellableRoot, '.json').length > 0) -}) - -for (const name of names(unspellableRoot, '.json')) { +for (const name of pairedNames(unspellableRoot, '.json', '.error')) { test(`unspellable/${name} is refused with the error it names`, () => { const parsed: unknown = JSON.parse(readFileSync(join(unspellableRoot, `${name}.json`), 'utf8')) assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`) @@ -162,6 +176,24 @@ for (const name of names(unspellableRoot, '.json')) { }) } +for (const name of pairedNames(normalizationRoot, '.md', '.json')) { + test(`normalization/${name} parses to the document beside it`, () => { + const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8')) + assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`) + const result = markdownToAdf(readFileSync(join(normalizationRoot, `${name}.md`), 'utf8')) + assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`) + assert.deepEqual(result.value, expected) + }) +} + +for (const name of pairedNames(errorsRoot, '.md', '.error')) { + test(`errors/${name} is refused with the error it names`, () => { + const result = markdownToAdf(readFileSync(join(errorsRoot, `${name}.md`), 'utf8')) + assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '') + assert.equal(result.error.code, readFileSync(join(errorsRoot, `${name}.error`), 'utf8').trimEnd()) + }) +} + test('the corpus holds JSON to gate', () => { assert.ok(corpusJsonPaths().length > 0) }) diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index 0d9dec1..619c05c 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -12,17 +12,43 @@ const controlCharacter = new RegExp(`[${controlCharacterRange}]`) const entityReference = new RegExp(entityReferenceSource) const nullCharacter = new RegExp(nullCharacterSource) const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ -const firstCharacterOpeners = [/^#{1,6}(?:[ \t]|$)/, /^>/, /^[*+-](?:[ \t]|$)/, /^`{3,}/, /^~{3,}/, /^:{2,}/, /^\|/] +const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/ +const codeFenceOpener = /^(`{3,}|~{3,})/ +const directiveClaim = /^:{2,}(?:[A-Za-z0-9]|[ \t]*$)/ +const pipeClaim = /^\|/ +// A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. +const firstCharacterOpeners = [atxHeadingOpener, /^>/, /^[*+-](?:[ \t]|$)/, codeFenceOpener, /^:{2,}/, pipeClaim] const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] const orderedListOpener = /^(\d{1,9})[.)](?:[ \t]|$)/ -const setextUnderline = /^(?:=+|-+)$/ +const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u +export function atxHeading(line: string): { level: number; text: string } | undefined { + const hashes = atxHeadingOpener.exec(line)?.[1] + if (hashes === undefined) return undefined + const text = trimSpace(line.slice(hashes.length)) + return { level: hashes.length, text: trimSpace(text.replace(/(?:^|(?<=[ \t]))#+$/, '')) } +} + +export function claimsDirectiveLine(line: string): boolean { + return directiveClaim.test(line) +} + export function claimsLine(line: string, position: LinePosition): boolean { return escapesLineClaim(line, 0, position) || orderedListOpener.test(line) } +export function claimsPipeLine(line: string): boolean { + return pipeClaim.test(line) +} + +export function closingCodeFence(line: string, marker: string): boolean { + const closing = codeFenceOpener.exec(line)?.[1] + if (closing === undefined || closing.charAt(0) !== marker.charAt(0) || closing.length < marker.length) return false + return /^[ \t]*$/.test(line.slice(closing.length)) +} + export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean { if (offset === 0) { if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true @@ -60,6 +86,13 @@ export function isUnicodeWhitespace(character: string): boolean { return unicodeWhitespace.test(character) } +export function openingCodeFence(line: string): { info: string; marker: string } | undefined { + const marker = codeFenceOpener.exec(line)?.[1] + if (marker === undefined) return undefined + const info = trimSpace(line.slice(marker.length)) + return marker.startsWith('`') && info.includes('`') ? undefined : { info, marker } +} + export function opensBracketedAutolink(text: string): boolean { return bracketedAutolink.test(text) } @@ -68,6 +101,16 @@ export function opensHtmlConstruct(text: string): boolean { return htmlConstructs.some((construct) => construct.test(text)) } +export function setextHeadingLevel(line: string): number | undefined { + const underline = setextUnderline.exec(line)?.[1] + if (underline === undefined) return undefined + return underline.startsWith('=') ? 1 : 2 +} + export function startsEntityReference(text: string): boolean { return anchoredEntityReference.test(text) } + +export function trimSpace(text: string): string { + return text.replace(/^[ \t]+|[ \t]+$/g, '') +} diff --git a/src/markdown/emit/line-escaping.ts b/src/markdown/emit/line-escaping.ts index 5591e4b..c02fc64 100644 --- a/src/markdown/emit/line-escaping.ts +++ b/src/markdown/emit/line-escaping.ts @@ -29,6 +29,7 @@ type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; deli const delimiters = ['*', '_', '`', '~'] +// The `:` keeps a `[label]: url` line escaped: unescaped, the parser swallows it as a link reference definition. const followsLinkText = /[([:]/ export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { diff --git a/src/markdown/parse/blocks.test.ts b/src/markdown/parse/blocks.test.ts new file mode 100644 index 0000000..e542a2a --- /dev/null +++ b/src/markdown/parse/blocks.test.ts @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import type { LinkDefinition } from './link-reference-definitions.ts' +import { parseBlocks } from './blocks.ts' + +function definitions(markdown: string): [string, LinkDefinition][] { + return [...parseBlocks(markdown).definitions] +} + +function kinds(markdown: string): string[] { + return parseBlocks(markdown).blocks.map((block) => block.kind) +} + +test('keeps the link reference definitions a paragraph gives up, the first of a label winning', () => { + assert.deepEqual(definitions('[a]: /url\n'), [['a', { destination: '/url' }]]) + assert.deepEqual(definitions('[Foo Bar]:\n\n"Title"\n'), [['foo bar', { destination: 'the url', title: 'Title' }]]) + assert.deepEqual(definitions("[a]: /url 'One'\n[a]: /other (Two)\n[b]: /b\n"), [ + ['a', { destination: '/url', title: 'One' }], + ['b', { destination: '/b' }], + ]) + assert.deepEqual(definitions('[a\\]b]: /url\n'), [['a\\]b', { destination: '/url' }]]) + assert.deepEqual(definitions('[a]: /url(x)y\n'), [['a', { destination: '/url(x)y' }]]) + assert.deepEqual(definitions('[a]: /url\\(x\n'), [['a', { destination: '/url\\(x' }]]) + assert.deepEqual(definitions('[a]: <>\n'), [['a', { destination: '' }]]) + assert.deepEqual(definitions('[a]: /url "He said \\"hi\\""\n'), [['a', { destination: '/url', title: 'He said \\"hi\\"' }]]) + assert.deepEqual(definitions('[a]: /url\\\n[b]: /b\n'), [ + ['a', { destination: '/url\\' }], + ['b', { destination: '/b' }], + ]) + assert.deepEqual(definitions('[\u00a0a]: /one\n[a]: /two\n'), [ + ['\u00a0a', { destination: '/one' }], + ['a', { destination: '/two' }], + ]) +}) + +test('leaves the paragraph a line no definition spells', () => { + assert.deepEqual(definitions('[]: /url\n'), []) + assert.deepEqual(definitions('[ ]: /url\n'), []) + assert.deepEqual(definitions('[a]: closed>\n'), []) + assert.deepEqual(definitions('[a]: { + assert.deepEqual(kinds('
\nx\n\nPart.\n'), ['html', 'paragraph']) + assert.deepEqual(kinds('\nPart.\n'), ['html', 'paragraph']) + assert.deepEqual(kinds('
x
\nPart.\n'), ['html', 'paragraph']) + assert.deepEqual(kinds('
\nx\n'), ['html']) + assert.deepEqual(kinds('Part.\n
\n'), ['paragraph', 'html']) +}) + +test('carries a claimed line as the block it opens, the refusal the node layer builds', () => { + assert.deepEqual(kinds(':::\nPart.\n'), ['claim', 'paragraph']) + assert.deepEqual(kinds('Part.\n| x |\n'), ['paragraph', 'claim']) +}) diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts new file mode 100644 index 0000000..e73d1f4 --- /dev/null +++ b/src/markdown/parse/blocks.ts @@ -0,0 +1,187 @@ +import type { LinkDefinition } from './link-reference-definitions.ts' +import type { OpenHtmlBlock } from './html-blocks.ts' +import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts' +import { openingHtmlBlock } from './html-blocks.ts' +import { readLinkDefinitions } from './link-reference-definitions.ts' + +export type ClaimedConstruct = 'directive' | 'pipe-table' + +export type LeafBlock = + | { construct: ClaimedConstruct; kind: 'claim' } + | { construct: string; kind: 'html' } + | { kind: 'code'; language: string; text: string } + | { kind: 'heading'; level: number; text: string } + | { kind: 'paragraph'; text: string } + | { kind: 'rule' } + +export type ParsedBlocks = { blocks: LeafBlock[]; definitions: Map } + +type Walk = ParsedBlocks & { paragraph: string[] } + +const blankLine = /^[ \t]*$/ +const indentedCodeColumns = 4 +const largestOpenerIndentation = 3 +const tabStop = 4 + +export function parseBlocks(markdown: string): ParsedBlocks { + const lines = normalizeInput(markdown).split('\n') + const walk: Walk = { blocks: [], definitions: new Map(), paragraph: [] } + let index = 0 + while (index < lines.length) { + const line = lines[index] ?? '' + if (blankLine.test(line)) { + closeParagraph(walk) + index += 1 + continue + } + if (leadingColumns(line) >= indentedCodeColumns && walk.paragraph.length === 0) { + index = readIndentedCode(walk, lines, index) + continue + } + const opened = openBlock(walk, lines, index, line) + if (opened !== undefined) { + index = opened + continue + } + walk.paragraph.push(line.replace(/^[ \t]+/, '')) + index += 1 + } + closeParagraph(walk) + return { blocks: walk.blocks, definitions: walk.definitions } +} + +function openBlock(walk: Walk, lines: readonly string[], index: number, line: string): number | undefined { + const opener = removeColumns(line, largestOpenerIndentation) + const claimed = claimedConstruct(opener) + if (claimed !== undefined) { + closeParagraph(walk) + walk.blocks.push({ construct: claimed, kind: 'claim' }) + return index + 1 + } + if (readLineBlock(walk, opener)) return index + 1 + const fence = openingCodeFence(opener) + if (fence !== undefined) { + closeParagraph(walk) + return readFencedCode(walk, lines, index + 1, fence, leadingColumns(line)) + } + const html = openingHtmlBlock(opener, walk.paragraph.length > 0) + if (html === undefined) return undefined + closeParagraph(walk) + return readHtmlBlock(walk, lines, index, html) +} + +function normalizeInput(markdown: string): string { + return markdown + .replace(/\r\n?/g, '\n') + .replaceAll('\u0000', '\ufffd') + .replace(/\n$/, '') +} + +function claimedConstruct(opener: string): ClaimedConstruct | undefined { + if (claimsDirectiveLine(opener)) return 'directive' + return claimsPipeLine(opener) ? 'pipe-table' : undefined +} + +// A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. +function readLineBlock(walk: Walk, opener: string): boolean { + const level = walk.paragraph.length === 0 ? undefined : setextHeadingLevel(opener) + if (level !== undefined) { + const text = takeParagraph(walk) + if (text !== undefined) { + walk.blocks.push({ kind: 'heading', level, text }) + return true + } + } + if (isThematicBreak(opener)) { + closeParagraph(walk) + walk.blocks.push({ kind: 'rule' }) + return true + } + const heading = atxHeading(opener) + if (heading === undefined) return false + closeParagraph(walk) + walk.blocks.push({ kind: 'heading', level: heading.level, text: heading.text }) + return true +} + +function readFencedCode(walk: Walk, lines: readonly string[], start: number, fence: { info: string; marker: string }, indentation: number): number { + const collected: string[] = [] + let index = start + while (index < lines.length) { + const line = lines[index] ?? '' + index += 1 + if (closingCodeFence(removeColumns(line, largestOpenerIndentation), fence.marker)) break + collected.push(removeColumns(line, indentation)) + } + walk.blocks.push({ kind: 'code', language: fence.info, text: collected.join('\n') }) + return index +} + +function readIndentedCode(walk: Walk, lines: readonly string[], start: number): number { + const collected: string[] = [] + const held: string[] = [] + let index = start + let end = start + while (index < lines.length) { + const line = lines[index] ?? '' + index += 1 + if (blankLine.test(line)) { + held.push(removeColumns(line, indentedCodeColumns)) + continue + } + if (leadingColumns(line) < indentedCodeColumns) break + collected.push(...held, removeColumns(line, indentedCodeColumns)) + held.length = 0 + end = index + } + walk.blocks.push({ kind: 'code', language: '', text: collected.join('\n') }) + return end +} + +function readHtmlBlock(walk: Walk, lines: readonly string[], start: number, html: OpenHtmlBlock): number { + let index = start + while (index < lines.length) { + const line = lines[index] ?? '' + if (html.closer === undefined && blankLine.test(line)) break + index += 1 + if (html.closer !== undefined && html.closer.test(line)) break + } + walk.blocks.push({ construct: html.construct, kind: 'html' }) + return index +} + +function closeParagraph(walk: Walk): void { + const text = takeParagraph(walk) + if (text !== undefined) walk.blocks.push({ kind: 'paragraph', text }) +} + +function takeParagraph(walk: Walk): string | undefined { + const text = readLinkDefinitions(walk.definitions, walk.paragraph.join('\n')) + walk.paragraph = [] + return text === '' ? undefined : text +} + +function leadingColumns(line: string): number { + let columns = 0 + for (const character of line) { + if (character === ' ') columns += 1 + else if (character === '\t') columns += tabStop - (columns % tabStop) + else break + } + return columns +} + +// CommonMark's tab stops: a tab the cut splits gives the columns it holds past the cut back as spaces. +function removeColumns(line: string, columns: number): string { + let removed = 0 + let index = 0 + while (removed < columns && index < line.length) { + const character = line.charAt(index) + if (character !== ' ' && character !== '\t') break + const width = character === ' ' ? 1 : tabStop - (removed % tabStop) + index += 1 + if (removed + width > columns) return ' '.repeat(removed + width - columns) + line.slice(index) + removed += width + } + return line.slice(index) +} diff --git a/src/markdown/parse/html-blocks.ts b/src/markdown/parse/html-blocks.ts new file mode 100644 index 0000000..ad5ef61 --- /dev/null +++ b/src/markdown/parse/html-blocks.ts @@ -0,0 +1,28 @@ +export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string } + +type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp } + +// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. +const blockTagNames = + 'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul' +const attributeSource = '(?:[ \\t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \\t]*=[ \\t]*(?:[^ \\t"\'=<>`]+|\'[^\']*\'|"[^"]*"))?)' +const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \\t]*/?>|)[ \\t]*$`) +const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/ + +const conditions: HtmlBlockCondition[] = [ + { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i }, + { closer: /-->/, construct: 'an HTML comment', interrupts: true, start: /^\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('
\nx\n
\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('
\nx\n
\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('\n')), 'unmappable-html') + assert.deepEqual(path(markdownToAdf('Part.\n\n
\n')), ['content', 1]) + assert.equal(code(markdownToAdf('
\nx\n\n:::\n')), 'unmappable-html') +}) + +test('swallows an HTML block ahead of the claim a line inside it would make', () => { + assert.equal(code(markdownToAdf('\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('
\n| x |\n
\n')), 'unmappable-html') +}) + +test('leaves a tag that opens no HTML block to the paragraph it sits in', () => { + assert.deepEqual(content(markdownToAdf('Part.\n\n')), [paragraph('Part. ')]) + assert.deepEqual(content(markdownToAdf('3 < 4\n')), [paragraph('3 < 4')]) +}) + +test('gives up the link reference definitions a paragraph opens with', () => { + assert.deepEqual(content(markdownToAdf('[a]: /url\n')), []) + assert.deepEqual(content(markdownToAdf('[a]: /url\n[b]: /other\nPart.\n')), [paragraph('Part.')]) + assert.deepEqual(content(markdownToAdf('[a]: /url\n"Title"\n\nPart.\n')), [paragraph('Part.')]) + assert.deepEqual(content(markdownToAdf('[a]: /url and more\n')), [paragraph('[a]: /url and more')]) + assert.deepEqual(content(markdownToAdf('Part.\n[a]: /url\n')), [paragraph('Part. [a]: /url')]) + assert.deepEqual(content(markdownToAdf('[a]: /url\n===\n')), [paragraph('===')]) +}) + +test('keeps the whitespace CommonMark strips no more of than a space or a tab', () => { + assert.deepEqual(content(markdownToAdf('\u00a0Part.\u00a0\n')), [paragraph('\u00a0Part.\u00a0')]) + assert.deepEqual(content(markdownToAdf(' \u3000Part.\t\n')), [paragraph('\u3000Part.')]) +}) + +test('normalizes the line endings and the null character CommonMark replaces', () => { + assert.deepEqual(content(markdownToAdf('One\r\ntwo.\r\n')), [paragraph('One two.')]) + assert.deepEqual(content(markdownToAdf('One\rtwo.\r')), [paragraph('One two.')]) + assert.deepEqual(content(markdownToAdf('```\r\nx\r\n```\r\n')), [{ content: [text('x')], type: 'codeBlock' }]) + assert.deepEqual(content(markdownToAdf('a\u0000b\n')), [paragraph('a\ufffdb')]) +}) diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts new file mode 100644 index 0000000..0a3078a --- /dev/null +++ b/src/markdown/parse/markdown-to-adf.ts @@ -0,0 +1,47 @@ +import type { AdfDocument, AdfNode } from '../../adf/document.ts' +import type { ClaimedConstruct, LeafBlock } from './blocks.ts' +import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' +import { parseBlocks } from './blocks.ts' +import { trimSpace } from '../commonmark-grammar.ts' + +export function markdownToAdf(markdown: string): Result { + const content: AdfNode[] = [] + for (const [index, block] of parseBlocks(markdown).blocks.entries()) { + const node = blockNode(block, ['content', index]) + if (!node.ok) return node + content.push(node.value) + } + return success(content.length === 0 ? { type: 'doc', version: 1 } : { content, type: 'doc', version: 1 }) +} + +function blockNode(block: LeafBlock, path: ConvertErrorPath): Result { + if (block.kind === 'claim') return claimFailure(block.construct, path) + if (block.kind === 'code') return success(codeBlockNode(block.language, block.text)) + if (block.kind === 'heading') return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) + if (block.kind === 'html') return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) + if (block.kind === 'paragraph') return success(withContent({ type: 'paragraph' }, block.text)) + return success({ type: 'rule' }) +} + +function claimFailure(construct: ClaimedConstruct, path: ConvertErrorPath): Result { + if (construct === 'directive') return failure('malformed-directive', 'the line claims a directive and parses as none', path) + return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path) +} + +function codeBlockNode(language: string, text: string): AdfNode { + const node: AdfNode = language === '' ? { type: 'codeBlock' } : { attrs: { language }, type: 'codeBlock' } + return text === '' ? node : { ...node, content: [{ text, type: 'text' }] } +} + +function withContent(node: AdfNode, text: string): AdfNode { + const content = inlineContent(text) + return content.length === 0 ? node : { ...node, content } +} + +function inlineContent(text: string): AdfNode[] { + const line = text + .split('\n') + .map((part) => trimSpace(part)) + .join(' ') + return line === '' ? [] : [{ text: line, type: 'text' }] +} diff --git a/src/result.ts b/src/result.ts index 6c705a9..41f5528 100644 --- a/src/result.ts +++ b/src/result.ts @@ -1,5 +1,8 @@ export type ConvertErrorCode = + | 'malformed-directive' + | 'malformed-pipe-table' | 'not-an-adf-document' + | 'unmappable-html' | 'unspellable-adjacent-lists' | 'unspellable-character' | 'unspellable-line-start' diff --git a/todo.md b/todo.md index 6135f99..2f07dad 100644 --- a/todo.md +++ b/todo.md @@ -196,7 +196,7 @@ detail is settled at its own milestone. `name (type)` list and asserts it equals the table, leaving the spec the source a human writes with no build step and no generated file. It is built at 3g, where a wrong entry starts refusing documents. - - [ ] **3b — The leaf blocks.** The line walk that opens and closes a block, ahead of any inline + - [x] **3b — The leaf blocks.** The line walk that opens and closes a block, ahead of any inline parsing: paragraph, ATX and setext heading, thematic break, fenced and indented code block, the HTML block whose lines it swallows whether or not the construct then errors, the link reference definitions a closing paragraph gives up, and the blank lines between @@ -210,7 +210,11 @@ detail is settled at its own milestone. rather than editing them. - [ ] **3c — The container blocks.** Blockquote, bullet and ordered list: the continuation a marker's width sets, lazy continuation, and the tightness ADF does not record — `> ` - repeated being two bytes a level, so this is the cheapest way to reach §11's 500. + repeated being two bytes a level, so this is the cheapest way to reach §11's 500. 3b's leaf + readers scan the physical line themselves, so a container re-cuts the walk rather than adding + to it: the open containers' prefix comes off the line first and the readers take one line at a + time, `LeafBlock` renamed with the union they join and `blockNode`'s chain gaining their + branches. **Settled** (the maintainer, 2026-08-27): a claimed line ends lazy continuation, so a closing fence on the line after a blockquote's open paragraph closes its container instead of continuing the paragraph CommonMark would fold it into. Claiming at block level is @@ -222,13 +226,13 @@ detail is settled at its own milestone. into the separation it names, and `spec/flavour.md`'s "none between a nested list and a CommonMark block above it" gaining that exception. Every fixture spelled tight today keeps its bytes, and `nested-list-tight` becomes a round-trip pair. - - [ ] **3d — Inline text.** The inline scanner over a block's content: backslash escapes, - entity references decoding to their characters, code spans and the literal they hold — - directive syntax and `~~` included — CommonMark's own hard breaks, a trailing backslash - and two trailing spaces alike, a soft line break as one space, and the raw inline tag, - comment and processing instruction refused by name, recognized by the - `commonmark-grammar.ts` predicates the emitter already escapes against, under 3b's - one-table rule. + - [ ] **3d — Inline text.** The inline scanner over a block's content: backslash escapes, entity + references decoding to their characters, code spans and the literal they hold — directive + syntax and `~~` included — CommonMark's own hard breaks, a trailing backslash and two + trailing spaces alike, a soft line break as one space, the fenced info string's own decoding + the block walk leaves raw, and the raw inline tag, comment and processing instruction + refused by name, recognized by the `commonmark-grammar.ts` predicates the emitter already + escapes against, under 3b's one-table rule. - [ ] **3e — Emphasis and links.** `_`, `*` and `~~` runs through `matchEmphasis` to the `em`, `strong` and `strike` marks; links inline and reference, 3b's definitions resolved here, autolinks, and the image gap's named errors — a titled image, and one amid other text. @@ -244,7 +248,8 @@ detail is settled at its own milestone. - [ ] **3f — The directive grammar.** The three forms — inline `:name[content]{attrs}`, container `:::name arg {attrs}`, leaf `::name arg {attrs}` — the attribute grammar with its quoting and escapes, the fence-length and nesting rules, and the malformed list - `spec/flavour.md` spells, each a named error. + `spec/flavour.md` spells, each a named error. `corpus.test.ts`'s `fenceNestingFault` stays a + second reading of the fence rule over emitted bytes: the double entry is the check. **Settled** (the maintainer, 2026-08-27): the code span, the entity and raw HTML bind first in input, as 2e3 already assumed of the emitted side — a raw `` ` ``, `&`, `<` or `|` inside `{attrs}` breaks the directive and is a named error, the author writing the @@ -281,7 +286,8 @@ detail is settled at its own milestone. named error, a carry inside a mark spelling another, and the three carve-outs' escapes reading as the literal text they hold. - [ ] **3k — The CommonMark spec suite.** Checked in at `corpus/commonmark-spec/`, pinned to - the version it ships, `corpus/README.md` gaining the kind. + the version it ships — the one `html-blocks.ts` names for its start conditions — + `corpus/README.md` gaining the kind. **Settled** (the maintainer, 2026-08-27): three checks an example must pass, the reference HTML each ships read as corpus data — which adds no format and no direction (§1). §2's canonical fixpoint: a named error, or markdown that parses and emits to itself byte for