diff --git a/corpus/round-trip/commonmark-subset/code-block.json b/corpus/round-trip/commonmark-subset/code-block.json index e22ce1e..ccc8f3d 100644 --- a/corpus/round-trip/commonmark-subset/code-block.json +++ b/corpus/round-trip/commonmark-subset/code-block.json @@ -6,7 +6,7 @@ }, "content": [ { - "text": "SELECT id\nFROM part\nWHERE qty > 0;", + "text": "SELECT id\nFROM part\nWHERE qty > 0;\n-- 1. the marker a container start would claim", "type": "text" } ], diff --git a/corpus/round-trip/commonmark-subset/code-block.md b/corpus/round-trip/commonmark-subset/code-block.md index 1b22ef2..554645b 100644 --- a/corpus/round-trip/commonmark-subset/code-block.md +++ b/corpus/round-trip/commonmark-subset/code-block.md @@ -2,4 +2,5 @@ SELECT id FROM part WHERE qty > 0; +-- 1. the marker a container start would claim ``` diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index b8cad49..09dbdf7 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -16,10 +16,11 @@ const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/ const codeFenceOpener = /^(`{3,}|~{3,})/ const directiveClaim = /^:{2,}(?:[A-Za-z0-9]|[ \t]*$)/ const pipeClaim = /^\|/ +const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. -const firstCharacterOpeners = [atxHeadingOpener, /^>/, /^[*+-](?:[ \t]|$)/, codeFenceOpener, /^:{2,}/, pipeClaim] +const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] -const orderedListOpener = /^(\d{1,9})[.)](?:[ \t]|$)/ +const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u @@ -86,8 +87,16 @@ export function isUnicodeWhitespace(character: string): boolean { return unicodeWhitespace.test(character) } -// The first marker of a list, `undefined` for a bullet: one answer both directions read, or the emitter -// spells a list the parser folds into the paragraph above it. +// `start` is the list's first number, `undefined` for a bullet. +export function listMarker(line: string): { delimiter: string; start: number | undefined; width: number } | undefined { + const ordered = orderedListOpener.exec(line) + if (ordered !== null) { + const digits = ordered[1] ?? '' + return { delimiter: ordered[2] ?? '', start: Number(digits), width: digits.length + 1 } + } + return bulletListOpener.test(line) ? { delimiter: line.charAt(0), start: undefined, width: 1 } : undefined +} + export function markerInterruptsParagraph(start: number | undefined, empty: boolean): boolean { return !empty && (start === undefined || start === 1) } diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index 7487267..cc626e8 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -5,6 +5,7 @@ import { claimsPipeLine, closingCodeFence, isThematicBreak, + listMarker, markerInterruptsParagraph, openingCodeFence, setextHeadingLevel, @@ -39,13 +40,9 @@ type OpenLeaf = type ContainerStart = { kind: 'blockquote'; rest: string } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: string } -type ItemMarker = { list: ListBlock; marker: string; width: number } - type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] } const blankLine = /^[ \t]*$/ -const bulletMarker = /^[-*+](?=[ \t]|$)/ -const orderedMarker = /^(\d{1,9})([.)])(?=[ \t]|$)/ const indentedCodeColumns = 4 const largestOpenerIndentation = 3 const tabStop = 4 @@ -59,6 +56,11 @@ export function parseBlocks(markdown: string): ParsedBlocks { function readLine(walk: Walk, line: string): void { const matched = matchContainers(walk, line) + // A leaf that swallows whole lines takes the marker too: no container opens inside a code or HTML block. + if (matched.depth === walk.stack.length && swallowsLines(walk.leaf)) { + readBlockLine(walk, matched.rest) + return + } const paragraphOpen = matched.depth === walk.stack.length && walk.leaf?.kind === 'paragraph' const opened = openContainers(walk, matched.rest, paragraphOpen, matched.depth) if (!opened.opened && matched.depth < walk.stack.length) { @@ -71,6 +73,10 @@ function readLine(walk: Walk, line: string): void { readBlockLine(walk, opened.rest) } +function swallowsLines(leaf: OpenLeaf | undefined): boolean { + return leaf?.kind === 'fenced-code' || leaf?.kind === 'html' +} + function matchContainers(walk: Walk, line: string): { depth: number; rest: string } { let depth = 0 let rest = line @@ -111,38 +117,32 @@ function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { const opener = removeColumns(line, largestOpenerIndentation) if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) } - if (isThematicBreak(opener) || (paragraphOpen && setextHeadingLevel(opener) !== undefined)) return undefined + if (isThematicBreak(opener)) return undefined return itemStart(line, opener, paragraphOpen, enclosing) } function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { - const marker = itemMarker(opener) + const marker = listMarker(opener) if (marker === undefined) return undefined const after = opener.slice(marker.width) const blank = blankLine.test(after) - if (paragraphOpen && !markerInterruptsParagraph(marker.list.kind === 'orderedList' ? marker.list.start : undefined, blank)) return undefined + if (paragraphOpen && !markerInterruptsParagraph(marker.start, blank)) return undefined const spaces = leadingColumns(after) const padding = blank || spaces > indentedCodeColumns ? 1 : spaces - const continued = enclosing?.kind === 'item' && enclosing.list.kind === marker.list.kind && enclosing.marker === marker.marker + const kind = marker.start === undefined ? 'bulletList' : 'orderedList' + const continued = enclosing?.kind === 'item' && enclosing.list.kind === kind && enclosing.marker === marker.delimiter return { fresh: !continued, indentation: leadingColumns(line) + marker.width + padding, kind: 'item', - list: continued ? enclosing.list : marker.list, - marker: marker.marker, + list: continued ? enclosing.list : openList(marker.start), + marker: marker.delimiter, rest: blank ? '' : removeColumns(after, padding), } } -function itemMarker(opener: string): ItemMarker | undefined { - const ordered = orderedMarker.exec(opener) - if (ordered !== null) { - const digits = ordered[1] ?? '' - const delimiter = ordered[2] ?? '' - return { list: { items: [], kind: 'orderedList', start: Number(digits) }, marker: delimiter, width: digits.length + 1 } - } - const bullet = bulletMarker.exec(opener)?.[0] - return bullet === undefined ? undefined : { list: { items: [], kind: 'bulletList' }, marker: bullet, width: 1 } +function openList(start: number | undefined): ListBlock { + return start === undefined ? { items: [], kind: 'bulletList' } : { items: [], kind: 'orderedList', start } } function openContainer(walk: Walk, start: ContainerStart): void { diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index 0d7e250..be8814a 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -85,6 +85,7 @@ test('reads a fenced code block, its info string the language', () => { assert.deepEqual(content(markdownToAdf('~~~ a`b\n```\n~~~\n')), [{ attrs: { language: 'a`b' }, content: [text('```')], type: 'codeBlock' }]) assert.deepEqual(content(markdownToAdf('``` a`b\n')), [paragraph('``` a`b')]) assert.deepEqual(content(markdownToAdf('```\n``` x\n```\n')), [{ content: [text('``` x')], type: 'codeBlock' }]) + assert.deepEqual(content(markdownToAdf('```\n- x\n> y\n```\n')), [{ content: [text('- x\n> y')], type: 'codeBlock' }]) }) test('strips the opening fence indentation from the content lines it holds', () => { @@ -124,6 +125,7 @@ test('refuses the raw HTML no element mapping carries', () => { assert.equal(code(markdownToAdf('\n')), 'unmappable-html') assert.deepEqual(path(markdownToAdf('Part.\n\n
\n')), ['content', 1]) assert.equal(code(markdownToAdf('
\nx\n\n:::\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('
\n- x\n
\n')), 'unmappable-html') }) test('swallows an HTML block ahead of the claim a line inside it would make', () => { diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index 9fdc898..b3d4dcf 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -22,7 +22,6 @@ function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: num return success(content) } -// Switched, not chained: `noImplicitReturns` then refuses the kind a later milestone adds and forgets. function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result { switch (block.kind) { case 'blockquote':