From e73de651be97452511a73ccfdee5e80c6c89fa68 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 28 Aug 2026 14:46:14 +0200 Subject: [PATCH 1/6] Read the container blocks, and part a nested list the tight spelling would swallow --- AGENTS.md | 5 +- corpus/errors/claim-after-blockquote.error | 1 + corpus/errors/claim-after-blockquote.md | 2 + corpus/errors/nesting-depth.error | 1 + corpus/errors/nesting-depth.md | 1 + corpus/normalization/lazy-continuation.json | 39 ++ corpus/normalization/lazy-continuation.md | 6 + corpus/normalization/loose-list.json | 89 +++++ corpus/normalization/loose-list.md | 8 + .../nested-list-separation.json} | 0 .../nested-list-separation.md | 3 + corpus/unspellable/nested-list-tight.error | 1 - package.json | 2 +- spec/flavour.md | 7 +- src/markdown/emit/adf-to-markdown.test.ts | 14 +- src/markdown/emit/adf-to-markdown.ts | 9 +- src/markdown/emit/inline-line.ts | 2 +- src/markdown/opaque-carry.ts | 2 +- src/markdown/parse/blocks.test.ts | 4 + src/markdown/parse/blocks.ts | 365 ++++++++++++------ src/markdown/parse/markdown-to-adf.test.ts | 85 ++++ src/markdown/parse/markdown-to-adf.ts | 37 +- src/result.ts | 1 + todo.md | 6 +- 24 files changed, 540 insertions(+), 150 deletions(-) create mode 100644 corpus/errors/claim-after-blockquote.error create mode 100644 corpus/errors/claim-after-blockquote.md create mode 100644 corpus/errors/nesting-depth.error create mode 100644 corpus/errors/nesting-depth.md create mode 100644 corpus/normalization/lazy-continuation.json create mode 100644 corpus/normalization/lazy-continuation.md create mode 100644 corpus/normalization/loose-list.json create mode 100644 corpus/normalization/loose-list.md rename corpus/{unspellable/nested-list-tight.json => round-trip/commonmark-subset/nested-list-separation.json} (100%) create mode 100644 corpus/round-trip/commonmark-subset/nested-list-separation.md delete mode 100644 corpus/unspellable/nested-list-tight.error diff --git a/AGENTS.md b/AGENTS.md index 721f3b2..4f0394a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,8 +81,9 @@ The error surface is a contract too. `ConvertError` is `{ code, message, path }` closed list a consumer may switch exhaustively, the message free text, the path the node's position from the document root. Adding, removing or renaming a code is breaking, so a milestone meeting a new failure cause reuses a code where one fits; the list is complete at `0.1.0`. A code names the -cause; where one cause recurs across node types, one code covers them all and `path` and `message` -say which. A claim code names the spelling claimed, never the node that spelling would have built: +cause; where one cause recurs across node types or across directions, one code covers them all and +`path` and `message` say which — `unsupported-nesting-depth` is the 500-level guard whichever +direction hits it. A claim code names the spelling claimed, never the node that spelling would have built: a malformed `:::table` is a `malformed-directive`. A cause the carry answers gets no code: a mark no spelling writes rides the carry with its node. diff --git a/corpus/errors/claim-after-blockquote.error b/corpus/errors/claim-after-blockquote.error new file mode 100644 index 0000000..4f35411 --- /dev/null +++ b/corpus/errors/claim-after-blockquote.error @@ -0,0 +1 @@ +malformed-directive diff --git a/corpus/errors/claim-after-blockquote.md b/corpus/errors/claim-after-blockquote.md new file mode 100644 index 0000000..094d5ec --- /dev/null +++ b/corpus/errors/claim-after-blockquote.md @@ -0,0 +1,2 @@ +> Part. +::: diff --git a/corpus/errors/nesting-depth.error b/corpus/errors/nesting-depth.error new file mode 100644 index 0000000..f54ac45 --- /dev/null +++ b/corpus/errors/nesting-depth.error @@ -0,0 +1 @@ +unsupported-nesting-depth diff --git a/corpus/errors/nesting-depth.md b/corpus/errors/nesting-depth.md new file mode 100644 index 0000000..6bc2663 --- /dev/null +++ b/corpus/errors/nesting-depth.md @@ -0,0 +1 @@ +> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > diff --git a/corpus/normalization/lazy-continuation.json b/corpus/normalization/lazy-continuation.json new file mode 100644 index 0000000..666c248 --- /dev/null +++ b/corpus/normalization/lazy-continuation.json @@ -0,0 +1,39 @@ +{ + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Ship it then tell them", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "blockquote" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Torque the bolts lazily", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/lazy-continuation.md b/corpus/normalization/lazy-continuation.md new file mode 100644 index 0000000..6d44a2a --- /dev/null +++ b/corpus/normalization/lazy-continuation.md @@ -0,0 +1,6 @@ +> Ship it +then tell them + +- Torque + the bolts +lazily diff --git a/corpus/normalization/loose-list.json b/corpus/normalization/loose-list.json new file mode 100644 index 0000000..3aaf16b --- /dev/null +++ b/corpus/normalization/loose-list.json @@ -0,0 +1,89 @@ +{ + "content": [ + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Bolt M8", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + }, + { + "content": [ + { + "content": [ + { + "text": "Nut M8", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + }, + { + "content": [ + { + "text": "Then:", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Washer M8", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "attrs": { + "order": 1 + }, + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Fibre", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + } + ], + "type": "orderedList" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/loose-list.md b/corpus/normalization/loose-list.md new file mode 100644 index 0000000..97056bc --- /dev/null +++ b/corpus/normalization/loose-list.md @@ -0,0 +1,8 @@ +* Bolt M8 + +* Nut M8 + +Then: + ++ Washer M8 + 1) Fibre diff --git a/corpus/unspellable/nested-list-tight.json b/corpus/round-trip/commonmark-subset/nested-list-separation.json similarity index 100% rename from corpus/unspellable/nested-list-tight.json rename to corpus/round-trip/commonmark-subset/nested-list-separation.json diff --git a/corpus/round-trip/commonmark-subset/nested-list-separation.md b/corpus/round-trip/commonmark-subset/nested-list-separation.md new file mode 100644 index 0000000..df96f82 --- /dev/null +++ b/corpus/round-trip/commonmark-subset/nested-list-separation.md @@ -0,0 +1,3 @@ +- Torque the bolts + + 2. Then sign off diff --git a/corpus/unspellable/nested-list-tight.error b/corpus/unspellable/nested-list-tight.error deleted file mode 100644 index 2dc039a..0000000 --- a/corpus/unspellable/nested-list-tight.error +++ /dev/null @@ -1 +0,0 @@ -unspellable-line-start diff --git a/package.json b/package.json index 0d74b30..895c904 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "node": ">=24" }, "scripts": { - "test": "node --test --experimental-test-coverage --test-coverage-exclude=\"src/**/*.test.ts\" --test-coverage-branches=96 --test-coverage-functions=100 --test-coverage-lines=100 \"src/**/*.test.ts\"", + "test": "node --test --experimental-test-coverage --test-coverage-exclude=\"src/**/*.test.ts\" --test-coverage-branches=97 --test-coverage-functions=100 --test-coverage-lines=100 \"src/**/*.test.ts\"", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/spec/flavour.md b/spec/flavour.md index bcc356e..4bac422 100644 --- a/spec/flavour.md +++ b/spec/flavour.md @@ -19,7 +19,9 @@ normalizes to it through the round-trip. - Bullet lists `- `; ordered lists incrementing `1.` `2.` `3.`, the first number the node's `order` attribute. Continuation lines align with the first character after the marker (two spaces for `- `, three for `1. `); blank lines inside an item are empty lines, none - between a nested list and a CommonMark block above it. Blank lines between items normalize + between a nested list and a CommonMark block above it — one where the nested list's own marker + cannot interrupt a paragraph (any ordered list, or a bullet list whose first item opens with an + empty block), which the block above would otherwise swallow. Blank lines between items normalize away; ADF does not record tightness. - Blockquotes prefix lines with `> `; a blank line inside a blockquote is a bare `>`. - ATX headings (`#` … `######`); setext input normalizes to ATX. @@ -92,7 +94,8 @@ above. **Claiming at block level**, symmetric with inline: a line whose leading run of two or more colons is followed immediately by a name character is claimed and must parse fully as a container opening or a leaf, else it is a named error. A bare colon-run line is a closing fence while a -container is open, a named error otherwise. +container is open, a named error otherwise. A claimed line also ends a lazy continuation: the +blockquote or list item whose paragraph CommonMark would fold it into closes instead. **Attributes**: `{key=value key2="two words"}`. `{attrs}` is optional in every form, and `{}` is valid — no attributes. A bare value matches `[A-Za-z0-9_-]+`; any other value is double-quoted diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 398714b..2e94731 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -159,8 +159,8 @@ test('breaks a mark run at the node it carries', () => { test('refuses a carried node nested deeper than the emitter carries', () => { let node: AdfNode = { type: 'blockCard' } for (let depth = 0; depth < 600; depth += 1) node = { content: [node], type: 'blockCard' } - assert.equal(code(adfToMarkdown(document(node))), 'unsupported-node-shape') - assert.equal(code(adfToMarkdown(document(paragraph(node)))), 'unsupported-node-shape') + assert.equal(code(adfToMarkdown(document(node))), 'unsupported-nesting-depth') + assert.equal(code(adfToMarkdown(document(paragraph(node)))), 'unsupported-nesting-depth') }) test('refuses a node whose content model the canonical form cannot emit', () => { @@ -261,19 +261,19 @@ test('refuses a node carrying one mark type twice', () => { assert.equal(code(adfToMarkdown(document(paragraph({ marks: [em, em], text: 'x', type: 'text' })))), 'unsupported-node-shape') }) -test('refuses a nested list the tight spelling would swallow', () => { +test('parts a nested list the tight spelling would swallow from the block above it', () => { const item = (...content: AdfNode[]): AdfNode => ({ content, type: 'listItem' }) const text = (value: string): AdfNode => ({ content: [{ text: value, type: 'text' }], type: 'paragraph' }) const outer = (...content: AdfNode[]): AdfDocument => document({ content: [item(...content)], type: 'bulletList' }) const ordered: AdfNode = { attrs: { order: 2 }, content: [item(text('b'))], type: 'orderedList' } - assert.equal(code(adfToMarkdown(outer(text('a'), ordered))), 'unspellable-line-start') - assert.equal(code(adfToMarkdown(outer(text('a'), { content: [item()], type: 'bulletList' }))), 'unspellable-line-start') + assert.equal(markdown(adfToMarkdown(outer(text('a'), ordered))), '- a\n\n 2. b\n') + assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item()], type: 'bulletList' }))), '- a\n\n -\n') assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item(text('b'))], type: 'bulletList' }))), '- a\n - b\n') }) test('refuses marks and attributes nested deeper than the emitter carries', () => { const marks: AdfMark[] = Array.from({ length: 600 }, (_, index) => ({ type: index % 2 === 0 ? 'em' : 'strong' })) - assert.equal(code(adfToMarkdown(document(paragraph({ marks, text: 'x', type: 'text' })))), 'unsupported-node-shape') + assert.equal(code(adfToMarkdown(document(paragraph({ marks, text: 'x', type: 'text' })))), 'unsupported-nesting-depth') let attrs: AdfMark['attrs'] = { depth: 'x' } for (let depth = 0; depth < 600; depth += 1) attrs = { depth: attrs } assert.equal(code(adfToMarkdown(document(paragraph({ marks: [{ attrs, type: 'em' }], text: 'x', type: 'text' })))), 'not-an-adf-document') @@ -380,7 +380,7 @@ test('spells one code span over a run of code-marked nodes', () => { test('refuses a document nested deeper than the emitter carries', () => { let node: AdfNode = paragraph({ text: 'x', type: 'text' }) for (let depth = 0; depth < 600; depth += 1) node = { content: [node], type: 'blockquote' } - assert.equal(code(adfToMarkdown(document(node))), 'unsupported-node-shape') + assert.equal(code(adfToMarkdown(document(node))), 'unsupported-nesting-depth') }) test('emits an empty list item without trailing whitespace', () => { diff --git a/src/markdown/emit/adf-to-markdown.ts b/src/markdown/emit/adf-to-markdown.ts index 4334c0e..d3f016e 100644 --- a/src/markdown/emit/adf-to-markdown.ts +++ b/src/markdown/emit/adf-to-markdown.ts @@ -31,7 +31,7 @@ export function adfToMarkdown(document: AdfDocument): Result { } function emitBlocks(nodes: readonly AdfNode[], container: BlockContainer, path: ConvertErrorPath, depth: number): Result { - if (depth > largestNesting) return failure('unsupported-node-shape', `the document nests deeper than the ${largestNesting} levels the emitter carries`, path) + if (depth > largestNesting) return failure('unsupported-nesting-depth', `the document nests deeper than the ${largestNesting} levels the emitter carries`, path) const blocks: PlacedBlock[] = [] for (const [index, node] of nodes.entries()) { const nodePath = [...path, 'content', index] @@ -60,12 +60,7 @@ function separationBetween(previous: PlacedBlock, next: PlacedBlock, container: if (previous.node.type === next.node.type) { return failure('unspellable-adjacent-lists', `two adjacent ${next.node.type} nodes read back as one list`, next.path) } - if (container === 'list-item') { - if (!interruptsParagraph(next.node)) { - return failure('unspellable-line-start', `a ${next.node.type} that cannot interrupt the block above it has no tight spelling`, next.path) - } - return success('\n') - } + if (container === 'list-item') return success(interruptsParagraph(next.node) ? '\n' : '\n\n') } if (container !== 'directive' || plainPair) return success('\n\n') if (previous.spelling === 'directive' && next.spelling === 'directive') return success('\n') diff --git a/src/markdown/emit/inline-line.ts b/src/markdown/emit/inline-line.ts index bd22c07..811f7b6 100644 --- a/src/markdown/emit/inline-line.ts +++ b/src/markdown/emit/inline-line.ts @@ -140,7 +140,7 @@ function refuseContentAndText(node: AdfNode, path: ConvertErrorPath): Result { if (depth > largestNesting) { - return failure('unsupported-node-shape', `the marks nest deeper than the ${largestNesting} levels the emitter carries`, context.path) + return failure('unsupported-nesting-depth', `the marks nest deeper than the ${largestNesting} levels the emitter carries`, context.path) } const runs = inlineRuns(nodes, depth, firstIndex, context.carried) const segments: InlineSegment[] = [] diff --git a/src/markdown/opaque-carry.ts b/src/markdown/opaque-carry.ts index 1e6ecd1..84f9126 100644 --- a/src/markdown/opaque-carry.ts +++ b/src/markdown/opaque-carry.ts @@ -23,7 +23,7 @@ export function carriedInline(node: AdfNode, path: ConvertErrorPath): Result { if (!isJsonValue(node)) { - return failure('unsupported-node-shape', `a carried node's JSON nests deeper than the ${largestNesting} levels the emitter carries`, path) + return failure('unsupported-nesting-depth', `a carried node's JSON nests deeper than the ${largestNesting} levels the emitter carries`, path) } return success(serializeCanonicalJson(node, spelling)) } diff --git a/src/markdown/parse/blocks.test.ts b/src/markdown/parse/blocks.test.ts index e542a2a..f625693 100644 --- a/src/markdown/parse/blocks.test.ts +++ b/src/markdown/parse/blocks.test.ts @@ -28,6 +28,10 @@ test('keeps the link reference definitions a paragraph gives up, the first of a ['a', { destination: '/url\\' }], ['b', { destination: '/b' }], ]) + assert.deepEqual(definitions('> [a]: /url\n\n- [b]: /other\n'), [ + ['a', { destination: '/url' }], + ['b', { destination: '/other' }], + ]) assert.deepEqual(definitions('[\u00a0a]: /one\n[a]: /two\n'), [ ['\u00a0a', { destination: '/one' }], ['a', { destination: '/two' }], diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index e73d1f4..f21e924 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -1,80 +1,277 @@ import type { LinkDefinition } from './link-reference-definitions.ts' -import type { OpenHtmlBlock } from './html-blocks.ts' import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts' import { openingHtmlBlock } from './html-blocks.ts' import { readLinkDefinitions } from './link-reference-definitions.ts' export type ClaimedConstruct = 'directive' | 'pipe-table' -export type LeafBlock = +export type Block = + | { blocks: Block[]; kind: 'blockquote' } | { construct: ClaimedConstruct; kind: 'claim' } | { construct: string; kind: 'html' } + | { items: Block[][]; kind: 'bulletList' } + | { items: Block[][]; kind: 'orderedList'; start: number } | { kind: 'code'; language: string; text: string } | { kind: 'heading'; level: number; text: string } | { kind: 'paragraph'; text: string } | { kind: 'rule' } -export type ParsedBlocks = { blocks: LeafBlock[]; definitions: Map } +export type ParsedBlocks = { blocks: Block[]; definitions: Map } -type Walk = ParsedBlocks & { paragraph: string[] } +type ListBlock = Extract + +type OpenContainer = Extract | { blocks: Block[]; indentation: number; kind: 'item'; list: ListBlock; marker: string } + +type OpenLeaf = + | { closer: RegExp | undefined; construct: string; kind: 'html' } + | { held: string[]; kind: 'indented-code'; lines: string[] } + | { indentation: number; info: string; kind: 'fenced-code'; lines: string[]; marker: string } + | { kind: 'paragraph'; lines: string[] } + +type ContainerStart = { kind: 'blockquote'; rest: string } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: string } + +type ItemMarker = { list: ListBlock; marker: string; width: number } + +type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] } const blankLine = /^[ \t]*$/ +const bulletMarker = /^[-*+](?=[ \t]|$)/ +const orderedMarker = /^(\d{1,9})([.)])(?=[ \t]|$)/ const indentedCodeColumns = 4 const largestOpenerIndentation = 3 const tabStop = 4 export function parseBlocks(markdown: string): ParsedBlocks { - const lines = normalizeInput(markdown).split('\n') - const walk: Walk = { blocks: [], definitions: new Map(), paragraph: [] } - let index = 0 - while (index < lines.length) { - const line = lines[index] ?? '' - if (blankLine.test(line)) { - closeParagraph(walk) - index += 1 - continue - } - if (leadingColumns(line) >= indentedCodeColumns && walk.paragraph.length === 0) { - index = readIndentedCode(walk, lines, index) - continue - } - const opened = openBlock(walk, lines, index, line) - if (opened !== undefined) { - index = opened - continue - } - walk.paragraph.push(line.replace(/^[ \t]+/, '')) - index += 1 - } - closeParagraph(walk) + const walk: Walk = { blocks: [], definitions: new Map(), leaf: undefined, stack: [] } + for (const line of normalizeInput(markdown).split('\n')) readLine(walk, line) + closeLeaf(walk) return { blocks: walk.blocks, definitions: walk.definitions } } -function openBlock(walk: Walk, lines: readonly string[], index: number, line: string): number | undefined { +function readLine(walk: Walk, line: string): void { + const matched = matchContainers(walk, line) + const paragraphOpen = matched.depth === walk.stack.length && walk.leaf?.kind === 'paragraph' + const opened = openContainers(walk, matched.rest, paragraphOpen, matched.depth) + if (!opened.opened && matched.depth < walk.stack.length) { + if (continuesLazily(walk, opened.rest)) { + appendParagraph(walk, opened.rest) + return + } + closeContainers(walk, matched.depth) + } + readBlockLine(walk, opened.rest) +} + +function matchContainers(walk: Walk, line: string): { depth: number; rest: string } { + let depth = 0 + let rest = line + for (const container of walk.stack) { + const next = continuesContainer(walk, container, rest) + if (next === undefined) break + depth += 1 + rest = next + } + return { depth, rest } +} + +function continuesContainer(walk: Walk, container: OpenContainer, line: string): string | undefined { + if (container.kind === 'blockquote') { + const opener = removeColumns(line, largestOpenerIndentation) + return opener.startsWith('>') ? removeColumns(opener.slice(1), 1) : undefined + } + // A list item begins with at most one blank line: an empty one gives the second up. + if (blankLine.test(line)) return container.blocks.length === 0 && walk.leaf === undefined ? undefined : '' + return leadingColumns(line) < container.indentation ? undefined : removeColumns(line, container.indentation) +} + +function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: number): { opened: boolean; rest: string } { + let opened = false + let rest = line + while (leadingColumns(rest) < indentedCodeColumns) { + const start = containerStart(rest, opened ? false : paragraphOpen, opened ? undefined : walk.stack[depth]) + if (start === undefined) break + if (!opened) closeContainers(walk, depth) + opened = true + openContainer(walk, start) + rest = start.rest + } + return { opened, rest } +} + +function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { + const opener = removeColumns(line, largestOpenerIndentation) + if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) } + if (isThematicBreak(opener) || (paragraphOpen && setextHeadingLevel(opener) !== undefined)) return undefined + const marker = itemMarker(opener) + if (marker === undefined) return undefined + const after = opener.slice(marker.width) + const blank = blankLine.test(after) + if (paragraphOpen && (blank || (marker.list.kind === 'orderedList' && marker.list.start !== 1))) return undefined + const spaces = leadingColumns(after) + const padding = blank || spaces > indentedCodeColumns ? 1 : spaces + const continued = enclosing?.kind === 'item' && enclosing.list.kind === marker.list.kind && enclosing.marker === marker.marker + return { + fresh: !continued, + indentation: leadingColumns(line) + marker.width + padding, + kind: 'item', + list: continued ? enclosing.list : marker.list, + marker: marker.marker, + rest: blank ? '' : removeColumns(after, padding), + } +} + +function itemMarker(opener: string): ItemMarker | undefined { + const ordered = orderedMarker.exec(opener) + if (ordered !== null) { + const digits = ordered[1] ?? '' + const delimiter = ordered[2] ?? '' + return { list: { items: [], kind: 'orderedList', start: Number(digits) }, marker: delimiter, width: digits.length + 1 } + } + const bullet = bulletMarker.exec(opener)?.[0] + return bullet === undefined ? undefined : { list: { items: [], kind: 'bulletList' }, marker: bullet, width: 1 } +} + +function openContainer(walk: Walk, start: ContainerStart): void { + const blocks: Block[] = [] + if (start.kind === 'blockquote') { + const blockquote: OpenContainer = { blocks, kind: 'blockquote' } + currentBlocks(walk).push(blockquote) + walk.stack.push(blockquote) + return + } + if (start.fresh) currentBlocks(walk).push(start.list) + start.list.items.push(blocks) + walk.stack.push({ blocks, indentation: start.indentation, kind: 'item', list: start.list, marker: start.marker }) +} + +function closeContainers(walk: Walk, depth: number): void { + closeLeaf(walk) + walk.stack.length = depth +} + +// A claimed line ends the lazy continuation CommonMark would fold it into (spec/flavour.md). +function continuesLazily(walk: Walk, line: string): boolean { + if (walk.leaf?.kind !== 'paragraph' || blankLine.test(line)) return false + if (leadingColumns(line) >= indentedCodeColumns) return true + const opener = removeColumns(line, largestOpenerIndentation) + if (claimedConstruct(opener) !== undefined || isThematicBreak(opener)) return false + return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, false) === undefined +} + +function readBlockLine(walk: Walk, line: string): void { + const leaf = walk.leaf + if (leaf?.kind === 'fenced-code') { + if (closingCodeFence(removeColumns(line, largestOpenerIndentation), leaf.marker)) closeLeaf(walk) + else leaf.lines.push(removeColumns(line, leaf.indentation)) + return + } + if (leaf?.kind === 'html') { + if (leaf.closer === undefined ? blankLine.test(line) : leaf.closer.test(line)) closeLeaf(walk) + return + } + if (leaf?.kind === 'indented-code') { + if (readIndentedCodeLine(leaf, line)) return + closeLeaf(walk) + } + if (blankLine.test(line)) { + closeLeaf(walk) + return + } + if (walk.leaf === undefined && leadingColumns(line) >= indentedCodeColumns) { + walk.leaf = { held: [], kind: 'indented-code', lines: [removeColumns(line, indentedCodeColumns)] } + return + } + openLeaf(walk, line) +} + +function readIndentedCodeLine(leaf: Extract, line: string): boolean { + if (blankLine.test(line)) { + leaf.held.push(removeColumns(line, indentedCodeColumns)) + return true + } + if (leadingColumns(line) < indentedCodeColumns) return false + leaf.lines.push(...leaf.held, removeColumns(line, indentedCodeColumns)) + leaf.held.length = 0 + return true +} + +function openLeaf(walk: Walk, line: string): void { const opener = removeColumns(line, largestOpenerIndentation) const claimed = claimedConstruct(opener) if (claimed !== undefined) { - closeParagraph(walk) - walk.blocks.push({ construct: claimed, kind: 'claim' }) - return index + 1 + closeLeaf(walk) + currentBlocks(walk).push({ construct: claimed, kind: 'claim' }) + return } - if (readLineBlock(walk, opener)) return index + 1 + if (readLineBlock(walk, opener)) return const fence = openingCodeFence(opener) if (fence !== undefined) { - closeParagraph(walk) - return readFencedCode(walk, lines, index + 1, fence, leadingColumns(line)) + closeLeaf(walk) + walk.leaf = { indentation: leadingColumns(line), info: fence.info, kind: 'fenced-code', lines: [], marker: fence.marker } + return } - const html = openingHtmlBlock(opener, walk.paragraph.length > 0) - if (html === undefined) return undefined - closeParagraph(walk) - return readHtmlBlock(walk, lines, index, html) + const html = openingHtmlBlock(opener, walk.leaf?.kind === 'paragraph') + if (html === undefined) { + appendParagraph(walk, line) + return + } + closeLeaf(walk) + walk.leaf = { closer: html.closer, construct: html.construct, kind: 'html' } + if (html.closer?.test(line) === true) closeLeaf(walk) } -function normalizeInput(markdown: string): string { - return markdown - .replace(/\r\n?/g, '\n') - .replaceAll('\u0000', '\ufffd') - .replace(/\n$/, '') +// A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. +function readLineBlock(walk: Walk, opener: string): boolean { + const level = walk.leaf?.kind === 'paragraph' ? setextHeadingLevel(opener) : undefined + if (level !== undefined) { + const text = takeParagraph(walk) + if (text !== undefined) { + currentBlocks(walk).push({ kind: 'heading', level, text }) + return true + } + } + if (isThematicBreak(opener)) { + closeLeaf(walk) + currentBlocks(walk).push({ kind: 'rule' }) + return true + } + const heading = atxHeading(opener) + if (heading === undefined) return false + closeLeaf(walk) + currentBlocks(walk).push({ kind: 'heading', level: heading.level, text: heading.text }) + return true +} + +function appendParagraph(walk: Walk, line: string): void { + const leaf = walk.leaf + const text = line.replace(/^[ \t]+/, '') + if (leaf?.kind === 'paragraph') leaf.lines.push(text) + else walk.leaf = { kind: 'paragraph', lines: [text] } +} + +function closeLeaf(walk: Walk): void { + const leaf = walk.leaf + if (leaf === undefined) return + if (leaf.kind === 'paragraph') { + const text = takeParagraph(walk) + if (text !== undefined) currentBlocks(walk).push({ kind: 'paragraph', text }) + return + } + walk.leaf = undefined + if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html' }) + else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? leaf.info : '', text: leaf.lines.join('\n') }) +} + +function takeParagraph(walk: Walk): string | undefined { + const leaf = walk.leaf + if (leaf?.kind !== 'paragraph') return undefined + walk.leaf = undefined + const text = readLinkDefinitions(walk.definitions, leaf.lines.join('\n')) + return text === '' ? undefined : text +} + +function currentBlocks(walk: Walk): Block[] { + return walk.stack.at(-1)?.blocks ?? walk.blocks } function claimedConstruct(opener: string): ClaimedConstruct | undefined { @@ -82,83 +279,11 @@ function claimedConstruct(opener: string): ClaimedConstruct | undefined { return claimsPipeLine(opener) ? 'pipe-table' : undefined } -// A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. -function readLineBlock(walk: Walk, opener: string): boolean { - const level = walk.paragraph.length === 0 ? undefined : setextHeadingLevel(opener) - if (level !== undefined) { - const text = takeParagraph(walk) - if (text !== undefined) { - walk.blocks.push({ kind: 'heading', level, text }) - return true - } - } - if (isThematicBreak(opener)) { - closeParagraph(walk) - walk.blocks.push({ kind: 'rule' }) - return true - } - const heading = atxHeading(opener) - if (heading === undefined) return false - closeParagraph(walk) - walk.blocks.push({ kind: 'heading', level: heading.level, text: heading.text }) - return true -} - -function readFencedCode(walk: Walk, lines: readonly string[], start: number, fence: { info: string; marker: string }, indentation: number): number { - const collected: string[] = [] - let index = start - while (index < lines.length) { - const line = lines[index] ?? '' - index += 1 - if (closingCodeFence(removeColumns(line, largestOpenerIndentation), fence.marker)) break - collected.push(removeColumns(line, indentation)) - } - walk.blocks.push({ kind: 'code', language: fence.info, text: collected.join('\n') }) - return index -} - -function readIndentedCode(walk: Walk, lines: readonly string[], start: number): number { - const collected: string[] = [] - const held: string[] = [] - let index = start - let end = start - while (index < lines.length) { - const line = lines[index] ?? '' - index += 1 - if (blankLine.test(line)) { - held.push(removeColumns(line, indentedCodeColumns)) - continue - } - if (leadingColumns(line) < indentedCodeColumns) break - collected.push(...held, removeColumns(line, indentedCodeColumns)) - held.length = 0 - end = index - } - walk.blocks.push({ kind: 'code', language: '', text: collected.join('\n') }) - return end -} - -function readHtmlBlock(walk: Walk, lines: readonly string[], start: number, html: OpenHtmlBlock): number { - let index = start - while (index < lines.length) { - const line = lines[index] ?? '' - if (html.closer === undefined && blankLine.test(line)) break - index += 1 - if (html.closer !== undefined && html.closer.test(line)) break - } - walk.blocks.push({ construct: html.construct, kind: 'html' }) - return index -} - -function closeParagraph(walk: Walk): void { - const text = takeParagraph(walk) - if (text !== undefined) walk.blocks.push({ kind: 'paragraph', text }) -} - -function takeParagraph(walk: Walk): string | undefined { - const text = readLinkDefinitions(walk.definitions, walk.paragraph.join('\n')) - walk.paragraph = [] - return text === '' ? undefined : text +function normalizeInput(markdown: string): string { + return markdown + .replace(/\r\n?/g, '\n') + .replaceAll('\u0000', '\ufffd') + .replace(/\n$/, '') } function leadingColumns(line: string): number { diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index a761fd6..c39219f 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -25,6 +25,22 @@ function paragraph(value: string): AdfNode { return { content: [text(value)], type: 'paragraph' } } +function item(...content: AdfNode[]): AdfNode { + return content.length === 0 ? { type: 'listItem' } : { content, type: 'listItem' } +} + +function bulletList(...content: AdfNode[]): AdfNode { + return { content, type: 'bulletList' } +} + +function orderedList(order: number, ...content: AdfNode[]): AdfNode { + return { attrs: { order }, content, type: 'orderedList' } +} + +function quote(...content: AdfNode[]): AdfNode { + return content.length === 0 ? { type: 'blockquote' } : { content, type: 'blockquote' } +} + test('builds an empty document from input holding no block', () => { assert.deepEqual(markdownToAdf(''), { ok: true, value: { type: 'doc', version: 1 } }) assert.deepEqual(content(markdownToAdf('\n \n\t\n')), []) @@ -140,3 +156,72 @@ test('normalizes the line endings and the null character CommonMark replaces', ( assert.deepEqual(content(markdownToAdf('```\r\nx\r\n```\r\n')), [{ content: [text('x')], type: 'codeBlock' }]) assert.deepEqual(content(markdownToAdf('a\u0000b\n')), [paragraph('a\ufffdb')]) }) + +test('reads a blockquote and the blocks its prefix carries', () => { + assert.deepEqual(content(markdownToAdf('> Ship it.\n>\n> Then tell them.\n')), [quote(paragraph('Ship it.'), paragraph('Then tell them.'))]) + assert.deepEqual(content(markdownToAdf('>Ship it.\n')), [quote(paragraph('Ship it.'))]) + assert.deepEqual(content(markdownToAdf(' > > Deep.\n')), [quote(quote(paragraph('Deep.')))]) + assert.deepEqual(content(markdownToAdf('>\n')), [quote()]) + assert.deepEqual(content(markdownToAdf('> One.\n\n> Two.\n')), [quote(paragraph('One.')), quote(paragraph('Two.'))]) + assert.deepEqual(content(markdownToAdf('Part.\n> Ship it.\n')), [paragraph('Part.'), quote(paragraph('Ship it.'))]) + assert.deepEqual(content(markdownToAdf(' > Code.\n')), [{ content: [text('> Code.')], type: 'codeBlock' }]) +}) + +test('reads a bullet list, the marker width setting the continuation', () => { + assert.deepEqual(content(markdownToAdf('- Bolt M8\n- Nut M8\n')), [bulletList(item(paragraph('Bolt M8')), item(paragraph('Nut M8')))]) + assert.deepEqual(content(markdownToAdf('- Washer M8\n - Fibre\n')), [bulletList(item(paragraph('Washer M8'), bulletList(item(paragraph('Fibre')))))]) + assert.deepEqual(content(markdownToAdf('-\n')), [bulletList(item())]) + assert.deepEqual(content(markdownToAdf('- One\n\n Two.\n')), [bulletList(item(paragraph('One'), paragraph('Two.')))]) + assert.deepEqual(content(markdownToAdf('- Code.\n')), [bulletList(item({ content: [text('Code.')], type: 'codeBlock' }))]) + assert.deepEqual(content(markdownToAdf('- a\n* b\n')), [bulletList(item(paragraph('a'))), bulletList(item(paragraph('b')))]) + assert.deepEqual(content(markdownToAdf('-\n\n Part.\n')), [bulletList(item()), paragraph('Part.')]) +}) + +test('reads an ordered list, its first marker the order attribute', () => { + assert.deepEqual(content(markdownToAdf('9. Bolt M8\n10. Nut M8\n')), [orderedList(9, item(paragraph('Bolt M8')), item(paragraph('Nut M8')))]) + assert.deepEqual(content(markdownToAdf('1) Loosen the clamp\n')), [orderedList(1, item(paragraph('Loosen the clamp')))]) + assert.deepEqual(content(markdownToAdf('1. a\n1) b\n')), [orderedList(1, item(paragraph('a'))), orderedList(1, item(paragraph('b')))]) + assert.deepEqual(content(markdownToAdf('0. Zero\n')), [orderedList(0, item(paragraph('Zero')))]) +}) + +test('drops the tightness ADF does not record', () => { + assert.deepEqual(content(markdownToAdf('- a\n\n- b\n')), [bulletList(item(paragraph('a')), item(paragraph('b')))]) + assert.deepEqual(content(markdownToAdf('- a\n\n 2. b\n')), [bulletList(item(paragraph('a'), orderedList(2, item(paragraph('b')))))]) +}) + +test('opens a list beside a paragraph only where the marker interrupts it', () => { + assert.deepEqual(content(markdownToAdf('Part.\n- a\n')), [paragraph('Part.'), bulletList(item(paragraph('a')))]) + assert.deepEqual(content(markdownToAdf('Part.\n1. a\n')), [paragraph('Part.'), orderedList(1, item(paragraph('a')))]) + assert.deepEqual(content(markdownToAdf('Part.\n2. a\n')), [paragraph('Part. 2. a')]) + assert.deepEqual(content(markdownToAdf('Part.\n*\n')), [paragraph('Part. *')]) + assert.deepEqual(content(markdownToAdf('Part.\n- - -\n')), [paragraph('Part.'), { type: 'rule' }]) + assert.deepEqual(content(markdownToAdf('Part.\n-\n')), [{ attrs: { level: 2 }, content: [text('Part.')], type: 'heading' }]) + assert.deepEqual(content(markdownToAdf('- a\n 2. b\n')), [bulletList(item(paragraph('a 2. b')))]) +}) + +test('folds a lazy continuation into the paragraph the container holds', () => { + assert.deepEqual(content(markdownToAdf('> One\ntwo.\n')), [quote(paragraph('One two.'))]) + assert.deepEqual(content(markdownToAdf('- One\ntwo.\n')), [bulletList(item(paragraph('One two.')))]) + assert.deepEqual(content(markdownToAdf('> One\n two.\n')), [quote(paragraph('One two.'))]) + assert.deepEqual(content(markdownToAdf('> One\n\ntwo.\n')), [quote(paragraph('One')), paragraph('two.')]) + assert.deepEqual(content(markdownToAdf('> One\n# Two\n')), [quote(paragraph('One')), { attrs: { level: 1 }, content: [text('Two')], type: 'heading' }]) + assert.deepEqual(content(markdownToAdf('> One\n---\n')), [quote(paragraph('One')), { type: 'rule' }]) + assert.deepEqual(content(markdownToAdf('> One\n```\n')), [quote(paragraph('One')), { type: 'codeBlock' }]) + assert.equal(code(markdownToAdf('> One\n
\n')), 'unmappable-html') +}) + +test('ends a lazy continuation at a claimed line', () => { + assert.equal(code(markdownToAdf('> Part.\n:::\n')), 'malformed-directive') + assert.deepEqual(path(markdownToAdf('> Part.\n:::\n')), ['content', 1]) + assert.equal(code(markdownToAdf('- Part.\n| x |\n')), 'malformed-pipe-table') +}) + +test('names the block the claim inside a container opens', () => { + assert.deepEqual(path(markdownToAdf('> Part.\n>\n> :::x\n')), ['content', 0, 'content', 1]) + assert.deepEqual(path(markdownToAdf('- Part.\n- | x |\n')), ['content', 0, 'content', 1, 'content', 0]) +}) + +test('refuses input nested deeper than the parser carries', () => { + assert.equal(code(markdownToAdf('> '.repeat(501))), 'unsupported-nesting-depth') + assert.ok(markdownToAdf('> '.repeat(500)).ok) +}) diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index 0a3078a..19dd22f 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -1,28 +1,55 @@ import type { AdfDocument, AdfNode } from '../../adf/document.ts' -import type { ClaimedConstruct, LeafBlock } from './blocks.ts' +import type { Block, ClaimedConstruct } from './blocks.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' +import { largestNesting } from '../../nesting.ts' import { parseBlocks } from './blocks.ts' import { trimSpace } from '../commonmark-grammar.ts' export function markdownToAdf(markdown: string): Result { + const content = blockNodes(parseBlocks(markdown).blocks, [], 0) + if (!content.ok) return content + return success(content.value.length === 0 ? { type: 'doc', version: 1 } : { content: content.value, type: 'doc', version: 1 }) +} + +function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: number): Result { + if (depth > largestNesting) return failure('unsupported-nesting-depth', `the input nests deeper than the ${largestNesting} levels the parser carries`, path) const content: AdfNode[] = [] - for (const [index, block] of parseBlocks(markdown).blocks.entries()) { - const node = blockNode(block, ['content', index]) + for (const [index, block] of blocks.entries()) { + const node = blockNode(block, [...path, 'content', index], depth) if (!node.ok) return node content.push(node.value) } - return success(content.length === 0 ? { type: 'doc', version: 1 } : { content, type: 'doc', version: 1 }) + return success(content) } -function blockNode(block: LeafBlock, path: ConvertErrorPath): Result { +function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result { + if (block.kind === 'blockquote') return containerNode({ type: 'blockquote' }, block.blocks, path, depth) + if (block.kind === 'bulletList') return listNode({ type: 'bulletList' }, block.items, path, depth) if (block.kind === 'claim') return claimFailure(block.construct, path) if (block.kind === 'code') return success(codeBlockNode(block.language, block.text)) if (block.kind === 'heading') return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) if (block.kind === 'html') return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) + if (block.kind === 'orderedList') return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth) if (block.kind === 'paragraph') return success(withContent({ type: 'paragraph' }, block.text)) return success({ type: 'rule' }) } +function containerNode(node: AdfNode, blocks: readonly Block[], path: ConvertErrorPath, depth: number): Result { + const content = blockNodes(blocks, path, depth + 1) + if (!content.ok) return content + return success(content.value.length === 0 ? node : { ...node, content: content.value }) +} + +function listNode(node: AdfNode, items: readonly Block[][], path: ConvertErrorPath, depth: number): Result { + const content: AdfNode[] = [] + for (const [index, blocks] of items.entries()) { + const item = containerNode({ type: 'listItem' }, blocks, [...path, 'content', index], depth) + if (!item.ok) return item + content.push(item.value) + } + return success({ ...node, content }) +} + function claimFailure(construct: ClaimedConstruct, path: ConvertErrorPath): Result { if (construct === 'directive') return failure('malformed-directive', 'the line claims a directive and parses as none', path) return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path) diff --git a/src/result.ts b/src/result.ts index 41f5528..94e8c9d 100644 --- a/src/result.ts +++ b/src/result.ts @@ -11,6 +11,7 @@ export type ConvertErrorCode = | 'unspellable-whitespace' | 'unspelled-block-separation' | 'unsupported-document-version' + | 'unsupported-nesting-depth' | 'unsupported-node-shape' export type ConvertErrorPath = readonly (number | string)[] diff --git a/todo.md b/todo.md index 2f07dad..ae732dc 100644 --- a/todo.md +++ b/todo.md @@ -208,7 +208,7 @@ detail is settled at its own milestone. error the claim promises meanwhile. The runner's parse half comes with it, and the first `normalization/` fixtures, holding inline-trivial content so 3d and 3e add beside them rather than editing them. - - [ ] **3c — The container blocks.** Blockquote, bullet and ordered list: the continuation a + - [x] **3c — The container blocks.** Blockquote, bullet and ordered list: the continuation a marker's width sets, lazy continuation, and the tightness ADF does not record — `> ` repeated being two bytes a level, so this is the cheapest way to reach §11's 500. 3b's leaf readers scan the physical line themselves, so a container re-cuts the walk rather than adding @@ -225,7 +225,7 @@ detail is settled at its own milestone. where the nested list would be swallowed — `interruptsParagraph` inverted from a refusal into the separation it names, and `spec/flavour.md`'s "none between a nested list and a CommonMark block above it" gaining that exception. Every fixture spelled tight today keeps - its bytes, and `nested-list-tight` becomes a round-trip pair. + its bytes, and `nested-list-tight` becomes the round-trip pair `nested-list-separation`. - [ ] **3d — Inline text.** The inline scanner over a block's content: backslash escapes, entity references decoding to their characters, code spans and the literal they hold — directive syntax and `~~` included — CommonMark's own hard breaks, a trailing backslash and two @@ -270,7 +270,7 @@ detail is settled at its own milestone. in a container body with no blank line between them. That reduces the three cases to one rule, separation only where its absence would merge the blocks: the `:::` fence is separation already, and 3c's claim ends the lazy continuation that would otherwise swallow - it. The fixture becomes a round-trip pair, and with `nested-list-tight` and 3e's pair that + it. The fixture becomes a round-trip pair, and with `nested-list-separation` and 3e's pair that empties `corpus/unspellable/`: this chunk settles the directory's own guard in `corpus.test.ts` too, and `unspelled-block-separation`, which loses its only cause here. The emitter's other refusals survive on causes no fixture in that directory covers, so -- 2.52.0 From a09b80e65de83f1e8ca403cbb0621fe8c5d55e54 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 19:50:41 +0200 Subject: [PATCH 2/6] Answer the architecture review: one interrupt rule both directions read --- AGENTS.md | 4 ++- spec/flavour.md | 6 ++-- src/markdown/commonmark-grammar.ts | 6 ++++ src/markdown/emit/adf-to-markdown.test.ts | 4 +++ src/markdown/emit/adf-to-markdown.ts | 8 +++-- src/markdown/parse/blocks.ts | 20 ++++++++++-- src/markdown/parse/markdown-to-adf.test.ts | 1 + src/markdown/parse/markdown-to-adf.ts | 38 +++++++++++++++------- src/nesting.ts | 1 + 9 files changed, 67 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4f0394a..3527e67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - `src/adf/` holds ADF's own knowledge and imports no format. Each format directory (`markdown/`, `html/`) parts into `emit/` (ADF→format) and `parse/` (format→ADF), its root holding what both directions read. A construct's reader lives in that root beside the regex the emitter escapes - against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`. + against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`. A rule both + directions must answer alike — whether a list marker interrupts a paragraph — is one function + there too, never a copy per direction, however conservative the copy would be. - The attribute vocabulary is ADF's: `adf/` walks it and narrows each value to its kind, and a format spells the narrowed value. A spelling that re-checks the type is the check's second copy. - Explicit over implicit; descriptive names; no catch-all files (`utils`, `helpers`, `misc`); a diff --git a/spec/flavour.md b/spec/flavour.md index 4bac422..839c3aa 100644 --- a/spec/flavour.md +++ b/spec/flavour.md @@ -19,9 +19,9 @@ normalizes to it through the round-trip. - Bullet lists `- `; ordered lists incrementing `1.` `2.` `3.`, the first number the node's `order` attribute. Continuation lines align with the first character after the marker (two spaces for `- `, three for `1. `); blank lines inside an item are empty lines, none - between a nested list and a CommonMark block above it — one where the nested list's own marker - cannot interrupt a paragraph (any ordered list, or a bullet list whose first item opens with an - empty block), which the block above would otherwise swallow. Blank lines between items normalize + between a nested list and a CommonMark block above it — one wherever the nested list's own + marker cannot interrupt a paragraph (an ordered list whose first number is not 1, or a list + whose first item is empty), whatever block sits above it. Blank lines between items normalize away; ADF does not record tightness. - Blockquotes prefix lines with `> `; a blank line inside a blockquote is a bare `>`. - ATX headings (`#` … `######`); setext input normalizes to ATX. diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index 619c05c..b8cad49 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -86,6 +86,12 @@ export function isUnicodeWhitespace(character: string): boolean { return unicodeWhitespace.test(character) } +// The first marker of a list, `undefined` for a bullet: one answer both directions read, or the emitter +// spells a list the parser folds into the paragraph above it. +export function markerInterruptsParagraph(start: number | undefined, empty: boolean): boolean { + return !empty && (start === undefined || start === 1) +} + export function openingCodeFence(line: string): { info: string; marker: string } | undefined { const marker = codeFenceOpener.exec(line)?.[1] if (marker === undefined) return undefined diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 2e94731..6647675 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -267,6 +267,7 @@ test('parts a nested list the tight spelling would swallow from the block above const outer = (...content: AdfNode[]): AdfDocument => document({ content: [item(...content)], type: 'bulletList' }) const ordered: AdfNode = { attrs: { order: 2 }, content: [item(text('b'))], type: 'orderedList' } assert.equal(markdown(adfToMarkdown(outer(text('a'), ordered))), '- a\n\n 2. b\n') + assert.equal(markdown(adfToMarkdown(outer(text('a'), { ...ordered, attrs: { order: 1 } }))), '- a\n 1. b\n') assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item()], type: 'bulletList' }))), '- a\n\n -\n') assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item(text('b'))], type: 'bulletList' }))), '- a\n - b\n') }) @@ -381,6 +382,9 @@ test('refuses a document nested deeper than the emitter carries', () => { let node: AdfNode = paragraph({ text: 'x', type: 'text' }) for (let depth = 0; depth < 600; depth += 1) node = { content: [node], type: 'blockquote' } assert.equal(code(adfToMarkdown(document(node))), 'unsupported-nesting-depth') + let carried: AdfNode = paragraph({ text: 'x', type: 'text' }) + for (let depth = 0; depth < 500; depth += 1) carried = { content: [carried], type: 'blockquote' } + assert.ok(adfToMarkdown(document(carried)).ok) }) test('emits an empty list item without trailing whitespace', () => { diff --git a/src/markdown/emit/adf-to-markdown.ts b/src/markdown/emit/adf-to-markdown.ts index d3f016e..1975ed2 100644 --- a/src/markdown/emit/adf-to-markdown.ts +++ b/src/markdown/emit/adf-to-markdown.ts @@ -7,7 +7,7 @@ import { carriesOnly, isAdfDocument } from '../../adf/document.ts' import { emitInlineLine } from './inline-line.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' import { fencedCodeBlock } from '../backtick-runs.ts' -import { holdsControlCharacter, holdsEntityReference, holdsNullCharacter, isThematicBreak } from '../commonmark-grammar.ts' +import { holdsControlCharacter, holdsEntityReference, holdsNullCharacter, isThematicBreak, markerInterruptsParagraph } from '../commonmark-grammar.ts' import { largestNesting } from '../../nesting.ts' import { spellDirectiveHeader } from './block-directive-spelling.ts' import { tryImage } from './image.ts' @@ -72,8 +72,10 @@ function separationBetween(previous: PlacedBlock, next: PlacedBlock, container: } function interruptsParagraph(node: AdfNode): boolean { - if (node.type === 'orderedList') return false - return ((node.content ?? [])[0]?.content ?? []).length > 0 + const items = node.content ?? [] + const empty = (items[0]?.content ?? []).length === 0 + if (node.type !== 'orderedList') return markerInterruptsParagraph(undefined, empty) + return markerInterruptsParagraph(listStart(node, items.length) ?? 0, empty) } function emitBlock(node: AdfNode, path: ConvertErrorPath, depth: number): Result { diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index f21e924..7487267 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -1,5 +1,14 @@ import type { LinkDefinition } from './link-reference-definitions.ts' -import { atxHeading, claimsDirectiveLine, claimsPipeLine, closingCodeFence, isThematicBreak, openingCodeFence, setextHeadingLevel } from '../commonmark-grammar.ts' +import { + atxHeading, + claimsDirectiveLine, + claimsPipeLine, + closingCodeFence, + isThematicBreak, + markerInterruptsParagraph, + openingCodeFence, + setextHeadingLevel, +} from '../commonmark-grammar.ts' import { openingHtmlBlock } from './html-blocks.ts' import { readLinkDefinitions } from './link-reference-definitions.ts' @@ -85,10 +94,11 @@ function continuesContainer(walk: Walk, container: OpenContainer, line: string): } function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: number): { opened: boolean; rest: string } { + const unmatched = walk.stack[depth] let opened = false let rest = line while (leadingColumns(rest) < indentedCodeColumns) { - const start = containerStart(rest, opened ? false : paragraphOpen, opened ? undefined : walk.stack[depth]) + const start = containerStart(rest, opened ? false : paragraphOpen, opened ? undefined : unmatched) if (start === undefined) break if (!opened) closeContainers(walk, depth) opened = true @@ -102,11 +112,15 @@ function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenCon const opener = removeColumns(line, largestOpenerIndentation) if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) } if (isThematicBreak(opener) || (paragraphOpen && setextHeadingLevel(opener) !== undefined)) return undefined + return itemStart(line, opener, paragraphOpen, enclosing) +} + +function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { const marker = itemMarker(opener) if (marker === undefined) return undefined const after = opener.slice(marker.width) const blank = blankLine.test(after) - if (paragraphOpen && (blank || (marker.list.kind === 'orderedList' && marker.list.start !== 1))) return undefined + if (paragraphOpen && !markerInterruptsParagraph(marker.list.kind === 'orderedList' ? marker.list.start : undefined, blank)) return undefined const spaces = leadingColumns(after) const padding = blank || spaces > indentedCodeColumns ? 1 : spaces const continued = enclosing?.kind === 'item' && enclosing.list.kind === marker.list.kind && enclosing.marker === marker.marker diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index c39219f..0d7e250 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -197,6 +197,7 @@ test('opens a list beside a paragraph only where the marker interrupts it', () = assert.deepEqual(content(markdownToAdf('Part.\n- - -\n')), [paragraph('Part.'), { type: 'rule' }]) assert.deepEqual(content(markdownToAdf('Part.\n-\n')), [{ attrs: { level: 2 }, content: [text('Part.')], type: 'heading' }]) assert.deepEqual(content(markdownToAdf('- a\n 2. b\n')), [bulletList(item(paragraph('a 2. b')))]) + assert.deepEqual(content(markdownToAdf('- a\n 1. b\n')), [bulletList(item(paragraph('a'), orderedList(1, item(paragraph('b')))))]) }) test('folds a lazy continuation into the paragraph the container holds', () => { diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index 19dd22f..9fdc898 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -22,16 +22,28 @@ function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: num return success(content) } +// Switched, not chained: `noImplicitReturns` then refuses the kind a later milestone adds and forgets. function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result { - if (block.kind === 'blockquote') return containerNode({ type: 'blockquote' }, block.blocks, path, depth) - if (block.kind === 'bulletList') return listNode({ type: 'bulletList' }, block.items, path, depth) - if (block.kind === 'claim') return claimFailure(block.construct, path) - if (block.kind === 'code') return success(codeBlockNode(block.language, block.text)) - if (block.kind === 'heading') return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) - if (block.kind === 'html') return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) - if (block.kind === 'orderedList') return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth) - if (block.kind === 'paragraph') return success(withContent({ type: 'paragraph' }, block.text)) - return success({ type: 'rule' }) + switch (block.kind) { + case 'blockquote': + return containerNode({ type: 'blockquote' }, block.blocks, path, depth) + case 'bulletList': + return listNode({ type: 'bulletList' }, block.items, path, depth) + case 'claim': + return claimFailure(block.construct, path) + case 'code': + return success(codeBlockNode(block.language, block.text)) + case 'heading': + return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) + case 'html': + return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) + case 'orderedList': + return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth) + case 'paragraph': + return success(withContent({ type: 'paragraph' }, block.text)) + case 'rule': + return success({ type: 'rule' }) + } } function containerNode(node: AdfNode, blocks: readonly Block[], path: ConvertErrorPath, depth: number): Result { @@ -51,8 +63,12 @@ function listNode(node: AdfNode, items: readonly Block[][], path: ConvertErrorPa } function claimFailure(construct: ClaimedConstruct, path: ConvertErrorPath): Result { - if (construct === 'directive') return failure('malformed-directive', 'the line claims a directive and parses as none', path) - return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path) + switch (construct) { + case 'directive': + return failure('malformed-directive', 'the line claims a directive and parses as none', path) + case 'pipe-table': + return failure('malformed-pipe-table', 'the line claims a pipe table and parses as none', path) + } } function codeBlockNode(language: string, text: string): AdfNode { diff --git a/src/nesting.ts b/src/nesting.ts index 1019551..89984d2 100644 --- a/src/nesting.ts +++ b/src/nesting.ts @@ -1 +1,2 @@ +// One level per block-list recursion in either direction — a list and its items count once — or the two guards disagree. export const largestNesting = 500 -- 2.52.0 From 42ec9ba0962de60935d4cb2a41a6d75386b0bd63 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 20:18:25 +0200 Subject: [PATCH 3/6] Answer the stability review: a code or HTML block swallows the marker line too --- .../commonmark-subset/code-block.json | 2 +- .../commonmark-subset/code-block.md | 1 + src/markdown/commonmark-grammar.ts | 17 +++++++-- src/markdown/parse/blocks.ts | 38 +++++++++---------- src/markdown/parse/markdown-to-adf.test.ts | 2 + src/markdown/parse/markdown-to-adf.ts | 1 - 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/corpus/round-trip/commonmark-subset/code-block.json b/corpus/round-trip/commonmark-subset/code-block.json index e22ce1e..ccc8f3d 100644 --- a/corpus/round-trip/commonmark-subset/code-block.json +++ b/corpus/round-trip/commonmark-subset/code-block.json @@ -6,7 +6,7 @@ }, "content": [ { - "text": "SELECT id\nFROM part\nWHERE qty > 0;", + "text": "SELECT id\nFROM part\nWHERE qty > 0;\n-- 1. the marker a container start would claim", "type": "text" } ], diff --git a/corpus/round-trip/commonmark-subset/code-block.md b/corpus/round-trip/commonmark-subset/code-block.md index 1b22ef2..554645b 100644 --- a/corpus/round-trip/commonmark-subset/code-block.md +++ b/corpus/round-trip/commonmark-subset/code-block.md @@ -2,4 +2,5 @@ SELECT id FROM part WHERE qty > 0; +-- 1. the marker a container start would claim ``` diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index b8cad49..09dbdf7 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -16,10 +16,11 @@ const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/ const codeFenceOpener = /^(`{3,}|~{3,})/ const directiveClaim = /^:{2,}(?:[A-Za-z0-9]|[ \t]*$)/ const pipeClaim = /^\|/ +const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. -const firstCharacterOpeners = [atxHeadingOpener, /^>/, /^[*+-](?:[ \t]|$)/, codeFenceOpener, /^:{2,}/, pipeClaim] +const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] -const orderedListOpener = /^(\d{1,9})[.)](?:[ \t]|$)/ +const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u @@ -86,8 +87,16 @@ export function isUnicodeWhitespace(character: string): boolean { return unicodeWhitespace.test(character) } -// The first marker of a list, `undefined` for a bullet: one answer both directions read, or the emitter -// spells a list the parser folds into the paragraph above it. +// `start` is the list's first number, `undefined` for a bullet. +export function listMarker(line: string): { delimiter: string; start: number | undefined; width: number } | undefined { + const ordered = orderedListOpener.exec(line) + if (ordered !== null) { + const digits = ordered[1] ?? '' + return { delimiter: ordered[2] ?? '', start: Number(digits), width: digits.length + 1 } + } + return bulletListOpener.test(line) ? { delimiter: line.charAt(0), start: undefined, width: 1 } : undefined +} + export function markerInterruptsParagraph(start: number | undefined, empty: boolean): boolean { return !empty && (start === undefined || start === 1) } diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index 7487267..cc626e8 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -5,6 +5,7 @@ import { claimsPipeLine, closingCodeFence, isThematicBreak, + listMarker, markerInterruptsParagraph, openingCodeFence, setextHeadingLevel, @@ -39,13 +40,9 @@ type OpenLeaf = type ContainerStart = { kind: 'blockquote'; rest: string } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: string } -type ItemMarker = { list: ListBlock; marker: string; width: number } - type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] } const blankLine = /^[ \t]*$/ -const bulletMarker = /^[-*+](?=[ \t]|$)/ -const orderedMarker = /^(\d{1,9})([.)])(?=[ \t]|$)/ const indentedCodeColumns = 4 const largestOpenerIndentation = 3 const tabStop = 4 @@ -59,6 +56,11 @@ export function parseBlocks(markdown: string): ParsedBlocks { function readLine(walk: Walk, line: string): void { const matched = matchContainers(walk, line) + // A leaf that swallows whole lines takes the marker too: no container opens inside a code or HTML block. + if (matched.depth === walk.stack.length && swallowsLines(walk.leaf)) { + readBlockLine(walk, matched.rest) + return + } const paragraphOpen = matched.depth === walk.stack.length && walk.leaf?.kind === 'paragraph' const opened = openContainers(walk, matched.rest, paragraphOpen, matched.depth) if (!opened.opened && matched.depth < walk.stack.length) { @@ -71,6 +73,10 @@ function readLine(walk: Walk, line: string): void { readBlockLine(walk, opened.rest) } +function swallowsLines(leaf: OpenLeaf | undefined): boolean { + return leaf?.kind === 'fenced-code' || leaf?.kind === 'html' +} + function matchContainers(walk: Walk, line: string): { depth: number; rest: string } { let depth = 0 let rest = line @@ -111,38 +117,32 @@ function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { const opener = removeColumns(line, largestOpenerIndentation) if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) } - if (isThematicBreak(opener) || (paragraphOpen && setextHeadingLevel(opener) !== undefined)) return undefined + if (isThematicBreak(opener)) return undefined return itemStart(line, opener, paragraphOpen, enclosing) } function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { - const marker = itemMarker(opener) + const marker = listMarker(opener) if (marker === undefined) return undefined const after = opener.slice(marker.width) const blank = blankLine.test(after) - if (paragraphOpen && !markerInterruptsParagraph(marker.list.kind === 'orderedList' ? marker.list.start : undefined, blank)) return undefined + if (paragraphOpen && !markerInterruptsParagraph(marker.start, blank)) return undefined const spaces = leadingColumns(after) const padding = blank || spaces > indentedCodeColumns ? 1 : spaces - const continued = enclosing?.kind === 'item' && enclosing.list.kind === marker.list.kind && enclosing.marker === marker.marker + const kind = marker.start === undefined ? 'bulletList' : 'orderedList' + const continued = enclosing?.kind === 'item' && enclosing.list.kind === kind && enclosing.marker === marker.delimiter return { fresh: !continued, indentation: leadingColumns(line) + marker.width + padding, kind: 'item', - list: continued ? enclosing.list : marker.list, - marker: marker.marker, + list: continued ? enclosing.list : openList(marker.start), + marker: marker.delimiter, rest: blank ? '' : removeColumns(after, padding), } } -function itemMarker(opener: string): ItemMarker | undefined { - const ordered = orderedMarker.exec(opener) - if (ordered !== null) { - const digits = ordered[1] ?? '' - const delimiter = ordered[2] ?? '' - return { list: { items: [], kind: 'orderedList', start: Number(digits) }, marker: delimiter, width: digits.length + 1 } - } - const bullet = bulletMarker.exec(opener)?.[0] - return bullet === undefined ? undefined : { list: { items: [], kind: 'bulletList' }, marker: bullet, width: 1 } +function openList(start: number | undefined): ListBlock { + return start === undefined ? { items: [], kind: 'bulletList' } : { items: [], kind: 'orderedList', start } } function openContainer(walk: Walk, start: ContainerStart): void { diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index 0d7e250..be8814a 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -85,6 +85,7 @@ test('reads a fenced code block, its info string the language', () => { assert.deepEqual(content(markdownToAdf('~~~ a`b\n```\n~~~\n')), [{ attrs: { language: 'a`b' }, content: [text('```')], type: 'codeBlock' }]) assert.deepEqual(content(markdownToAdf('``` a`b\n')), [paragraph('``` a`b')]) assert.deepEqual(content(markdownToAdf('```\n``` x\n```\n')), [{ content: [text('``` x')], type: 'codeBlock' }]) + assert.deepEqual(content(markdownToAdf('```\n- x\n> y\n```\n')), [{ content: [text('- x\n> y')], type: 'codeBlock' }]) }) test('strips the opening fence indentation from the content lines it holds', () => { @@ -124,6 +125,7 @@ test('refuses the raw HTML no element mapping carries', () => { assert.equal(code(markdownToAdf('\n')), 'unmappable-html') assert.deepEqual(path(markdownToAdf('Part.\n\n
\n')), ['content', 1]) assert.equal(code(markdownToAdf('
\nx\n\n:::\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('
\n- x\n
\n')), 'unmappable-html') }) test('swallows an HTML block ahead of the claim a line inside it would make', () => { diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index 9fdc898..b3d4dcf 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -22,7 +22,6 @@ function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: num return success(content) } -// Switched, not chained: `noImplicitReturns` then refuses the kind a later milestone adds and forgets. function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result { switch (block.kind) { case 'blockquote': -- 2.52.0 From 99581c8f60b0e5f56db12d6b318430987de5cd1d Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 20:24:28 +0200 Subject: [PATCH 4/6] Measure a tab from the column the containers cut it to --- src/markdown/parse/blocks.ts | 105 +++++++++++---------- src/markdown/parse/markdown-to-adf.test.ts | 10 ++ 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index cc626e8..cf9d3bf 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -38,7 +38,10 @@ type OpenLeaf = | { indentation: number; info: string; kind: 'fenced-code'; lines: string[]; marker: string } | { kind: 'paragraph'; lines: string[] } -type ContainerStart = { kind: 'blockquote'; rest: string } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: string } +type ContainerStart = { kind: 'blockquote'; rest: Line } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: Line } + +// The line from an absolute column on: a tab a cut splits keeps the stop it is measured against. +type Line = { column: number; text: string } type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] } @@ -49,12 +52,12 @@ const tabStop = 4 export function parseBlocks(markdown: string): ParsedBlocks { const walk: Walk = { blocks: [], definitions: new Map(), leaf: undefined, stack: [] } - for (const line of normalizeInput(markdown).split('\n')) readLine(walk, line) + for (const text of normalizeInput(markdown).split('\n')) readLine(walk, { column: 0, text }) closeLeaf(walk) return { blocks: walk.blocks, definitions: walk.definitions } } -function readLine(walk: Walk, line: string): void { +function readLine(walk: Walk, line: Line): void { const matched = matchContainers(walk, line) // A leaf that swallows whole lines takes the marker too: no container opens inside a code or HTML block. if (matched.depth === walk.stack.length && swallowsLines(walk.leaf)) { @@ -65,7 +68,7 @@ function readLine(walk: Walk, line: string): void { const opened = openContainers(walk, matched.rest, paragraphOpen, matched.depth) if (!opened.opened && matched.depth < walk.stack.length) { if (continuesLazily(walk, opened.rest)) { - appendParagraph(walk, opened.rest) + appendParagraph(walk, opened.rest.text) return } closeContainers(walk, matched.depth) @@ -77,7 +80,7 @@ function swallowsLines(leaf: OpenLeaf | undefined): boolean { return leaf?.kind === 'fenced-code' || leaf?.kind === 'html' } -function matchContainers(walk: Walk, line: string): { depth: number; rest: string } { +function matchContainers(walk: Walk, line: Line): { depth: number; rest: Line } { let depth = 0 let rest = line for (const container of walk.stack) { @@ -89,17 +92,21 @@ function matchContainers(walk: Walk, line: string): { depth: number; rest: strin return { depth, rest } } -function continuesContainer(walk: Walk, container: OpenContainer, line: string): string | undefined { - if (container.kind === 'blockquote') { - const opener = removeColumns(line, largestOpenerIndentation) - return opener.startsWith('>') ? removeColumns(opener.slice(1), 1) : undefined - } +function continuesContainer(walk: Walk, container: OpenContainer, line: Line): Line | undefined { + if (container.kind === 'blockquote') return blockquoteRest(removeColumns(line, largestOpenerIndentation)) // A list item begins with at most one blank line: an empty one gives the second up. - if (blankLine.test(line)) return container.blocks.length === 0 && walk.leaf === undefined ? undefined : '' + if (blankLine.test(line.text)) { + return container.blocks.length === 0 && walk.leaf === undefined ? undefined : { column: line.column, text: '' } + } return leadingColumns(line) < container.indentation ? undefined : removeColumns(line, container.indentation) } -function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: number): { opened: boolean; rest: string } { +function blockquoteRest(opener: Line): Line | undefined { + if (!opener.text.startsWith('>')) return undefined + return removeColumns({ column: opener.column + 1, text: opener.text.slice(1) }, 1) +} + +function openContainers(walk: Walk, line: Line, paragraphOpen: boolean, depth: number): { opened: boolean; rest: Line } { const unmatched = walk.stack[depth] let opened = false let rest = line @@ -114,18 +121,19 @@ function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth: return { opened, rest } } -function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { +function containerStart(line: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { const opener = removeColumns(line, largestOpenerIndentation) - if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) } - if (isThematicBreak(opener)) return undefined + const blockquote = blockquoteRest(opener) + if (blockquote !== undefined) return { kind: 'blockquote', rest: blockquote } + if (isThematicBreak(opener.text)) return undefined return itemStart(line, opener, paragraphOpen, enclosing) } -function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { - const marker = listMarker(opener) +function itemStart(line: Line, opener: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined { + const marker = listMarker(opener.text) if (marker === undefined) return undefined - const after = opener.slice(marker.width) - const blank = blankLine.test(after) + const after: Line = { column: opener.column + marker.width, text: opener.text.slice(marker.width) } + const blank = blankLine.test(after.text) if (paragraphOpen && !markerInterruptsParagraph(marker.start, blank)) return undefined const spaces = leadingColumns(after) const padding = blank || spaces > indentedCodeColumns ? 1 : spaces @@ -137,7 +145,7 @@ function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosi kind: 'item', list: continued ? enclosing.list : openList(marker.start), marker: marker.delimiter, - rest: blank ? '' : removeColumns(after, padding), + rest: blank ? after : removeColumns(after, padding), } } @@ -164,53 +172,53 @@ function closeContainers(walk: Walk, depth: number): void { } // A claimed line ends the lazy continuation CommonMark would fold it into (spec/flavour.md). -function continuesLazily(walk: Walk, line: string): boolean { - if (walk.leaf?.kind !== 'paragraph' || blankLine.test(line)) return false +function continuesLazily(walk: Walk, line: Line): boolean { + if (walk.leaf?.kind !== 'paragraph' || blankLine.test(line.text)) return false if (leadingColumns(line) >= indentedCodeColumns) return true - const opener = removeColumns(line, largestOpenerIndentation) + const opener = removeColumns(line, largestOpenerIndentation).text if (claimedConstruct(opener) !== undefined || isThematicBreak(opener)) return false return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, false) === undefined } -function readBlockLine(walk: Walk, line: string): void { +function readBlockLine(walk: Walk, line: Line): void { const leaf = walk.leaf if (leaf?.kind === 'fenced-code') { - if (closingCodeFence(removeColumns(line, largestOpenerIndentation), leaf.marker)) closeLeaf(walk) - else leaf.lines.push(removeColumns(line, leaf.indentation)) + if (closingCodeFence(removeColumns(line, largestOpenerIndentation).text, leaf.marker)) closeLeaf(walk) + else leaf.lines.push(removeColumns(line, leaf.indentation).text) return } if (leaf?.kind === 'html') { - if (leaf.closer === undefined ? blankLine.test(line) : leaf.closer.test(line)) closeLeaf(walk) + if (leaf.closer === undefined ? blankLine.test(line.text) : leaf.closer.test(line.text)) closeLeaf(walk) return } if (leaf?.kind === 'indented-code') { if (readIndentedCodeLine(leaf, line)) return closeLeaf(walk) } - if (blankLine.test(line)) { + if (blankLine.test(line.text)) { closeLeaf(walk) return } if (walk.leaf === undefined && leadingColumns(line) >= indentedCodeColumns) { - walk.leaf = { held: [], kind: 'indented-code', lines: [removeColumns(line, indentedCodeColumns)] } + walk.leaf = { held: [], kind: 'indented-code', lines: [removeColumns(line, indentedCodeColumns).text] } return } openLeaf(walk, line) } -function readIndentedCodeLine(leaf: Extract, line: string): boolean { - if (blankLine.test(line)) { - leaf.held.push(removeColumns(line, indentedCodeColumns)) +function readIndentedCodeLine(leaf: Extract, line: Line): boolean { + if (blankLine.test(line.text)) { + leaf.held.push(removeColumns(line, indentedCodeColumns).text) return true } if (leadingColumns(line) < indentedCodeColumns) return false - leaf.lines.push(...leaf.held, removeColumns(line, indentedCodeColumns)) + leaf.lines.push(...leaf.held, removeColumns(line, indentedCodeColumns).text) leaf.held.length = 0 return true } -function openLeaf(walk: Walk, line: string): void { - const opener = removeColumns(line, largestOpenerIndentation) +function openLeaf(walk: Walk, line: Line): void { + const opener = removeColumns(line, largestOpenerIndentation).text const claimed = claimedConstruct(opener) if (claimed !== undefined) { closeLeaf(walk) @@ -226,12 +234,12 @@ function openLeaf(walk: Walk, line: string): void { } const html = openingHtmlBlock(opener, walk.leaf?.kind === 'paragraph') if (html === undefined) { - appendParagraph(walk, line) + appendParagraph(walk, line.text) return } closeLeaf(walk) walk.leaf = { closer: html.closer, construct: html.construct, kind: 'html' } - if (html.closer?.test(line) === true) closeLeaf(walk) + if (html.closer?.test(line.text) === true) closeLeaf(walk) } // A setext underline over a paragraph the definitions emptied is no heading: it opens the next block. @@ -300,27 +308,28 @@ function normalizeInput(markdown: string): string { .replace(/\n$/, '') } -function leadingColumns(line: string): number { +function leadingColumns(line: Line): number { let columns = 0 - for (const character of line) { + for (const character of line.text) { if (character === ' ') columns += 1 - else if (character === '\t') columns += tabStop - (columns % tabStop) + else if (character === '\t') columns += tabStop - ((line.column + columns) % tabStop) else break } return columns } // CommonMark's tab stops: a tab the cut splits gives the columns it holds past the cut back as spaces. -function removeColumns(line: string, columns: number): string { - let removed = 0 +function removeColumns(line: Line, columns: number): Line { + const target = line.column + columns + let column = line.column let index = 0 - while (removed < columns && index < line.length) { - const character = line.charAt(index) + while (column < target && index < line.text.length) { + const character = line.text.charAt(index) if (character !== ' ' && character !== '\t') break - const width = character === ' ' ? 1 : tabStop - (removed % tabStop) + const width = character === ' ' ? 1 : tabStop - (column % tabStop) index += 1 - if (removed + width > columns) return ' '.repeat(removed + width - columns) + line.slice(index) - removed += width + if (column + width > target) return { column: target, text: ' '.repeat(column + width - target) + line.text.slice(index) } + column += width } - return line.slice(index) + return { column, text: line.text.slice(index) } } diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index be8814a..e879ba7 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -186,6 +186,16 @@ test('reads an ordered list, its first marker the order attribute', () => { assert.deepEqual(content(markdownToAdf('0. Zero\n')), [orderedList(0, item(paragraph('Zero')))]) }) +test('measures a tab from the column the containers cut it to', () => { + assert.deepEqual(content(markdownToAdf('>\t\tfoo\n')), [quote({ content: [text(' foo')], type: 'codeBlock' })]) + assert.deepEqual(content(markdownToAdf('-\t\tfoo\n')), [bulletList(item({ content: [text(' foo')], type: 'codeBlock' }))]) + assert.deepEqual(content(markdownToAdf('- foo\n\n\t\tbar\n')), [bulletList(item(paragraph('foo'), { content: [text(' bar')], type: 'codeBlock' }))]) + assert.deepEqual(content(markdownToAdf('-\t foo\n')), [bulletList(item(paragraph('foo')))]) + assert.deepEqual(content(markdownToAdf(' - foo\n - bar\n\t - baz\n')), [ + bulletList(item(paragraph('foo'), bulletList(item(paragraph('bar'), bulletList(item(paragraph('baz'))))))), + ]) +}) + test('drops the tightness ADF does not record', () => { assert.deepEqual(content(markdownToAdf('- a\n\n- b\n')), [bulletList(item(paragraph('a')), item(paragraph('b')))]) assert.deepEqual(content(markdownToAdf('- a\n\n 2. b\n')), [bulletList(item(paragraph('a'), orderedList(2, item(paragraph('b')))))]) -- 2.52.0 From 7fc0cdb8278af8cf4ed15361c70aed62b55ef65f Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 20:48:04 +0200 Subject: [PATCH 5/6] Refuse two adjacent lists on the marker spelling, not the node type --- corpus/round-trip/commonmark-subset/code-block.json | 2 +- corpus/round-trip/commonmark-subset/code-block.md | 2 +- src/markdown/emit/adf-to-markdown.test.ts | 5 ++++- src/markdown/emit/adf-to-markdown.ts | 11 +++++------ src/markdown/parse/blocks.ts | 2 +- src/markdown/parse/markdown-to-adf.test.ts | 1 + 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/corpus/round-trip/commonmark-subset/code-block.json b/corpus/round-trip/commonmark-subset/code-block.json index ccc8f3d..8008692 100644 --- a/corpus/round-trip/commonmark-subset/code-block.json +++ b/corpus/round-trip/commonmark-subset/code-block.json @@ -6,7 +6,7 @@ }, "content": [ { - "text": "SELECT id\nFROM part\nWHERE qty > 0;\n-- 1. the marker a container start would claim", + "text": "SELECT id\nFROM part\nWHERE qty > 0;\n- 1. the markers a container start would claim", "type": "text" } ], diff --git a/corpus/round-trip/commonmark-subset/code-block.md b/corpus/round-trip/commonmark-subset/code-block.md index 554645b..527bb73 100644 --- a/corpus/round-trip/commonmark-subset/code-block.md +++ b/corpus/round-trip/commonmark-subset/code-block.md @@ -2,5 +2,5 @@ SELECT id FROM part WHERE qty > 0; --- 1. the marker a container start would claim +- 1. the markers a container start would claim ``` diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 6647675..ba97844 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -130,9 +130,12 @@ test('refuses a line whose start block parsing would claim', () => { assert.equal(code(adfToMarkdown(document(paragraph({ marks: [{ type: 'code' }], text: '```', type: 'text' })))), 'unspellable-line-start') }) -test('refuses two adjacent lists of the same kind', () => { +test('refuses two adjacent lists of the same kind, the marker spelling being what merges', () => { const list: AdfNode = { content: [{ content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }], type: 'bulletList' } assert.equal(code(adfToMarkdown(document(list, list))), 'unspellable-adjacent-lists') + const carried: AdfNode = { ...list, attrs: { unknown: 'x' } } + assert.ok(markdown(adfToMarkdown(document(carried, carried))).startsWith('```adf\n')) + assert.ok(markdown(adfToMarkdown(document(carried, list))).endsWith('```\n\n- x\n')) }) test('carries a node type no section spells', () => { diff --git a/src/markdown/emit/adf-to-markdown.ts b/src/markdown/emit/adf-to-markdown.ts index 1975ed2..c03f15b 100644 --- a/src/markdown/emit/adf-to-markdown.ts +++ b/src/markdown/emit/adf-to-markdown.ts @@ -14,13 +14,12 @@ import { tryImage } from './image.ts' import { tryPipeTable } from './pipe-table.ts' type BlockContainer = 'directive' | 'document' | 'list-item' -type BlockSpelling = 'commonmark' | 'directive' +type BlockSpelling = 'commonmark' | 'directive' | 'list' type EmittedBody = { fenceColons: number; text: string } type EmittedBlock = EmittedBody & { spelling: BlockSpelling } type PlacedBlock = EmittedBlock & { node: AdfNode; path: ConvertErrorPath } const largestListMarker = 999999999 -const listTypes = ['bulletList', 'orderedList'] export function adfToMarkdown(document: AdfDocument): Result { if (!isAdfDocument(document)) return failure('not-an-adf-document', 'the value is not an ADF document', []) @@ -55,9 +54,9 @@ function emitBlocks(nodes: readonly AdfNode[], container: BlockContainer, path: } function separationBetween(previous: PlacedBlock, next: PlacedBlock, container: BlockContainer): Result { - const plainPair = previous.spelling === 'commonmark' && next.spelling === 'commonmark' - if (plainPair && listTypes.includes(next.node.type)) { - if (previous.node.type === next.node.type) { + const plainPair = previous.spelling !== 'directive' && next.spelling !== 'directive' + if (next.spelling === 'list') { + if (previous.spelling === 'list' && previous.node.type === next.node.type) { return failure('unspellable-adjacent-lists', `two adjacent ${next.node.type} nodes read back as one list`, next.path) } if (container === 'list-item') return success(interruptsParagraph(next.node) ? '\n' : '\n\n') @@ -221,7 +220,7 @@ function emitList(node: AdfNode, path: ConvertErrorPath, depth: number): Result< fenceColons = Math.max(fenceColons, emitted.value.fenceColons) lines.push(emitted.value.text) } - return success({ fenceColons, spelling: 'commonmark', text: lines.join('\n') }) + return success({ fenceColons, spelling: 'list', text: lines.join('\n') }) } function listStart(node: AdfNode, items: number): number | undefined { diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index cf9d3bf..616bc1e 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -59,7 +59,7 @@ export function parseBlocks(markdown: string): ParsedBlocks { function readLine(walk: Walk, line: Line): void { const matched = matchContainers(walk, line) - // A leaf that swallows whole lines takes the marker too: no container opens inside a code or HTML block. + // CommonMark: no container opens inside an open code or HTML block. if (matched.depth === walk.stack.length && swallowsLines(walk.leaf)) { readBlockLine(walk, matched.rest) return diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index e879ba7..4e3ab18 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -199,6 +199,7 @@ test('measures a tab from the column the containers cut it to', () => { test('drops the tightness ADF does not record', () => { assert.deepEqual(content(markdownToAdf('- a\n\n- b\n')), [bulletList(item(paragraph('a')), item(paragraph('b')))]) assert.deepEqual(content(markdownToAdf('- a\n\n 2. b\n')), [bulletList(item(paragraph('a'), orderedList(2, item(paragraph('b')))))]) + assert.deepEqual(content(markdownToAdf('- a\n\n -\n')), [bulletList(item(paragraph('a'), bulletList(item())))]) }) test('opens a list beside a paragraph only where the marker interrupts it', () => { -- 2.52.0 From bb93bc61d5936fd92f7ef28545fbf0b1fbb78d3e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 21:01:53 +0200 Subject: [PATCH 6/6] Keep the tight spelling to a CommonMark block above the nested list --- src/markdown/emit/adf-to-markdown.test.ts | 7 ++++++- src/markdown/emit/adf-to-markdown.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index ba97844..80b106e 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -134,8 +134,9 @@ test('refuses two adjacent lists of the same kind, the marker spelling being wha const list: AdfNode = { content: [{ content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }], type: 'bulletList' } assert.equal(code(adfToMarkdown(document(list, list))), 'unspellable-adjacent-lists') const carried: AdfNode = { ...list, attrs: { unknown: 'x' } } - assert.ok(markdown(adfToMarkdown(document(carried, carried))).startsWith('```adf\n')) + assert.ok(markdown(adfToMarkdown(document(carried, carried))).includes('```\n\n```adf\n')) assert.ok(markdown(adfToMarkdown(document(carried, list))).endsWith('```\n\n- x\n')) + assert.ok(markdown(adfToMarkdown(document(list, carried))).startsWith('- x\n\n```adf\n')) }) test('carries a node type no section spells', () => { @@ -273,6 +274,10 @@ test('parts a nested list the tight spelling would swallow from the block above assert.equal(markdown(adfToMarkdown(outer(text('a'), { ...ordered, attrs: { order: 1 } }))), '- a\n 1. b\n') assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item()], type: 'bulletList' }))), '- a\n\n -\n') assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item(text('b'))], type: 'bulletList' }))), '- a\n - b\n') + const list: AdfNode = { content: [item(text('b'))], type: 'bulletList' } + const panel: AdfNode = { attrs: { panelType: 'info' }, content: [text('p')], type: 'panel' } + assert.equal(markdown(adfToMarkdown(outer(panel, list))), '- :::panel info\n p\n :::\n\n - b\n') + assert.ok(markdown(adfToMarkdown(outer(text('a'), { ...list, attrs: { unknown: 'x' } }))).startsWith('- a\n\n ```adf\n')) }) test('refuses marks and attributes nested deeper than the emitter carries', () => { diff --git a/src/markdown/emit/adf-to-markdown.ts b/src/markdown/emit/adf-to-markdown.ts index c03f15b..84c2a53 100644 --- a/src/markdown/emit/adf-to-markdown.ts +++ b/src/markdown/emit/adf-to-markdown.ts @@ -55,7 +55,7 @@ function emitBlocks(nodes: readonly AdfNode[], container: BlockContainer, path: function separationBetween(previous: PlacedBlock, next: PlacedBlock, container: BlockContainer): Result { const plainPair = previous.spelling !== 'directive' && next.spelling !== 'directive' - if (next.spelling === 'list') { + if (plainPair && next.spelling === 'list') { if (previous.spelling === 'list' && previous.node.type === next.node.type) { return failure('unspellable-adjacent-lists', `two adjacent ${next.node.type} nodes read back as one list`, next.path) } -- 2.52.0