From 4ad80e79c5d483b23a5947bf322fd05a3445ea5f Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Mon, 31 Aug 2026 22:33:54 +0200 Subject: [PATCH] Read the emphasis, the links and the lone image CommonMark spells (#34) --- AGENTS.md | 7 +- corpus/errors/image-in-text.error | 1 + corpus/errors/image-in-text.md | 1 + corpus/normalization/emphasis-spellings.json | 88 +++++ corpus/normalization/emphasis-spellings.md | 3 + corpus/normalization/link-references.json | 64 ++++ corpus/normalization/link-references.md | 4 + .../link-destination-parenthesis.json | 0 .../link-destination-parenthesis.md | 1 + .../commonmark-subset}/link-title-quote.json | 0 .../commonmark-subset/link-title-quote.md | 1 + .../link-destination-parenthesis.error | 1 - corpus/unspellable/link-title-quote.error | 1 - spec/flavour.md | 10 +- src/adf/editor-normal.ts | 32 ++ src/markdown/backtick-runs.ts | 6 +- src/markdown/commonmark-grammar.ts | 19 +- src/markdown/emit/adf-to-markdown.test.ts | 16 +- src/markdown/emit/destination-spelling.ts | 38 -- src/markdown/emit/inline-line.ts | 8 +- src/markdown/emit/line-escaping.ts | 9 +- src/markdown/emphasis-matching.ts | 9 +- src/markdown/link-syntax.ts | 129 +++++++ src/markdown/parse/blocks.test.ts | 8 +- src/markdown/parse/blocks.ts | 2 +- src/markdown/parse/inline-content.ts | 353 +++++++++++++++--- .../parse/link-reference-definitions.ts | 80 +--- src/markdown/parse/markdown-to-adf.test.ts | 158 +++++++- src/markdown/parse/markdown-to-adf.ts | 39 +- src/result.ts | 1 + todo.md | 28 +- 31 files changed, 899 insertions(+), 218 deletions(-) create mode 100644 corpus/errors/image-in-text.error create mode 100644 corpus/errors/image-in-text.md create mode 100644 corpus/normalization/emphasis-spellings.json create mode 100644 corpus/normalization/emphasis-spellings.md create mode 100644 corpus/normalization/link-references.json create mode 100644 corpus/normalization/link-references.md rename corpus/{unspellable => round-trip/commonmark-subset}/link-destination-parenthesis.json (100%) create mode 100644 corpus/round-trip/commonmark-subset/link-destination-parenthesis.md rename corpus/{unspellable => round-trip/commonmark-subset}/link-title-quote.json (100%) create mode 100644 corpus/round-trip/commonmark-subset/link-title-quote.md delete mode 100644 corpus/unspellable/link-destination-parenthesis.error delete mode 100644 corpus/unspellable/link-title-quote.error create mode 100644 src/adf/editor-normal.ts delete mode 100644 src/markdown/emit/destination-spelling.ts create mode 100644 src/markdown/link-syntax.ts diff --git a/AGENTS.md b/AGENTS.md index 853a8d7..1ea936b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,7 +148,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - `src/adf/` holds ADF's own knowledge and imports no format. Each format directory (`markdown/`, `html/`) parts into `emit/` (ADF→format) and `parse/` (format→ADF), its root holding what both directions read. A construct's reader lives in that root beside the regex the emitter escapes - against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`. A rule both + against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`, unless it is + part of a construct the root already holds — a grammar stays in one file rather than splitting + across the seam. A rule both directions must answer alike — whether a list marker interrupts a paragraph — is one function there too, never a copy per direction, however conservative the copy would be. - The attribute vocabulary is ADF's: `adf/` walks it and narrows each value to its kind, and a @@ -180,7 +182,8 @@ One-line commit messages and PR titles; short PR summaries. No AI-attribution ma No wiki markup (§1), no network or filesystem I/O, no name→id resolution (§3), no ADF schema validation or exported validator — a refusal that keeps the round-trip is not schema validation, -so the one a spelled node carrying the same mark type twice earns stays, no shipped CSS (§4), no +so the one a spelled node carrying the same mark type twice earns stays, and input nesting a +spelling inside its own kind (`*(*a*)*`) names that mark once, no shipped CSS (§4), no streaming APIs, no performance budget past §11's scanning rule — nothing here is tuned, and no figure is promised. A CLI is a later goal (`todo.md`), not a non-goal. diff --git a/corpus/errors/image-in-text.error b/corpus/errors/image-in-text.error new file mode 100644 index 0000000..7fe8b78 --- /dev/null +++ b/corpus/errors/image-in-text.error @@ -0,0 +1 @@ +unmappable-image diff --git a/corpus/errors/image-in-text.md b/corpus/errors/image-in-text.md new file mode 100644 index 0000000..ede305f --- /dev/null +++ b/corpus/errors/image-in-text.md @@ -0,0 +1 @@ +See ![the moon](https://example.com/moon.png) tonight. diff --git a/corpus/normalization/emphasis-spellings.json b/corpus/normalization/emphasis-spellings.json new file mode 100644 index 0000000..1d72db8 --- /dev/null +++ b/corpus/normalization/emphasis-spellings.json @@ -0,0 +1,88 @@ +{ + "content": [ + { + "content": [ + { + "marks": [ + { + "type": "em" + } + ], + "text": "Torque", + "type": "text" + }, + { + "text": " the ", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "M8 bolt", + "type": "text" + }, + { + "text": " to 25 Nm, then ", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + }, + { + "type": "strong" + } + ], + "text": "log it", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "text": "Un", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "bolt", + "type": "text" + }, + { + "text": "ing needs a ", + "type": "text" + }, + { + "marks": [ + { + "type": "strike" + } + ], + "text": "10 mm", + "type": "text" + }, + { + "text": " 13 mm spanner.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/emphasis-spellings.md b/corpus/normalization/emphasis-spellings.md new file mode 100644 index 0000000..acc1f01 --- /dev/null +++ b/corpus/normalization/emphasis-spellings.md @@ -0,0 +1,3 @@ +*Torque* the __M8 bolt__ to 25 Nm, then ***log it***. + +Un*bolt*ing needs a ~~10 mm~~ 13 mm spanner. diff --git a/corpus/normalization/link-references.json b/corpus/normalization/link-references.json new file mode 100644 index 0000000..6bc3045 --- /dev/null +++ b/corpus/normalization/link-references.json @@ -0,0 +1,64 @@ +{ + "content": [ + { + "content": [ + { + "text": "See ", + "type": "text" + }, + { + "marks": [ + { + "attrs": { + "href": "https://example.com/changelog" + }, + "type": "link" + } + ], + "text": "the changelog", + "type": "text" + }, + { + "text": " and ", + "type": "text" + }, + { + "marks": [ + { + "attrs": { + "href": "https://example.com/guide", + "title": "Setup guide" + }, + "type": "link" + } + ], + "text": "the guide", + "type": "text" + }, + { + "text": ", or mail ", + "type": "text" + }, + { + "marks": [ + { + "attrs": { + "href": "mailto:ops@example.com" + }, + "type": "link" + } + ], + "text": "ops@example.com", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/link-references.md b/corpus/normalization/link-references.md new file mode 100644 index 0000000..eb13e09 --- /dev/null +++ b/corpus/normalization/link-references.md @@ -0,0 +1,4 @@ +See [the changelog][changes] and [the guide], or mail . + +[changes]: https://example.com/changelog +[the guide]: https://example.com/guide "Setup guide" diff --git a/corpus/unspellable/link-destination-parenthesis.json b/corpus/round-trip/commonmark-subset/link-destination-parenthesis.json similarity index 100% rename from corpus/unspellable/link-destination-parenthesis.json rename to corpus/round-trip/commonmark-subset/link-destination-parenthesis.json diff --git a/corpus/round-trip/commonmark-subset/link-destination-parenthesis.md b/corpus/round-trip/commonmark-subset/link-destination-parenthesis.md new file mode 100644 index 0000000..6457e90 --- /dev/null +++ b/corpus/round-trip/commonmark-subset/link-destination-parenthesis.md @@ -0,0 +1 @@ +[The torque table](https://example.com/a\)b) diff --git a/corpus/unspellable/link-title-quote.json b/corpus/round-trip/commonmark-subset/link-title-quote.json similarity index 100% rename from corpus/unspellable/link-title-quote.json rename to corpus/round-trip/commonmark-subset/link-title-quote.json diff --git a/corpus/round-trip/commonmark-subset/link-title-quote.md b/corpus/round-trip/commonmark-subset/link-title-quote.md new file mode 100644 index 0000000..e61387b --- /dev/null +++ b/corpus/round-trip/commonmark-subset/link-title-quote.md @@ -0,0 +1 @@ +[The torque table](https://example.com/torque "He said \"hi\"") diff --git a/corpus/unspellable/link-destination-parenthesis.error b/corpus/unspellable/link-destination-parenthesis.error deleted file mode 100644 index 8da7bcc..0000000 --- a/corpus/unspellable/link-destination-parenthesis.error +++ /dev/null @@ -1 +0,0 @@ -unspellable-link-destination diff --git a/corpus/unspellable/link-title-quote.error b/corpus/unspellable/link-title-quote.error deleted file mode 100644 index a1f2950..0000000 --- a/corpus/unspellable/link-title-quote.error +++ /dev/null @@ -1 +0,0 @@ -unspellable-link-title diff --git a/spec/flavour.md b/spec/flavour.md index 839c3aa..7a93968 100644 --- a/spec/flavour.md +++ b/spec/flavour.md @@ -37,9 +37,10 @@ normalizes to it through the round-trip. CommonMark admits no spelling — the end of a block, inside an ATX heading — or where the node carries an attribute, it is the inline directive. - An empty paragraph — real payloads carry them — is `::paragraph`. -- Links `[text](url)`; `<…>` around a destination containing spaces; title in double quotes. - `` autolink form only when the text equals the destination and the destination is a valid - CommonMark autolink (absolute URI). +- Links `[text](url)`; `<…>` around a destination containing spaces; title in double quotes. A + backslash escapes a parenthesis the destination leaves unbalanced, and a quote inside the title; + a balanced pair stays bare. `` autolink form only when the text equals the destination and + the destination is a valid CommonMark autolink (absolute URI). - Paragraphs on one line — no soft wrapping; a soft line break in input becomes a single space. - Entity references in input decode to their characters; output backslash-escapes only where text would otherwise parse as syntax, scanning the assembled line rather than each text node: escape @@ -244,7 +245,8 @@ The moon, at night. **The CommonMark image.** A paragraph whose entire inline content is one image `![alt](url)` is a `mediaSingle` with attrs exactly `{"layout":"center"}` holding an `external` `media` — `url` -from the destination, `alt` the description's plain-text content when non-empty. `adfToMarkdown` +from the destination, `alt` the description's plain-text content when non-empty — a link or image +inside it contributing its own text, and a break of either kind a space. `adfToMarkdown` emits the image form for exactly that shape — those attrs and no others, no marks on either node, no caption, and a `media` carrying nothing beyond `alt`, `type` and `url` — and only where CommonMark spells the pair: a destination or a description the image form cannot hold, an empty diff --git a/src/adf/editor-normal.ts b/src/adf/editor-normal.ts new file mode 100644 index 0000000..7f65586 --- /dev/null +++ b/src/adf/editor-normal.ts @@ -0,0 +1,32 @@ +import type { AdfMark, AdfNode } from './document.ts' +import { serializeCanonicalJson } from '../canonical-json.ts' + +export function sameMark(candidate: AdfMark, mark: AdfMark): boolean { + return markKey(candidate) === markKey(mark) +} + +// AGENTS.md §2: adjacent text nodes carrying identical marks are one node. +export function mergeAdjacentText(nodes: readonly AdfNode[]): AdfNode[] { + const merged: AdfNode[] = [] + for (const node of nodes) { + const previous = merged[merged.length - 1] + if (previous !== undefined && previous.type === 'text' && node.type === 'text' && sameMarks(previous, node)) { + merged[merged.length - 1] = { ...previous, text: `${previous.text ?? ''}${node.text ?? ''}` } + continue + } + merged.push(node) + } + return merged +} + +function sameMarks(previous: AdfNode, node: AdfNode): boolean { + return marksKey(previous.marks ?? []) === marksKey(node.marks ?? []) +} + +function marksKey(marks: readonly AdfMark[]): string { + return marks.map(markKey).join('\n') +} + +function markKey(mark: AdfMark): string { + return `${mark.type} ${serializeCanonicalJson(mark.attrs ?? {}, 'compact')}` +} diff --git a/src/markdown/backtick-runs.ts b/src/markdown/backtick-runs.ts index 940573a..469f972 100644 --- a/src/markdown/backtick-runs.ts +++ b/src/markdown/backtick-runs.ts @@ -1,7 +1,7 @@ +import { runLength } from './emphasis-matching.ts' + export function backtickRun(text: string, index: number): number { - let length = 0 - while (text.charAt(index + length) === '`') length += 1 - return length + return text.charAt(index) === '`' ? runLength(text, index) : 0 } // Where the run of exactly `opener` backticks closing a code span begins, `undefined` where none does. diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index edeb4e7..570da7c 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -56,7 +56,10 @@ const pipeClaim = /^\|/ const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] -const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y +// CommonMark 0.31.2, Autolinks: the email production, whose label may not open or close with a hyphen. +const emailNameSource = "[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+" +const emailLabelSource = '[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?' +const emailAutolink = new RegExp(`<${emailNameSource}@${emailLabelSource}(?:\\.${emailLabelSource})*>`, 'y') const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ @@ -197,13 +200,21 @@ export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlB } export function opensBracketedAutolink(text: string, index: number): boolean { - bracketedAutolink.lastIndex = index - return bracketedAutolink.test(text) + return readBracketedAutolink(text, index) !== undefined } export function opensEmailAutolink(text: string, index: number): boolean { + return readEmailAutolink(text, index) !== undefined +} + +export function readBracketedAutolink(text: string, index: number): number | undefined { + bracketedAutolink.lastIndex = index + return bracketedAutolink.exec(text)?.[0].length +} + +export function readEmailAutolink(text: string, index: number): number | undefined { emailAutolink.lastIndex = index - return emailAutolink.test(text) + return emailAutolink.exec(text)?.[0].length } export function setextHeadingLevel(line: string): number | undefined { diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 7ab0676..aebacc5 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -76,7 +76,6 @@ test('spells a code block language no info string holds as an attribute', () => test('refuses a link destination CommonMark cannot spell', () => { const link = (href: string): AdfDocument => document(paragraph({ marks: [{ attrs: { href }, type: 'link' }], text: 't', type: 'text' })) - assert.equal(code(adfToMarkdown(link('https://example.com/a)b'))), 'unspellable-link-destination') assert.equal(code(adfToMarkdown(link('https://example.com/a b>c'))), 'unspellable-link-destination') assert.equal(code(adfToMarkdown(link(' { assert.equal(code(adfToMarkdown(link('https://example.com/a\nb'))), 'unspellable-link-destination') const entity = 'https://example.com/?a=1&b=2' assert.equal(code(adfToMarkdown(document(paragraph({ marks: [{ attrs: { href: entity }, type: 'link' }], text: entity, type: 'text' })))), 'unspellable-link-destination') - assert.equal(markdown(adfToMarkdown(link('https://en.example.com/a_(b)'))), '[t](https://en.example.com/a_(b))\n') }) -test('refuses a link title CommonMark cannot spell', () => { +test('escapes the parenthesis a link destination leaves unbalanced, and no other', () => { + const link = (href: string): string => markdown(adfToMarkdown(document(paragraph({ marks: [{ attrs: { href }, type: 'link' }], text: 't', type: 'text' })))) + assert.equal(link('https://en.example.com/a_(b)'), '[t](https://en.example.com/a_(b))\n') + assert.equal(link('https://example.com/a)b'), '[t](https://example.com/a\\)b)\n') + assert.equal(link('https://example.com/a(b'), '[t](https://example.com/a\\(b)\n') + assert.equal(link('https://example.com/)(') , '[t](https://example.com/\\)\\()\n') + assert.equal(link('https://example.com/a (b'), '[t]()\n') +}) + +test('escapes the quote a link title holds, and refuses the rest', () => { const titled = (title: string): AdfDocument => document(paragraph({ marks: [{ attrs: { href: 'https://example.com/', title }, type: 'link' }], text: 't', type: 'text' })) - assert.equal(code(adfToMarkdown(titled('He said "hi"'))), 'unspellable-link-title') + assert.equal(markdown(adfToMarkdown(titled('He said "hi"'))), '[t](https://example.com/ "He said \\"hi\\"")\n') assert.equal(code(adfToMarkdown(titled('a\nb'))), 'unspellable-link-title') + assert.equal(code(adfToMarkdown(titled('a\\b'))), 'unspellable-link-title') }) test('carries a link mark the link spelling cannot write', () => { diff --git a/src/markdown/emit/destination-spelling.ts b/src/markdown/emit/destination-spelling.ts deleted file mode 100644 index 028ba80..0000000 --- a/src/markdown/emit/destination-spelling.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' -import { holdsControlCharacter } from '../commonmark-grammar.ts' -import { holdsEntityReference } from '../entity-references.ts' - -export function spellDestination(href: string, path: ConvertErrorPath): Result { - if (holdsControlCharacter(href)) return failure('unspellable-link-destination', 'a link destination holds a control character', path) - if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination', path) - if (holdsEntityReference(href)) { - return failure('unspellable-link-destination', 'a link destination holds an entity reference that decodes on the way back', path) - } - if (href.includes(' ')) { - if (/[<>]/.test(href)) { - return failure('unspellable-link-destination', 'no canonical escape spells an angle bracket beside a space in a link destination', path) - } - return success(`<${href}>`) - } - if (href.startsWith('<')) return failure('unspellable-link-destination', 'a bare link destination cannot begin with an angle bracket', path) - if (!balanced(href)) return failure('unspellable-link-destination', 'no canonical escape spells an unbalanced parenthesis in a link destination', path) - return success(href) -} - -export function spellTitle(title: string, path: ConvertErrorPath): Result { - if (/["\n\r\\]/.test(title)) { - return failure('unspellable-link-title', 'no canonical escape spells a quote, backslash or newline in a link title', path) - } - if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title holds an entity reference that decodes on the way back', path) - return success(` "${title}"`) -} - -function balanced(href: string): boolean { - let depth = 0 - for (const character of href) { - if (character === '(') depth += 1 - if (character === ')') depth -= 1 - if (depth < 0) return false - } - return depth === 0 -} diff --git a/src/markdown/emit/inline-line.ts b/src/markdown/emit/inline-line.ts index ba61347..71ab82a 100644 --- a/src/markdown/emit/inline-line.ts +++ b/src/markdown/emit/inline-line.ts @@ -9,9 +9,9 @@ import { inlineDirective } from '../../adf/inline-directives.ts' import { largestNesting } from '../../nesting.ts' import { longestBacktickRun } from '../backtick-runs.ts' import { markSpelling, spellMarkAttributes } from '../mark-spellings.ts' -import { serializeCanonicalJson } from '../../canonical-json.ts' +import { sameMark } from '../../adf/editor-normal.ts' import { spellAttributes, spellStringAttribute } from '../directive-attributes.ts' -import { spellDestination, spellTitle } from './destination-spelling.ts' +import { spellDestination, spellTitle } from '../link-syntax.ts' import { spellInlineNodeAttributes } from './inline-directive-spelling.ts' type EmittedLine = { line: string; segments: InlineSegment[] } @@ -297,7 +297,3 @@ function emitLink(nodes: readonly AdfNode[], mark: AdfMark, depth: number, range return success({ segments: [syntax('['), ...inner.value.segments, syntax(`](${destination.value}${spelledTitle.value})`)] }) } -function sameMark(candidate: AdfMark, mark: AdfMark): boolean { - if (candidate.type !== mark.type) return false - return serializeCanonicalJson(candidate.attrs ?? {}, 'compact') === serializeCanonicalJson(mark.attrs ?? {}, 'compact') -} diff --git a/src/markdown/emit/line-escaping.ts b/src/markdown/emit/line-escaping.ts index 34ddb70..f139af7 100644 --- a/src/markdown/emit/line-escaping.ts +++ b/src/markdown/emit/line-escaping.ts @@ -1,5 +1,5 @@ import { backtickRun, closingBacktickRun } from '../backtick-runs.ts' -import { delimiterFlags, isWordCharacter, matchEmphasis } from '../emphasis-matching.ts' +import { delimiterFlags, isWordCharacter, matchEmphasis, runLength } from '../emphasis-matching.ts' import { backslashEscape, escapesLineClaim, inlineHtmlConstruct, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts' import { opensInlineDirective } from '../directive-attributes.ts' import { readEntityReference } from '../entity-references.ts' @@ -260,13 +260,6 @@ function startsRun(scan: string, index: number, escaped: ReadonlySet): b return scan.charAt(index - 1) !== scan.charAt(index) } -function runLength(scan: string, index: number): number { - const character = scan.charAt(index) - let length = 0 - while (scan.charAt(index + length) === character) length += 1 - return length -} - function charAt(text: string, index: number): string { return index < 0 ? '' : text.charAt(index) } diff --git a/src/markdown/emphasis-matching.ts b/src/markdown/emphasis-matching.ts index a5897d0..b8a8942 100644 --- a/src/markdown/emphasis-matching.ts +++ b/src/markdown/emphasis-matching.ts @@ -2,7 +2,7 @@ import { isUnicodeWhitespace } from './commonmark-grammar.ts' type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } -type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } +export type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } type Candidate = { head: number @@ -64,6 +64,13 @@ export function matchEmphasis(runs: readonly Run[]): E return pairings } +export function runLength(text: string, index: number): number { + const character = text.charAt(index) + let length = 0 + while (text.charAt(index + length) === character) length += 1 + return length +} + function candidates(runs: readonly Run[]): Candidate | undefined { let first: Candidate | undefined let previous: Candidate | undefined diff --git a/src/markdown/link-syntax.ts b/src/markdown/link-syntax.ts new file mode 100644 index 0000000..022adbb --- /dev/null +++ b/src/markdown/link-syntax.ts @@ -0,0 +1,129 @@ +import { backslashEscape, decodeTextEscapes, holdsControlCharacter } from './commonmark-grammar.ts' +import { failure, success, type ConvertErrorPath, type Result } from '../result.ts' +import { holdsEntityReference } from './entity-references.ts' + +export type LinkDefinition = { destination: string; title?: string } + +export type LinkPart = { length: number; value: string } + +const bracketedDestination = /<((?:[^\n<>\\]|\\[^\n])*)>/y +const linkLabel = /\[((?:[^[\]\\]|\\[\s\S]){0,999})\]/y +const titleClosers: Readonly> = { '"': '"', "'": "'", '(': ')' } + +// The label is matched by its normalized raw text, escapes and all. +export function readLabel(text: string, offset: number): LinkPart | undefined { + linkLabel.lastIndex = offset + const value = linkLabel.exec(text)?.[1] + return value === undefined ? undefined : { length: value.length + 2, value } +} + +export function normalizeLabel(raw: string): string { + return raw + .replace(/^[ \t\n]+|[ \t\n]+$/g, '') + .replace(/[ \t\n]+/g, ' ') + .toLowerCase() +} + +export function readDestination(text: string, offset: number): LinkPart | undefined { + bracketedDestination.lastIndex = offset + const bracketed = bracketedDestination.exec(text)?.[0] + if (bracketed !== undefined) return { length: bracketed.length, value: decodeTextEscapes(bracketed.slice(1, -1)) } + if (text.charAt(offset) === '<') return undefined + let depth = 0 + let index = offset + while (index < text.length) { + const character = text.charAt(index) + if (character === ' ' || holdsControlCharacter(character)) break + if (backslashEscape(text, index) !== undefined) { + index += 2 + continue + } + if (character === '(') depth += 1 + if (character === ')') { + depth -= 1 + if (depth < 0) break + } + index += 1 + } + return index <= offset ? undefined : { length: index - offset, value: decodeTextEscapes(text.slice(offset, index)) } +} + +// `offset` sits on the `(` a link text closes into. +export function readInlineTarget(text: string, offset: number): { definition: LinkDefinition; length: number } | undefined { + let index = skipLinkWhitespace(text, offset + 1) + let destination = '' + if (text.charAt(index) !== ')') { + const read = readDestination(text, index) + if (read === undefined) return undefined + destination = read.value + index += read.length + } + const afterDestination = index + index = skipLinkWhitespace(text, index) + const title = index > afterDestination ? readTitle(text, index) : undefined + if (title !== undefined) index = skipLinkWhitespace(text, index + title.length) + if (text.charAt(index) !== ')') return undefined + return { definition: title === undefined ? { destination } : { destination, title: title.value }, length: index + 1 - offset } +} + +export function readTitle(text: string, offset: number): LinkPart | undefined { + const opener = text.charAt(offset) + const closer = titleClosers[opener] + if (closer === undefined) return undefined + let index = offset + 1 + while (index < text.length) { + const character = text.charAt(index) + if (backslashEscape(text, index) !== undefined) { + index += 2 + continue + } + if (character === closer) return { length: index + 1 - offset, value: decodeTextEscapes(text.slice(offset + 1, index)) } + if (character === opener) return undefined + index += 1 + } + return undefined +} + +// The label, the destination and the title each take at most one line ending with them. +export function skipLinkWhitespace(text: string, offset: number): number { + const rest = text.slice(offset) + return offset + rest.length - rest.replace(/^[ \t]*\n?[ \t]*/, '').length +} + +export function spellDestination(href: string, path: ConvertErrorPath): Result { + if (holdsControlCharacter(href)) return failure('unspellable-link-destination', 'a link destination holds a control character', path) + if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination', path) + if (holdsEntityReference(href)) { + return failure('unspellable-link-destination', 'a link destination holds an entity reference that decodes on the way back', path) + } + if (href.includes(' ')) { + if (/[<>]/.test(href)) { + return failure('unspellable-link-destination', 'no canonical escape spells an angle bracket beside a space in a link destination', path) + } + return success(`<${href}>`) + } + if (href.startsWith('<')) return failure('unspellable-link-destination', 'a bare link destination cannot begin with an angle bracket', path) + return success(escapeUnbalanced(href)) +} + +export function spellTitle(title: string, path: ConvertErrorPath): Result { + if (/[\n\r\\]/.test(title)) { + return failure('unspellable-link-title', 'no canonical escape spells a backslash or newline in a link title', path) + } + if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title holds an entity reference that decodes on the way back', path) + return success(` "${title.replaceAll('"', '\\"')}"`) +} + +function escapeUnbalanced(href: string): string { + const open: number[] = [] + const unbalanced = new Set() + for (let index = 0; index < href.length; index += 1) { + const character = href.charAt(index) + if (character === '(') open.push(index) + if (character === ')' && open.pop() === undefined) unbalanced.add(index) + } + for (const index of open) unbalanced.add(index) + let spelled = '' + for (let index = 0; index < href.length; index += 1) spelled += (unbalanced.has(index) ? '\\' : '') + href.charAt(index) + return spelled +} diff --git a/src/markdown/parse/blocks.test.ts b/src/markdown/parse/blocks.test.ts index f625693..cab466b 100644 --- a/src/markdown/parse/blocks.test.ts +++ b/src/markdown/parse/blocks.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' -import type { LinkDefinition } from './link-reference-definitions.ts' +import type { LinkDefinition } from '../link-syntax.ts' import { parseBlocks } from './blocks.ts' function definitions(markdown: string): [string, LinkDefinition][] { @@ -21,9 +21,10 @@ test('keeps the link reference definitions a paragraph gives up, the first of a ]) assert.deepEqual(definitions('[a\\]b]: /url\n'), [['a\\]b', { destination: '/url' }]]) assert.deepEqual(definitions('[a]: /url(x)y\n'), [['a', { destination: '/url(x)y' }]]) - assert.deepEqual(definitions('[a]: /url\\(x\n'), [['a', { destination: '/url\\(x' }]]) + assert.deepEqual(definitions('[a]: /url\\(x\n'), [['a', { destination: '/url(x' }]]) + assert.deepEqual(definitions('[a]: /url&x\n'), [['a', { destination: '/url&x' }]]) assert.deepEqual(definitions('[a]: <>\n'), [['a', { destination: '' }]]) - assert.deepEqual(definitions('[a]: /url "He said \\"hi\\""\n'), [['a', { destination: '/url', title: 'He said \\"hi\\"' }]]) + assert.deepEqual(definitions('[a]: /url "He said \\"hi\\""\n'), [['a', { destination: '/url', title: 'He said "hi"' }]]) assert.deepEqual(definitions('[a]: /url\\\n[b]: /b\n'), [ ['a', { destination: '/url\\' }], ['b', { destination: '/b' }], @@ -47,6 +48,7 @@ test('leaves the paragraph a line no definition spells', () => { assert.deepEqual(definitions('[a]: /url "One" and more\n'), []) assert.deepEqual(definitions('[a]:\n'), []) assert.deepEqual(definitions('[a]: /url "unclosed\n'), []) + assert.deepEqual(definitions('[a]: /url (a(b)\n'), []) assert.deepEqual(kinds('[a]: /url\nPart.\n'), ['paragraph']) }) diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index b9b2126..7b6dd5d 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -1,4 +1,4 @@ -import type { LinkDefinition } from './link-reference-definitions.ts' +import type { LinkDefinition } from '../link-syntax.ts' import { atxHeading, claimsDirectiveLine, diff --git a/src/markdown/parse/inline-content.ts b/src/markdown/parse/inline-content.ts index 85e9e70..c329909 100644 --- a/src/markdown/parse/inline-content.ts +++ b/src/markdown/parse/inline-content.ts @@ -1,69 +1,326 @@ -import type { AdfNode } from '../../adf/document.ts' -import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts' +import type { AdfMark, AdfNode } from '../../adf/document.ts' +import type { EmphasisPairing } from '../emphasis-matching.ts' +import type { LinkDefinition } from '../link-syntax.ts' +import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct, readBracketedAutolink, readEmailAutolink } from '../commonmark-grammar.ts' import { backtickRun, closingBacktickRun } from '../backtick-runs.ts' +import { delimiterFlags, matchEmphasis, runLength } from '../emphasis-matching.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' +import { mergeAdjacentText } from '../../adf/editor-normal.ts' +import { normalizeLabel, readInlineTarget, readLabel } from '../link-syntax.ts' -type Run = { nodes: AdfNode[]; text: string; undecodedFrom: number } +export type InlineContent = { image: AdfNode; nodes?: undefined } | { image?: undefined; nodes: AdfNode[] } + +export type LinkDefinitions = ReadonlyMap + +type Bracket = { active: boolean; image: boolean; kind: 'open'; start: number } + +type Pairing = EmphasisPairing + +type Piece = + | Bracket + | { alt: string; kind: 'image'; node: AdfNode } + | { kind: 'nodes'; nodes: AdfNode[] } + | { canClose: boolean; canOpen: boolean; character: string; kind: 'run'; length: number } + +type Run = { canClose: boolean; canOpen: boolean; character: string; index: number; length: number } + +type Scan = { definitions: LinkDefinitions; path: ConvertErrorPath; pending: string; pieces: Piece[]; source: string } const hardBreakSpaces = / {2,}$/ +const imageAlone = 'an image fits only as a paragraph of its own' const trailingSpace = /[ \t]+$/ -export function parseInlineContent(source: string, path: ConvertErrorPath): Result { - const run: Run = { nodes: [], text: '', undecodedFrom: 0 } +export function parseInlineContent(source: string, definitions: LinkDefinitions, path: ConvertErrorPath): Result { + const scan: Scan = { definitions, path, pending: '', pieces: [], source } let index = 0 while (index < source.length) { - const character = source.charAt(index) - if (character === '\\' && source.charAt(index + 1) === '\n') { - // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash. - takeRun(run, source, index, index + 2, false) - pushNode(run, { type: 'hardBreak' }) - index += 2 - continue - } - if (character === '\n') { - const hard = hardBreakSpaces.test(source.slice(run.undecodedFrom, index)) - takeRun(run, source, index, index + 1, true) - if (hard) pushNode(run, { type: 'hardBreak' }) - else run.text += ' ' - index += 1 - continue - } - if (character === '`') { - const span = readCodeSpan(source, index) - if (span === undefined) { - index += backtickRun(source, index) - continue + switch (source.charAt(index)) { + case '\\': + index = readBackslash(scan, index) + break + case '\n': + index = readLineEnding(scan, index) + break + case '`': + index = readBackticks(scan, index) + break + case '<': { + const angle = readAngle(scan, index) + if (!angle.ok) return angle + index = angle.value + break } - takeRun(run, source, index, span.end, false) - pushNode(run, { marks: [{ type: 'code' }], text: span.text, type: 'text' }) - index = span.end - continue + case '!': + case '[': + index = openBracket(scan, index) + break + case ']': { + const closed = closeBracket(scan, index) + if (!closed.ok) return closed + index = closed.value + break + } + case '*': + case '_': + case '~': + index = readDelimiterRun(scan, index) + break + default: + scan.pending += source.charAt(index) + index += 1 } - if (character === '<') { - const construct = inlineHtmlConstruct(source, index) - if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, path) - } - index += backslashEscape(source, index) === undefined ? 1 : 2 } - takeRun(run, source, source.length, source.length, true) - pushText(run) - return success(run.nodes) + flush(scan, true) + return assemble(scan) } -function takeRun(run: Run, source: string, end: number, resume: number, strip: boolean): void { - const raw = source.slice(run.undecodedFrom, end) - run.text += decodeTextEscapes(strip ? raw.replace(trailingSpace, '') : raw) - run.undecodedFrom = resume +function readBackslash(scan: Scan, index: number): number { + if (scan.source.charAt(index + 1) === '\n') { + // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash. + flush(scan, false) + pushNode(scan, { type: 'hardBreak' }) + return index + 2 + } + const width = backslashEscape(scan.source, index) === undefined ? 1 : 2 + scan.pending += scan.source.slice(index, index + width) + return index + width } -function pushText(run: Run): void { - if (run.text !== '') run.nodes.push({ text: run.text, type: 'text' }) - run.text = '' +function readLineEnding(scan: Scan, index: number): number { + const hard = hardBreakSpaces.test(scan.pending) + flush(scan, true) + if (hard) pushNode(scan, { type: 'hardBreak' }) + else scan.pending = ' ' + return index + 1 } -function pushNode(run: Run, node: AdfNode): void { - pushText(run) - run.nodes.push(node) +function readBackticks(scan: Scan, index: number): number { + const span = readCodeSpan(scan.source, index) + if (span === undefined) { + const run = backtickRun(scan.source, index) + scan.pending += scan.source.slice(index, index + run) + return index + run + } + flush(scan, false) + pushNode(scan, { marks: [{ type: 'code' }], text: span.text, type: 'text' }) + return span.end +} + +function readAngle(scan: Scan, index: number): Result { + const autolink = readAutolink(scan.source, index) + if (autolink !== undefined) { + flush(scan, false) + pushNode(scan, autolink.node) + return success(index + autolink.length) + } + const construct = inlineHtmlConstruct(scan.source, index) + if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, scan.path) + scan.pending += '<' + return success(index + 1) +} + +function openBracket(scan: Scan, index: number): number { + const image = scan.source.charAt(index) === '!' + if (image && scan.source.charAt(index + 1) !== '[') { + scan.pending += '!' + return index + 1 + } + const width = image ? 2 : 1 + flush(scan, false) + scan.pieces.push({ active: true, image, kind: 'open', start: index + width }) + return index + width +} + +function flush(scan: Scan, strip: boolean): void { + const raw = strip ? scan.pending.replace(trailingSpace, '') : scan.pending + scan.pending = '' + if (raw !== '') scan.pieces.push({ kind: 'nodes', nodes: [{ text: decodeTextEscapes(raw), type: 'text' }] }) +} + +function pushNode(scan: Scan, node: AdfNode): void { + scan.pieces.push({ kind: 'nodes', nodes: [node] }) +} + +function assemble(scan: Scan): Result { + const only = scan.pieces[0] + if (scan.pieces.length === 1 && only?.kind === 'image') return success({ image: only.node }) + if (holdsImage(scan.pieces)) return failure('unmappable-image', imageAlone, scan.path) + return success({ nodes: resolveNodes(scan.pieces) }) +} + +function holdsImage(pieces: readonly Piece[]): boolean { + return pieces.some((piece) => piece.kind === 'image') +} + +function readDelimiterRun(scan: Scan, index: number): number { + const character = scan.source.charAt(index) + const length = runLength(scan.source, index) + const flags = delimiterFlags(character, scan.source.charAt(index - 1), scan.source.charAt(index + length)) + if ((character === '~' && length !== 2) || (!flags.canOpen && !flags.canClose)) scan.pending += scan.source.slice(index, index + length) + else { + flush(scan, false) + scan.pieces.push({ canClose: flags.canClose, canOpen: flags.canOpen, character, kind: 'run', length }) + } + return index + length +} + +function readAutolink(source: string, index: number): { length: number; node: AdfNode } | undefined { + const bracketed = readBracketedAutolink(source, index) + if (bracketed !== undefined) return { length: bracketed, node: linkedText(source.slice(index + 1, index + bracketed - 1), '') } + const email = readEmailAutolink(source, index) + if (email === undefined) return undefined + return { length: email, node: linkedText(source.slice(index + 1, index + email - 1), 'mailto:') } +} + +function linkedText(text: string, scheme: string): AdfNode { + return { marks: [{ attrs: { href: `${scheme}${text}` }, type: 'link' }], text, type: 'text' } +} + +function closeBracket(scan: Scan, index: number): Result { + flush(scan, false) + const open = lastBracket(scan.pieces) + if (open === undefined) return success(literalClose(scan, index)) + const target = open.bracket.active ? resolveTarget(scan, open.bracket, index) : undefined + if (target === undefined) return success(unopened(scan, open, index)) + const inner = scan.pieces.slice(open.index + 1) + if (open.bracket.image) { + const closed = closeImage(scan, open.index, inner, target.definition) + if (!closed.ok) return closed + return success(index + 1 + target.length) + } + const closed = closeLink(scan, open.index, inner, target.definition) + if (!closed.ok) return closed + return success(closed.value ? index + 1 + target.length : unopened(scan, open, index)) +} + +function literalClose(scan: Scan, index: number): number { + scan.pending += ']' + return index + 1 +} + +function unopened(scan: Scan, open: { bracket: Bracket; index: number }, index: number): number { + scan.pieces[open.index] = { kind: 'nodes', nodes: bracketNodes(open.bracket) } + return literalClose(scan, index) +} + +function bracketNodes(bracket: Bracket): AdfNode[] { + return [{ text: bracket.image ? '![' : '[', type: 'text' }] +} + +function lastBracket(pieces: readonly Piece[]): { bracket: Bracket; index: number } | undefined { + for (let index = pieces.length - 1; index >= 0; index -= 1) { + const piece = pieces[index] + if (piece?.kind === 'open') return { bracket: piece, index } + } + return undefined +} + +function resolveTarget(scan: Scan, bracket: Bracket, index: number): { definition: LinkDefinition; length: number } | undefined { + const after = index + 1 + if (scan.source.charAt(after) === '(') { + const inline = readInlineTarget(scan.source, after) + if (inline !== undefined) return { definition: inline.definition, length: inline.length } + } + const label = scan.source.charAt(after) === '[' ? readLabel(scan.source, after) : undefined + const name = label === undefined || label.value === '' ? scan.source.slice(bracket.start, index) : label.value + const definition = scan.definitions.get(normalizeLabel(name)) + if (definition === undefined) return undefined + return { definition, length: label?.length ?? 0 } +} + +// `false` where the link text is empty: the mark has no node to ride, so the brackets stay text. +function closeLink(scan: Scan, at: number, inner: readonly Piece[], definition: LinkDefinition): Result { + if (holdsImage(inner)) return failure('unmappable-image', imageAlone, scan.path) + const nodes = resolveNodes(inner) + if (nodes.length === 0) return success(false) + const attrs = definition.title === undefined ? { href: definition.destination } : { href: definition.destination, title: definition.title } + scan.pieces.length = at + // CommonMark: no link nests inside another, though an image's description holds one. + for (const piece of scan.pieces) if (piece.kind === 'open' && !piece.image) piece.active = false + scan.pieces.push({ kind: 'nodes', nodes: applyMark(nodes, { attrs, type: 'link' }) }) + return success(true) +} + +function closeImage(scan: Scan, at: number, inner: readonly Piece[], definition: LinkDefinition): Result { + if (definition.title !== undefined) return failure('unmappable-image', 'no media node carries a link title', scan.path) + const alt = imageAlt(inner) + const attrs = alt === '' ? { type: 'external', url: definition.destination } : { alt, type: 'external', url: definition.destination } + scan.pieces.length = at + scan.pieces.push({ alt, kind: 'image', node: { attrs: { layout: 'center' }, content: [{ attrs, type: 'media' }], type: 'mediaSingle' } }) + return success(null) +} + +function imageAlt(inner: readonly Piece[]): string { + return resolveNodes(inner) + .map((node) => (node.type === 'hardBreak' ? ' ' : (node.text ?? ''))) + .join('') +} + +function resolveNodes(pieces: readonly Piece[]): AdfNode[] { + const nodes = pieces.map(pieceNodes) + const runs = delimiterRuns(pieces) + const pairings = matchEmphasis(runs) + writeUnpaired(nodes, runs, pairings) + markPairings(nodes, pairings) + return mergeAdjacentText(nodes.flat()) +} + +// Only `imageAlt` reaches the image arm: everywhere else an image amid other content is refused first. +function pieceNodes(piece: Piece): AdfNode[] { + switch (piece.kind) { + case 'image': + return piece.alt === '' ? [] : [{ text: piece.alt, type: 'text' }] + case 'nodes': + return piece.nodes + case 'open': + return bracketNodes(piece) + case 'run': + return [] + } +} + +function delimiterRuns(pieces: readonly Piece[]): Run[] { + const runs: Run[] = [] + for (const [index, piece] of pieces.entries()) { + if (piece.kind === 'run') runs.push({ canClose: piece.canClose, canOpen: piece.canOpen, character: piece.character, index, length: piece.length }) + } + return runs +} + +// A run gives its delimiters up from the head closing and the tail opening; what is left between them is text. +function writeUnpaired(nodes: AdfNode[][], runs: readonly Run[], pairings: readonly Pairing[]): void { + const heads = new Map() + const tails = new Map() + for (const pairing of pairings) { + heads.set(pairing.closer, pairing.closerOffset + pairing.used) + tails.set(pairing.opener, pairing.openerOffset) + } + for (const run of runs) { + const head = heads.get(run) ?? 0 + const tail = tails.get(run) ?? run.length + if (tail > head) nodes[run.index] = [{ text: run.character.repeat(tail - head), type: 'text' }] + } +} + +// Innermost pairing first, so prepending leaves the marks array outermost first (spec/flavour.md, Marks). +function markPairings(nodes: AdfNode[][], pairings: readonly Pairing[]): void { + for (const pairing of pairings) { + const mark: AdfMark = { type: markType(pairing.opener.character, pairing.used) } + for (let index = pairing.opener.index + 1; index < pairing.closer.index; index += 1) nodes[index] = applyMark(nodes[index] ?? [], mark) + } +} + +function markType(character: string, used: number): string { + if (character === '~') return 'strike' + return used === 2 ? 'strong' : 'em' +} + +// A node cannot carry one mark type twice (AGENTS.md §14). +function applyMark(nodes: readonly AdfNode[], mark: AdfMark): AdfNode[] { + return nodes.map((node) => { + const marks = node.marks ?? [] + return marks.some((carried) => carried.type === mark.type) ? node : { ...node, marks: [mark, ...marks] } + }) } function readCodeSpan(source: string, index: number): { end: number; text: string } | undefined { diff --git a/src/markdown/parse/link-reference-definitions.ts b/src/markdown/parse/link-reference-definitions.ts index b235adf..8edbefd 100644 --- a/src/markdown/parse/link-reference-definitions.ts +++ b/src/markdown/parse/link-reference-definitions.ts @@ -1,14 +1,9 @@ -import { holdsControlCharacter, isAsciiPunctuation } from '../commonmark-grammar.ts' - -export type LinkDefinition = { destination: string; title?: string } +import type { LinkDefinition, LinkPart } from '../link-syntax.ts' +import { normalizeLabel, readDestination, readLabel, readTitle, skipLinkWhitespace } from '../link-syntax.ts' type ReadDefinition = { definition: LinkDefinition; label: string; length: number } -type ReadValue = { length: number; value: string } -const bracketedDestination = /^<((?:[^\n<>\\]|\\[^\n])*)>/ -const label = /^\[((?:[^[\]\\]|\\[\s\S]){1,999})\]:/ const restOfLine = /^[ \t]*(?:\n|$)/ -const titleClosers: Readonly> = { '"': '"', "'": "'", '(': ')' } export function readLinkDefinitions(definitions: Map, text: string): string { let rest = text @@ -22,11 +17,11 @@ export function readLinkDefinitions(definitions: Map, te } function readDefinition(text: string): ReadDefinition | undefined { - const matched = label.exec(text) - if (matched === null) return undefined - const name = normalizeLabel(matched[1] ?? '') + const label = readLabel(text, 0) + if (label === undefined || text.charAt(label.length) !== ':') return undefined + const name = normalizeLabel(label.value) if (name === '') return undefined - const afterLabel = skipSpace(text, matched[0].length) + const afterLabel = skipLinkWhitespace(text, label.length + 1) const destination = readDestination(text, afterLabel) if (destination === undefined) return undefined const afterDestination = afterLabel + destination.length @@ -37,15 +32,8 @@ function readDefinition(text: string): ReadDefinition | undefined { return { definition: { destination: destination.value }, label: name, length: plain } } -function normalizeLabel(raw: string): string { - return raw - .replace(/^[ \t\n]+|[ \t\n]+$/g, '') - .replace(/[ \t\n]+/g, ' ') - .toLowerCase() -} - -function readTitledEnd(text: string, offset: number): ReadValue | undefined { - const afterSpace = skipSpace(text, offset) +function readTitledEnd(text: string, offset: number): LinkPart | undefined { + const afterSpace = skipLinkWhitespace(text, offset) if (afterSpace === offset) return undefined const title = readTitle(text, afterSpace) if (title === undefined) return undefined @@ -53,58 +41,6 @@ function readTitledEnd(text: string, offset: number): ReadValue | undefined { return end === undefined ? undefined : { length: end, value: title.value } } -function readDestination(text: string, offset: number): ReadValue | undefined { - const bracketed = bracketedDestination.exec(text.slice(offset)) - if (bracketed !== null) return { length: bracketed[0].length, value: bracketed[0].slice(1, -1) } - if (text.charAt(offset) === '<') return undefined - let depth = 0 - let index = offset - while (index < text.length) { - const character = text.charAt(index) - if (character === ' ' || holdsControlCharacter(character)) break - if (escapesNext(text, index)) { - index += 2 - continue - } - if (character === '(') depth += 1 - if (character === ')') { - depth -= 1 - if (depth < 0) break - } - index += 1 - } - return index <= offset ? undefined : { length: index - offset, value: text.slice(offset, index) } -} - -function readTitle(text: string, offset: number): ReadValue | undefined { - const opener = text.charAt(offset) - const closer = titleClosers[opener] - if (closer === undefined) return undefined - let index = offset + 1 - while (index < text.length) { - const character = text.charAt(index) - if (escapesNext(text, index)) { - index += 2 - continue - } - if (character === closer) return { length: index + 1 - offset, value: text.slice(offset + 1, index) } - if (character === opener) return undefined - index += 1 - } - return undefined -} - -// A backslash escapes ASCII punctuation only, so a line ending always ends the destination it follows. -function escapesNext(text: string, index: number): boolean { - return text.charAt(index) === '\\' && isAsciiPunctuation(text.charAt(index + 1)) -} - -// The label, the destination and the title each take at most one line ending with them. -function skipSpace(text: string, offset: number): number { - const rest = text.slice(offset) - return offset + rest.length - rest.replace(/^[ \t]*\n?[ \t]*/, '').length -} - function endOfLine(text: string, offset: number): number | undefined { const rest = restOfLine.exec(text.slice(offset))?.[0] return rest === undefined ? undefined : offset + rest.length diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index 3861b27..3be28cd 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -1,10 +1,14 @@ import assert from 'node:assert/strict' import test from 'node:test' -import type { AdfDocument, AdfNode } from '../../adf/document.ts' +import type { AdfDocument, AdfMark, AdfNode } from '../../adf/document.ts' import type { Result } from '../../result.ts' import { markdownToAdf } from './markdown-to-adf.ts' +const em: AdfMark = { type: 'em' } +const strike: AdfMark = { type: 'strike' } +const strong: AdfMark = { type: 'strong' } + function code(result: Result): string { return result.ok ? `built ${JSON.stringify(result.value)}` : result.error.code } @@ -49,6 +53,19 @@ function quote(...content: AdfNode[]): AdfNode { return content.length === 0 ? { type: 'blockquote' } : { content, type: 'blockquote' } } +function marked(value: string, ...marks: AdfMark[]): AdfNode { + return { marks, text: value, type: 'text' } +} + +function link(href: string, title?: string): AdfMark { + return { attrs: title === undefined ? { href } : { href, title }, type: 'link' } +} + +function image(url: string, alt?: string): AdfNode { + const media: AdfNode = { attrs: alt === undefined ? { type: 'external', url } : { alt, type: 'external', url }, type: 'media' } + return { attrs: { layout: 'center' }, content: [media], type: 'mediaSingle' } +} + test('builds an empty document from input holding no block', () => { assert.deepEqual(markdownToAdf(''), { ok: true, value: { type: 'doc', version: 1 } }) assert.deepEqual(content(markdownToAdf('\n \n\t\n')), []) @@ -323,3 +340,142 @@ test('leaves the angle bracket that opens no HTML construct to the text it sits assert.deepEqual(content(markdownToAdf('``\n')), [{ content: [codeSpan('')], type: 'paragraph' }]) assert.deepEqual(content(markdownToAdf('\\\n')), [paragraph('')]) }) + +test('reads the emphasis CommonMark matches, the marks nesting outermost first', () => { + assert.deepEqual(content(markdownToAdf('*a*\n')), [{ content: [marked('a', em)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('_a_\n')), [{ content: [marked('a', em)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('**a**\n')), [{ content: [marked('a', strong)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('__a__\n')), [{ content: [marked('a', strong)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('***a***\n')), [{ content: [marked('a', em, strong)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('*a **b** c*\n')), [ + { content: [marked('a ', em), marked('b', em, strong), marked(' c', em)], type: 'paragraph' }, + ]) + assert.deepEqual(content(markdownToAdf('a*b*c\n')), [{ content: [text('a'), marked('b', em), text('c')], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('# *a*\n')), [{ attrs: { level: 1 }, content: [marked('a', em)], type: 'heading' }]) +}) + +test('leaves a delimiter run CommonMark pairs with nothing in the text it sits in', () => { + assert.deepEqual(content(markdownToAdf('a_b_c\n')), [paragraph('a_b_c')]) + assert.deepEqual(content(markdownToAdf('*a\n')), [paragraph('*a')]) + assert.deepEqual(content(markdownToAdf('a * b\n')), [paragraph('a * b')]) + assert.deepEqual(content(markdownToAdf('**a*\n')), [{ content: [text('*'), marked('a', em)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('*a**\n')), [{ content: [marked('a', em), text('*')], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('\\*a\\*\n')), [paragraph('*a*')]) + assert.deepEqual(content(markdownToAdf('`*a*`\n')), [{ content: [codeSpan('*a*')], type: 'paragraph' }]) +}) + +test('reads two tildes as strike, a single tilde and a longer run literal', () => { + assert.deepEqual(content(markdownToAdf('~~a~~\n')), [{ content: [marked('a', strike)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('a ~b~ c\n')), [paragraph('a ~b~ c')]) + assert.deepEqual(content(markdownToAdf('a ~~~b~~~ c\n')), [paragraph('a ~~~b~~~ c')]) + assert.deepEqual(content(markdownToAdf('~~a **b**~~\n')), [{ content: [marked('a ', strike), marked('b', strike, strong)], type: 'paragraph' }]) +}) + +test('reads an inline link, its destination and title', () => { + assert.deepEqual(content(markdownToAdf('[a](/url)\n')), [{ content: [marked('a', link('/url'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a](/url "t")\n')), [{ content: [marked('a', link('/url', 't'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a](\n/url\n"t" )\n')), [{ content: [marked('a', link('/url', 't'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a]()\n')), [{ content: [marked('a', link('u v'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a]()\n')), [{ content: [marked('a', link(''))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a](/x(y))\n')), [{ content: [marked('a', link('/x(y)'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[**a**](/u)\n')), [{ content: [marked('a', link('/u'), strong)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a `b`](/u)\n')), [ + { content: [marked('a ', link('/u')), { marks: [link('/u'), { type: 'code' }], text: 'b', type: 'text' }], type: 'paragraph' }, + ]) +}) + +test('decodes the escapes and the references a destination and a title hold', () => { + assert.deepEqual(content(markdownToAdf('[a](/x\\)y)\n')), [{ content: [marked('a', link('/x)y'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a](/u "He said \\"hi\\"")\n')), [{ content: [marked('a', link('/u', 'He said "hi"'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a](/x&y)\n')), [{ content: [marked('a', link('/x&y'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a][r]\n\n[r]: /x\\)y "He said \\"hi\\""\n')), [ + { content: [marked('a', link('/x)y', 'He said "hi"'))], type: 'paragraph' }, + ]) +}) + +test('leaves the bracket pair no link parses as the text it holds', () => { + assert.deepEqual(content(markdownToAdf('[a\n')), [paragraph('[a')]) + assert.deepEqual(content(markdownToAdf('[a] (/u)\n')), [paragraph('[a] (/u)')]) + assert.deepEqual(content(markdownToAdf('a ] b\n')), [paragraph('a ] b')]) + assert.deepEqual(content(markdownToAdf('[a](/u\n')), [paragraph('[a](/u')]) + assert.deepEqual(content(markdownToAdf('[a](/u x)\n')), [paragraph('[a](/u x)')]) + assert.deepEqual(content(markdownToAdf('[a]( { + assert.deepEqual(content(markdownToAdf('[a][r]\n\n[r]: /url\n')), [{ content: [marked('a', link('/url'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a][]\n\n[a]: /url\n')), [{ content: [marked('a', link('/url'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a]\n\n[a]: /url\n')), [{ content: [marked('a', link('/url'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[Foo\nBar][]\n\n[foo bar]: /url\n')), [{ content: [marked('Foo Bar', link('/url'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('[a][z]\n\n[a]: /url\n')), [paragraph('[a][z]')]) + assert.deepEqual(content(markdownToAdf('[a]\n')), [paragraph('[a]')]) + assert.deepEqual(content(markdownToAdf('[][]\n')), [paragraph('[][]')]) +}) + +test('reads an autolink, the email form as the mailto link it means', () => { + assert.deepEqual(content(markdownToAdf('\n')), [ + { content: [marked('https://example.com/', link('https://example.com/'))], type: 'paragraph' }, + ]) + assert.deepEqual(content(markdownToAdf('a c\n')), [ + { content: [text('a '), marked('a@b.example.com', link('mailto:a@b.example.com')), text(' c')], type: 'paragraph' }, + ]) + assert.deepEqual(content(markdownToAdf('a d\n')), [ + { content: [text('a '), marked('a@b-c.example.com', link('mailto:a@b-c.example.com')), text(' d')], type: 'paragraph' }, + ]) + assert.deepEqual(content(markdownToAdf('a d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('a d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('\n')), [ + { content: [marked('https://example.com/?a=\\*', link('https://example.com/?a=\\*'))], type: 'paragraph' }, + ]) + assert.equal(code(markdownToAdf(' \n')), 'unmappable-html') +}) + +test('reads a lone image as the media the flavour spells for it', () => { + assert.deepEqual(content(markdownToAdf('![The moon](https://example.com/moon.png)\n')), [ + image('https://example.com/moon.png', 'The moon'), + ]) + assert.deepEqual(content(markdownToAdf('![](/u)\n')), [image('/u')]) + assert.deepEqual(content(markdownToAdf('![*a*](/u)\n')), [image('/u', 'a')]) + assert.deepEqual(content(markdownToAdf('- ![a](/u)\n')), [bulletList(item(image('/u', 'a')))]) +}) + +test('flattens the description of a lone image to the plain text alt holds', () => { + assert.deepEqual(content(markdownToAdf('![a [b](/u) c](/v)\n')), [image('/v', 'a b c')]) + assert.deepEqual(content(markdownToAdf('![a ![b](/c) d](/e)\n')), [image('/e', 'a b d')]) + assert.deepEqual(content(markdownToAdf('![a\nb](/u)\n')), [image('/u', 'a b')]) + assert.deepEqual(content(markdownToAdf('![a \nb](/u)\n')), [image('/u', 'a b')]) + assert.deepEqual(content(markdownToAdf('![a `b`](/u)\n')), [image('/u', 'a b')]) +}) + +test('leaves the brackets of an empty link text the text they are', () => { + assert.deepEqual(content(markdownToAdf('[](/u)\n')), [paragraph('[](/u)')]) + assert.deepEqual(content(markdownToAdf('a [](/u) b\n')), [paragraph('a [](/u) b')]) + // The pair gives the label back the way an unresolved one does, so the shortcut behind it still reads. + assert.deepEqual(content(markdownToAdf('[][r]\n\n[r]: /u\n')), [{ content: [text('[]'), marked('r', link('/u'))], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('![](/u)\n')), [image('/u')]) +}) + +test('refuses the image no ADF node carries where it sits', () => { + assert.equal(content(markdownToAdf('![a](/u "t")\n')), 'unmappable-image: no media node carries a link title') + assert.equal(content(markdownToAdf('See ![a](/u).\n')), 'unmappable-image: an image fits only as a paragraph of its own') + assert.equal(code(markdownToAdf('# ![a](/u)\n')), 'unmappable-image') + assert.equal(code(markdownToAdf('*![a](/u)*\n')), 'unmappable-image') + assert.equal(code(markdownToAdf('[![a](/u)](/v)\n')), 'unmappable-image') + assert.equal(code(markdownToAdf('![a](/u)![b](/v)\n')), 'unmappable-image') + assert.equal(code(markdownToAdf('![a ![b](/c) d\n')), 'unmappable-image') + assert.deepEqual(path(markdownToAdf('Part.\n\nSee ![a](/u).\n')), ['content', 1]) + assert.deepEqual(content(markdownToAdf('![a]\n')), [paragraph('![a]')]) + assert.deepEqual(content(markdownToAdf('a ! b\n')), [paragraph('a ! b')]) +}) + +test('carries the mark a spelling nested inside its own kind names once', () => { + assert.deepEqual(content(markdownToAdf('*(*a*)*\n')), [{ content: [marked('(a)', em)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf(`${'*'.repeat(600)}a${'*'.repeat(600)}\n`)), [{ content: [marked('a', strong)], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('*a **b** c*\n')), [ + { content: [marked('a ', em), marked('b', em, strong), marked(' c', em)], type: 'paragraph' }, + ]) +}) diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index fc2c1ed..8a5c985 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -1,60 +1,62 @@ import type { AdfDocument, AdfNode } from '../../adf/document.ts' import type { Block, ClaimedConstruct } from './blocks.ts' +import type { LinkDefinitions } from './inline-content.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' import { largestNesting } from '../../nesting.ts' import { parseBlocks } from './blocks.ts' import { parseInlineContent } from './inline-content.ts' export function markdownToAdf(markdown: string): Result { - const content = blockNodes(parseBlocks(markdown).blocks, [], 0) + const parsed = parseBlocks(markdown) + const content = blockNodes(parsed.blocks, parsed.definitions, [], 0) if (!content.ok) return content return success(content.value.length === 0 ? { type: 'doc', version: 1 } : { content: content.value, type: 'doc', version: 1 }) } -function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: number): Result { +function blockNodes(blocks: readonly Block[], definitions: LinkDefinitions, path: ConvertErrorPath, depth: number): Result { if (depth > largestNesting) return failure('unsupported-nesting-depth', `the input nests deeper than the ${largestNesting} levels the parser carries`, path) const content: AdfNode[] = [] for (const [index, block] of blocks.entries()) { - const node = blockNode(block, [...path, 'content', index], depth) + const node = blockNode(block, definitions, [...path, 'content', index], depth) if (!node.ok) return node content.push(node.value) } return success(content) } -function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result { +function blockNode(block: Block, definitions: LinkDefinitions, path: ConvertErrorPath, depth: number): Result { switch (block.kind) { case 'blockquote': - return containerNode({ type: 'blockquote' }, block.blocks, path, depth) + return containerNode({ type: 'blockquote' }, block.blocks, definitions, path, depth) case 'bulletList': - return listNode({ type: 'bulletList' }, block.items, path, depth) + return listNode({ type: 'bulletList' }, block.items, definitions, path, depth) case 'claim': return claimFailure(block.construct, path) case 'code': return success(codeBlockNode(block.language, block.text)) case 'heading': - return contentNode({ attrs: { level: block.level }, type: 'heading' }, block.text, path) + return contentNode({ attrs: { level: block.level }, type: 'heading' }, block.text, definitions, path) case 'html': return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) case 'orderedList': - return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth) + return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, definitions, path, depth) case 'paragraph': - return contentNode({ type: 'paragraph' }, block.text, path) + return contentNode({ type: 'paragraph' }, block.text, definitions, path) case 'rule': return success({ type: 'rule' }) } } -function containerNode(node: AdfNode, blocks: readonly Block[], path: ConvertErrorPath, depth: number): Result { - const content = blockNodes(blocks, path, depth + 1) +function containerNode(node: AdfNode, blocks: readonly Block[], definitions: LinkDefinitions, path: ConvertErrorPath, depth: number): Result { + const content = blockNodes(blocks, definitions, path, depth + 1) if (!content.ok) return content return success(content.value.length === 0 ? node : { ...node, content: content.value }) } -function listNode(node: AdfNode, items: readonly Block[][], path: ConvertErrorPath, depth: number): Result { +function listNode(node: AdfNode, items: readonly Block[][], definitions: LinkDefinitions, path: ConvertErrorPath, depth: number): Result { const content: AdfNode[] = [] for (const [index, blocks] of items.entries()) { - const item = containerNode({ type: 'listItem' }, blocks, [...path, 'content', index], depth) + const item = containerNode({ type: 'listItem' }, blocks, definitions, [...path, 'content', index], depth) if (!item.ok) return item content.push(item.value) } @@ -75,8 +77,13 @@ function codeBlockNode(language: string, text: string): AdfNode { return text === '' ? node : { ...node, content: [{ text, type: 'text' }] } } -function contentNode(node: AdfNode, text: string, path: ConvertErrorPath): Result { - const content = parseInlineContent(text, path) +// spec/flavour.md, The CommonMark image: only a paragraph gives an image the block it needs. +function contentNode(node: AdfNode, text: string, definitions: LinkDefinitions, path: ConvertErrorPath): Result { + const content = parseInlineContent(text, definitions, path) if (!content.ok) return content - return success(content.value.length === 0 ? node : { ...node, content: content.value }) + const image = content.value.image + if (image !== undefined) { + return node.type === 'paragraph' ? success(image) : failure('unmappable-image', `no ADF node carries an image inside a ${node.type}`, path) + } + return success(content.value.nodes.length === 0 ? node : { ...node, content: content.value.nodes }) } diff --git a/src/result.ts b/src/result.ts index 94e8c9d..2bdcdb4 100644 --- a/src/result.ts +++ b/src/result.ts @@ -3,6 +3,7 @@ export type ConvertErrorCode = | 'malformed-pipe-table' | 'not-an-adf-document' | 'unmappable-html' + | 'unmappable-image' | 'unspellable-adjacent-lists' | 'unspellable-character' | 'unspellable-line-start' diff --git a/todo.md b/todo.md index 35924d7..dcb2ff0 100644 --- a/todo.md +++ b/todo.md @@ -240,7 +240,7 @@ detail is settled at its own milestone. autolink parting off as 3e's own predicate: refusing on the superset would refuse `1 ` is, autolinks decoding neither escapes nor references. + The image gap mints `unmappable-image`, mirroring `unmappable-html` — a construct in input + no ADF node carries. And the CommonMark image shape lands here rather than at 3h: once + `[…](…)` reads, a lone `![alt](url)` would otherwise misparse as text plus a link, so 3h + keeps the rest of the media family and loses only that line. + **Settled** (the maintainer, 2026-08-31, on the review): an empty link text — `[](/u)` — + leaves the brackets the text they are rather than minting a refusal or dropping the + destination, giving the label back the way an unresolved pair does, so the shortcut behind + `[][r]` still reads. A description holding an image flattens to that image's own alt, which + is what alt text means and what keeps the documented gap to mid-text and titled images; a + break of either kind inside one reads as a space. And a destination or title whose entity + reference decodes to a control character — `[a](/x y)` — joins 3k's exception list + beside the two above: the reader takes cmark's reading, the emitter has no spelling for it. - [ ] **3f — The directive grammar.** The three forms — inline `:name[content]{attrs}`, container `:::name arg {attrs}`, leaf `::name arg {attrs}` — the attribute grammar with its quoting and escapes, the fence-length and nesting rules, and the malformed list @@ -269,9 +285,9 @@ detail is settled at its own milestone. guard is built here if the answer there was yes. - [ ] **3h — The block nodes.** `block-nodes/` reads back: the plain-versus-directive choice 2f settles, read from the other side; the `codeBlock` directive's fenced body and the - `language` attribute a bare fence leaves it; the media family's composition and the - CommonMark image shape; both table forms, the pipe table's cell split and its named - errors; and the one paragraph an inline body takes. + `language` attribute a bare fence leaves it; the media family's composition; both table + forms, the pipe table's cell split and its named errors; and the one paragraph an inline + body takes. **Settled** (the maintainer, 2026-08-27): 1d's last pick, the one `container-block-separation` holds — a CommonMark block and a directive block sit adjacent in a container body with no blank line between them. That reduces the three cases to one @@ -300,7 +316,9 @@ detail is settled at its own milestone. canonical fixpoint: a named error, or markdown that parses and emits to itself byte for byte. That HTML's text, tags stripped and entities decoded, against the parsed document's concatenated `text`. And a count of the dozen elements the CommonMark subset covers - against the marks and nodes they map to. The fixpoint alone is self-consistency a parser + against the marks and nodes they map to — counting distinct mark types per text node, since + 3e collapses a spelling nested inside its own kind and `*(*a*)*` is two `` against one + `em`. The fixpoint alone is self-consistency a parser returning the empty document passes, and the text alone one dropping every emphasis; the counts close both. The exception list stays the maintainer's. One outcome is no exception and must not be filed as one: valid CommonMark parsing to a document