From 2245ab01a9033f31e2bbe0235f00d7628a79536c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Mon, 31 Aug 2026 21:50:06 +0200 Subject: [PATCH] Part the inline scan into named readers, and share the mark identity both directions read --- AGENTS.md | 7 +- src/adf/editor-normal.ts | 32 ++++ src/markdown/backtick-runs.ts | 6 +- src/markdown/commonmark-grammar.ts | 5 +- src/markdown/emit/inline-line.ts | 6 +- src/markdown/emphasis-matching.ts | 2 +- src/markdown/parse/inline-content.ts | 201 ++++++++++++--------- src/markdown/parse/markdown-to-adf.test.ts | 5 + todo.md | 4 +- 9 files changed, 166 insertions(+), 102 deletions(-) create mode 100644 src/adf/editor-normal.ts diff --git a/AGENTS.md b/AGENTS.md index 853a8d7..1ea936b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,7 +148,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - `src/adf/` holds ADF's own knowledge and imports no format. Each format directory (`markdown/`, `html/`) parts into `emit/` (ADF→format) and `parse/` (format→ADF), its root holding what both directions read. A construct's reader lives in that root beside the regex the emitter escapes - against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`. A rule both + against, so the two cannot drift; a reader with no emit counterpart goes in `parse/`, unless it is + part of a construct the root already holds — a grammar stays in one file rather than splitting + across the seam. A rule both directions must answer alike — whether a list marker interrupts a paragraph — is one function there too, never a copy per direction, however conservative the copy would be. - The attribute vocabulary is ADF's: `adf/` walks it and narrows each value to its kind, and a @@ -180,7 +182,8 @@ One-line commit messages and PR titles; short PR summaries. No AI-attribution ma No wiki markup (§1), no network or filesystem I/O, no name→id resolution (§3), no ADF schema validation or exported validator — a refusal that keeps the round-trip is not schema validation, -so the one a spelled node carrying the same mark type twice earns stays, no shipped CSS (§4), no +so the one a spelled node carrying the same mark type twice earns stays, and input nesting a +spelling inside its own kind (`*(*a*)*`) names that mark once, no shipped CSS (§4), no streaming APIs, no performance budget past §11's scanning rule — nothing here is tuned, and no figure is promised. A CLI is a later goal (`todo.md`), not a non-goal. diff --git a/src/adf/editor-normal.ts b/src/adf/editor-normal.ts new file mode 100644 index 0000000..7f65586 --- /dev/null +++ b/src/adf/editor-normal.ts @@ -0,0 +1,32 @@ +import type { AdfMark, AdfNode } from './document.ts' +import { serializeCanonicalJson } from '../canonical-json.ts' + +export function sameMark(candidate: AdfMark, mark: AdfMark): boolean { + return markKey(candidate) === markKey(mark) +} + +// AGENTS.md §2: adjacent text nodes carrying identical marks are one node. +export function mergeAdjacentText(nodes: readonly AdfNode[]): AdfNode[] { + const merged: AdfNode[] = [] + for (const node of nodes) { + const previous = merged[merged.length - 1] + if (previous !== undefined && previous.type === 'text' && node.type === 'text' && sameMarks(previous, node)) { + merged[merged.length - 1] = { ...previous, text: `${previous.text ?? ''}${node.text ?? ''}` } + continue + } + merged.push(node) + } + return merged +} + +function sameMarks(previous: AdfNode, node: AdfNode): boolean { + return marksKey(previous.marks ?? []) === marksKey(node.marks ?? []) +} + +function marksKey(marks: readonly AdfMark[]): string { + return marks.map(markKey).join('\n') +} + +function markKey(mark: AdfMark): string { + return `${mark.type} ${serializeCanonicalJson(mark.attrs ?? {}, 'compact')}` +} diff --git a/src/markdown/backtick-runs.ts b/src/markdown/backtick-runs.ts index 940573a..469f972 100644 --- a/src/markdown/backtick-runs.ts +++ b/src/markdown/backtick-runs.ts @@ -1,7 +1,7 @@ +import { runLength } from './emphasis-matching.ts' + export function backtickRun(text: string, index: number): number { - let length = 0 - while (text.charAt(index + length) === '`') length += 1 - return length + return text.charAt(index) === '`' ? runLength(text, index) : 0 } // Where the run of exactly `opener` backticks closing a code span begins, `undefined` where none does. diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index cb1452b..570da7c 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -56,7 +56,10 @@ const pipeClaim = /^\|/ const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] -const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y +// CommonMark 0.31.2, Autolinks: the email production, whose label may not open or close with a hyphen. +const emailNameSource = "[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+" +const emailLabelSource = '[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?' +const emailAutolink = new RegExp(`<${emailNameSource}@${emailLabelSource}(?:\\.${emailLabelSource})*>`, 'y') const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ diff --git a/src/markdown/emit/inline-line.ts b/src/markdown/emit/inline-line.ts index c112bb8..71ab82a 100644 --- a/src/markdown/emit/inline-line.ts +++ b/src/markdown/emit/inline-line.ts @@ -9,7 +9,7 @@ import { inlineDirective } from '../../adf/inline-directives.ts' import { largestNesting } from '../../nesting.ts' import { longestBacktickRun } from '../backtick-runs.ts' import { markSpelling, spellMarkAttributes } from '../mark-spellings.ts' -import { serializeCanonicalJson } from '../../canonical-json.ts' +import { sameMark } from '../../adf/editor-normal.ts' import { spellAttributes, spellStringAttribute } from '../directive-attributes.ts' import { spellDestination, spellTitle } from '../link-syntax.ts' import { spellInlineNodeAttributes } from './inline-directive-spelling.ts' @@ -297,7 +297,3 @@ function emitLink(nodes: readonly AdfNode[], mark: AdfMark, depth: number, range return success({ segments: [syntax('['), ...inner.value.segments, syntax(`](${destination.value}${spelledTitle.value})`)] }) } -function sameMark(candidate: AdfMark, mark: AdfMark): boolean { - if (candidate.type !== mark.type) return false - return serializeCanonicalJson(candidate.attrs ?? {}, 'compact') === serializeCanonicalJson(mark.attrs ?? {}, 'compact') -} diff --git a/src/markdown/emphasis-matching.ts b/src/markdown/emphasis-matching.ts index d69104b..b8a8942 100644 --- a/src/markdown/emphasis-matching.ts +++ b/src/markdown/emphasis-matching.ts @@ -2,7 +2,7 @@ import { isUnicodeWhitespace } from './commonmark-grammar.ts' type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } -type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } +export type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } type Candidate = { head: number diff --git a/src/markdown/parse/inline-content.ts b/src/markdown/parse/inline-content.ts index ebd2438..ad415fa 100644 --- a/src/markdown/parse/inline-content.ts +++ b/src/markdown/parse/inline-content.ts @@ -1,11 +1,12 @@ import type { AdfMark, AdfNode } from '../../adf/document.ts' +import type { EmphasisPairing } from '../emphasis-matching.ts' import type { LinkDefinition } from '../link-syntax.ts' import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct, readBracketedAutolink, readEmailAutolink } from '../commonmark-grammar.ts' import { backtickRun, closingBacktickRun } from '../backtick-runs.ts' import { delimiterFlags, matchEmphasis, runLength } from '../emphasis-matching.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' +import { mergeAdjacentText } from '../../adf/editor-normal.ts' import { normalizeLabel, readInlineTarget, readLabel } from '../link-syntax.ts' -import { serializeCanonicalJson } from '../../canonical-json.ts' export type InlineContent = { image: AdfNode; nodes?: undefined } | { image?: undefined; nodes: AdfNode[] } @@ -13,13 +14,14 @@ export type LinkDefinitions = ReadonlyMap type Bracket = { active: boolean; image: boolean; kind: 'open'; start: number } +type Pairing = EmphasisPairing + type Piece = Bracket | { kind: 'nodes'; nodes: AdfNode[] } | { canClose: boolean; canOpen: boolean; character: string; kind: 'run'; length: number } type Run = { canClose: boolean; canOpen: boolean; character: string; index: number; length: number } type Scan = { definitions: LinkDefinitions; image: AdfNode | undefined; path: ConvertErrorPath; pending: string; pieces: Piece[]; source: string } -const emphasisCharacters = '*_~' const hardBreakSpaces = / {2,}$/ const imageAlone = 'an image fits only as a paragraph of its own' const trailingSpace = /[ \t]+$/ @@ -28,76 +30,103 @@ export function parseInlineContent(source: string, definitions: LinkDefinitions, const scan: Scan = { definitions, image: undefined, path, pending: '', pieces: [], source } let index = 0 while (index < source.length) { - const character = source.charAt(index) - if (character === '\\' && source.charAt(index + 1) === '\n') { - // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash. - flush(scan, false) - pushNode(scan, { type: 'hardBreak' }) - index += 2 - continue - } - if (backslashEscape(source, index) !== undefined) { - scan.pending += source.slice(index, index + 2) - index += 2 - continue - } - if (character === '\n') { - const hard = hardBreakSpaces.test(scan.pending) - flush(scan, true) - if (hard) pushNode(scan, { type: 'hardBreak' }) - else scan.pending = ' ' - index += 1 - continue - } - if (character === '`') { - const span = readCodeSpan(source, index) - if (span === undefined) { - const run = backtickRun(source, index) - scan.pending += source.slice(index, index + run) - index += run - continue + switch (source.charAt(index)) { + case '\\': + index = readBackslash(scan, index) + break + case '\n': + index = readLineEnding(scan, index) + break + case '`': + index = readBackticks(scan, index) + break + case '<': { + const angle = readAngle(scan, index) + if (!angle.ok) return angle + index = angle.value + break } - flush(scan, false) - pushNode(scan, { marks: [{ type: 'code' }], text: span.text, type: 'text' }) - index = span.end - continue - } - if (character === '<') { - const autolink = readAutolink(source, index) - if (autolink !== undefined) { - flush(scan, false) - pushNode(scan, autolink.node) - index += autolink.length - continue + case '!': + case '[': + index = openBracket(scan, index) + break + case ']': { + const closed = closeBracket(scan, index) + if (!closed.ok) return closed + index = closed.value + break } - const construct = inlineHtmlConstruct(source, index) - if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, path) + case '*': + case '_': + case '~': + index = readDelimiterRun(scan, index) + break + default: + scan.pending += source.charAt(index) + index += 1 } - if (character === '[' || (character === '!' && source.charAt(index + 1) === '[')) { - const image = character === '!' - const width = image ? 2 : 1 - flush(scan, false) - scan.pieces.push({ active: true, image, kind: 'open', start: index + width }) - index += width - continue - } - if (character === ']') { - const closed = closeBracket(scan, index) - if (!closed.ok) return closed - index = closed.value - continue - } - if (emphasisCharacters.includes(character)) { - index = readDelimiterRun(scan, index) - continue - } - scan.pending += character - index += 1 } flush(scan, true) return assemble(scan) } +function readBackslash(scan: Scan, index: number): number { + if (scan.source.charAt(index + 1) === '\n') { + // CommonMark strips the spaces the two-space break is spelled with, and keeps those before a backslash. + flush(scan, false) + pushNode(scan, { type: 'hardBreak' }) + return index + 2 + } + const width = backslashEscape(scan.source, index) === undefined ? 1 : 2 + scan.pending += scan.source.slice(index, index + width) + return index + width +} + +function readLineEnding(scan: Scan, index: number): number { + const hard = hardBreakSpaces.test(scan.pending) + flush(scan, true) + if (hard) pushNode(scan, { type: 'hardBreak' }) + else scan.pending = ' ' + return index + 1 +} + +function readBackticks(scan: Scan, index: number): number { + const span = readCodeSpan(scan.source, index) + if (span === undefined) { + const run = backtickRun(scan.source, index) + scan.pending += scan.source.slice(index, index + run) + return index + run + } + flush(scan, false) + pushNode(scan, { marks: [{ type: 'code' }], text: span.text, type: 'text' }) + return span.end +} + +function readAngle(scan: Scan, index: number): Result { + const autolink = readAutolink(scan.source, index) + if (autolink !== undefined) { + flush(scan, false) + pushNode(scan, autolink.node) + return success(index + autolink.length) + } + const construct = inlineHtmlConstruct(scan.source, index) + if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, scan.path) + scan.pending += '<' + return success(index + 1) +} + +function openBracket(scan: Scan, index: number): number { + const image = scan.source.charAt(index) === '!' + if (image && scan.source.charAt(index + 1) !== '[') { + scan.pending += '!' + return index + 1 + } + const width = image ? 2 : 1 + flush(scan, false) + scan.pieces.push({ active: true, image, kind: 'open', start: index + width }) + return index + width +} + function flush(scan: Scan, strip: boolean): void { const raw = strip ? scan.pending.replace(trailingSpace, '') : scan.pending scan.pending = '' @@ -209,11 +238,23 @@ function buildImage(scan: Scan, inner: readonly Piece[], definition: LinkDefinit function resolveNodes(pieces: readonly Piece[]): AdfNode[] { const nodes = pieces.map((piece) => (piece.kind === 'nodes' ? piece.nodes : piece.kind === 'open' ? bracketNodes(piece) : [])) + const runs = delimiterRuns(pieces) + const pairings = matchEmphasis(runs) + writeUnpaired(nodes, runs, pairings) + markPairings(nodes, pairings) + return mergeAdjacentText(nodes.flat()) +} + +function delimiterRuns(pieces: readonly Piece[]): Run[] { const runs: Run[] = [] for (const [index, piece] of pieces.entries()) { if (piece.kind === 'run') runs.push({ canClose: piece.canClose, canOpen: piece.canOpen, character: piece.character, index, length: piece.length }) } - const pairings = matchEmphasis(runs) + return runs +} + +// A run gives its delimiters up from the head closing and the tail opening; what is left between them is text. +function writeUnpaired(nodes: AdfNode[][], runs: readonly Run[], pairings: readonly Pairing[]): void { const heads = new Map() const tails = new Map() for (const pairing of pairings) { @@ -225,13 +266,14 @@ function resolveNodes(pieces: readonly Piece[]): AdfNode[] { const tail = tails.get(run) ?? run.length if (tail > head) nodes[run.index] = [{ text: run.character.repeat(tail - head), type: 'text' }] } +} + +// Innermost pairing first, so prepending leaves the marks array outermost first (spec/flavour.md, Marks). +function markPairings(nodes: AdfNode[][], pairings: readonly Pairing[]): void { for (const pairing of pairings) { const mark: AdfMark = { type: markType(pairing.opener.character, pairing.used) } - for (let index = pairing.opener.index + 1; index < pairing.closer.index; index += 1) { - nodes[index] = applyMark(nodes[index] ?? [], mark) - } + for (let index = pairing.opener.index + 1; index < pairing.closer.index; index += 1) nodes[index] = applyMark(nodes[index] ?? [], mark) } - return mergeText(nodes.flat()) } function markType(character: string, used: number): string { @@ -239,8 +281,7 @@ function markType(character: string, used: number): string { return used === 2 ? 'strong' : 'em' } -// CommonMark nests a spelling inside its own kind (`*(*a*)*`); the mark it names is idempotent, and -// a node carrying it twice is the shape AGENTS.md §14 has the emitter refuse. +// A node cannot carry one mark type twice (AGENTS.md §14). function applyMark(nodes: readonly AdfNode[], mark: AdfMark): AdfNode[] { return nodes.map((node) => { const marks = node.marks ?? [] @@ -248,24 +289,6 @@ function applyMark(nodes: readonly AdfNode[], mark: AdfMark): AdfNode[] { }) } -// Editor-normal (AGENTS.md §2): adjacent text nodes carrying identical marks are one node. -function mergeText(nodes: readonly AdfNode[]): AdfNode[] { - const merged: AdfNode[] = [] - for (const node of nodes) { - const previous = merged[merged.length - 1] - if (previous !== undefined && previous.type === 'text' && node.type === 'text' && markKey(previous) === markKey(node)) { - merged[merged.length - 1] = { ...previous, text: `${previous.text ?? ''}${node.text ?? ''}` } - continue - } - merged.push(node) - } - return merged -} - -function markKey(node: AdfNode): string { - return (node.marks ?? []).map((mark) => `${mark.type}${serializeCanonicalJson(mark.attrs ?? {}, 'compact')}`).join(' ') -} - function readCodeSpan(source: string, index: number): { end: number; text: string } | undefined { const opener = backtickRun(source, index) const closer = closingBacktickRun(source, index + opener, opener) diff --git a/src/markdown/parse/markdown-to-adf.test.ts b/src/markdown/parse/markdown-to-adf.test.ts index 2a76461..dc4ba46 100644 --- a/src/markdown/parse/markdown-to-adf.test.ts +++ b/src/markdown/parse/markdown-to-adf.test.ts @@ -423,6 +423,11 @@ test('reads an autolink, the email form as the mailto link it means', () => { assert.deepEqual(content(markdownToAdf('a c\n')), [ { content: [text('a '), marked('a@b.example.com', link('mailto:a@b.example.com')), text(' c')], type: 'paragraph' }, ]) + assert.deepEqual(content(markdownToAdf('a d\n')), [ + { content: [text('a '), marked('a@b-c.example.com', link('mailto:a@b-c.example.com')), text(' d')], type: 'paragraph' }, + ]) + assert.deepEqual(content(markdownToAdf('a d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('a d\n')), [paragraph('a d')]) assert.deepEqual(content(markdownToAdf('\n')), [ { content: [marked('https://example.com/?a=\\*', link('https://example.com/?a=\\*'))], type: 'paragraph' }, ]) diff --git a/todo.md b/todo.md index 5216a37..757be72 100644 --- a/todo.md +++ b/todo.md @@ -308,7 +308,9 @@ detail is settled at its own milestone. canonical fixpoint: a named error, or markdown that parses and emits to itself byte for byte. That HTML's text, tags stripped and entities decoded, against the parsed document's concatenated `text`. And a count of the dozen elements the CommonMark subset covers - against the marks and nodes they map to. The fixpoint alone is self-consistency a parser + against the marks and nodes they map to — counting distinct mark types per text node, since + 3e collapses a spelling nested inside its own kind and `*(*a*)*` is two `` against one + `em`. The fixpoint alone is self-consistency a parser returning the empty document passes, and the text alone one dropping every emphasis; the counts close both. The exception list stays the maintainer's. One outcome is no exception and must not be filed as one: valid CommonMark parsing to a document