diff --git a/AGENTS.md b/AGENTS.md index 63e8d3c..aa1e4bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,6 +122,8 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - Emphasis is spelled against CommonMark's matching, never flanking alone: a delimiter run in text escapes wherever CommonMark could open or close with it, leaving the emitter's own delimiters the only ones in play, and a pair that matching hands to another delimiter rides the carry instead. + `matchEmphasis` is a line-for-line transcription of the reference `process_emphasis` and stays one + function: split into named steps it drifts from the algorithm whose fidelity is the whole point. - A readable spelling tried ahead of a general one — the image, the pipe table, a pipe cell — returns `string | undefined`, never a `Result`: any failure is the fallback signal, and the general form owns the refusal. Refusing there refuses a document the general form spells. diff --git a/src/corpus.test.ts b/src/corpus.test.ts index 60ff98d..c729a68 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -66,22 +66,33 @@ for (const directory of emittingDirectories) { } } +function roundTripFixtures(): { name: string; path: string }[] { + return emittingDirectories.flatMap((directory) => + fixtureNames(directory, '.json').map((name) => ({ name: `${directory}/${name}`, path: join(roundTripRoot, directory, `${name}.json`) })), + ) +} + // One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it. -test('no two round-trip documents share one spelling', () => { - const documents = new Map() +test('no two round-trip documents share one markdown spelling', () => { const spellings = new Map() - for (const directory of emittingDirectories) { - for (const name of fixtureNames(directory, '.md')) { - const fixture = `${directory}/${name}` - const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8')) - assert.ok(isJsonValue(parsed)) - const document = serializeCanonicalJson(parsed, 'compact') - const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8') - assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`) - assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`) - documents.set(document, fixture) - spellings.set(markdown, fixture) - } + for (const fixture of roundTripFixtures()) { + const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8')) + assert.ok(isAdfDocument(parsed), `${fixture.name} is not an ADF document`) + const result = adfToMarkdown(parsed) + assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`) + assert.equal(spellings.get(result.value), undefined, `${fixture.name} and ${spellings.get(result.value)} share one markdown spelling`) + spellings.set(result.value, fixture.name) + } +}) + +test('no round-trip fixture repeats the document another holds', () => { + const documents = new Map() + for (const fixture of roundTripFixtures()) { + const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8')) + assert.ok(isJsonValue(parsed)) + const document = serializeCanonicalJson(parsed, 'compact') + assert.equal(documents.get(document), undefined, `${fixture.name} repeats the document ${documents.get(document)} holds`) + documents.set(document, fixture.name) } }) diff --git a/src/emphasis-matching.ts b/src/emphasis-matching.ts index 2b55059..8792ce0 100644 --- a/src/emphasis-matching.ts +++ b/src/emphasis-matching.ts @@ -1,40 +1,37 @@ -export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number } +import { isUnicodeWhitespace } from './commonmark-grammar.ts' -export type EmphasisRun = { - canClose: boolean - canOpen: boolean - character: string - delimiters: readonly EmphasisDelimiter[] - end: number - start: number -} +export type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } -type Candidate = { +export type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } + +type Candidate = { head: number - next: Candidate | undefined + next: Candidate | undefined original: number - previous: Candidate | undefined + previous: Candidate | undefined remaining: number - run: EmphasisRun + run: Run tail: number } -// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document. -export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined { - const matched = matchDelimiters(runs) - // The last opener left unpaired is the innermost: the smallest carry that changes the line. - let innermost: number | undefined - for (const run of runs) { - for (const delimiter of run.delimiters) { - if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair - } - } - return innermost +const unicodePunctuation = /[\p{P}\p{S}]/u + +// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. +export function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { + const left = isLeftFlanking(before, after) + const right = isRightFlanking(before, after) + if (character !== '_') return { canClose: right, canOpen: left } + return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } } -function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { - const matched = new Set() - const bottoms = new Map() +export function isWordCharacter(character: string): boolean { + return character !== '' && !isWhitespace(character) && !isPunctuation(character) +} + +// Flanking decides which delimiters may pair; this decides which ones do, and a pair it leaves out reads back as another document. +export function matchEmphasis(runs: readonly Run[]): EmphasisPairing[] { + const pairings: EmphasisPairing[] = [] + const bottoms = new Map | undefined>() let closer = candidates(runs) while (closer !== undefined) { if (!closer.run.canClose) { @@ -53,9 +50,9 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { continue } const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1 - record(matched, opener, closer, used) opener.remaining -= used opener.tail -= used + pairings.push({ closer: closer.run, closerOffset: closer.head, opener: opener.run, openerOffset: opener.tail, used }) closer.head += used closer.remaining -= used opener.next = closer @@ -66,15 +63,22 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { unlink(closer) closer = following } - return matched + return pairings } -function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { - let first: Candidate | undefined - let previous: Candidate | undefined +function candidates(runs: readonly Run[]): Candidate | undefined { + let first: Candidate | undefined + let previous: Candidate | undefined for (const run of runs) { - const length = run.end - run.start - const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end } + const candidate: Candidate = { + head: 0, + next: undefined, + original: run.length, + previous, + remaining: run.length, + run, + tail: run.length, + } if (previous === undefined) first = candidate else previous.next = candidate previous = candidate @@ -82,19 +86,33 @@ function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { return first } -function pairs(opener: Candidate, closer: Candidate): boolean { +function pairs(opener: Candidate, closer: Candidate): boolean { if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0 return !odd } -function record(matched: Set, opener: Candidate, closer: Candidate, used: number): void { - const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail) - const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used) - if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) -} - -function unlink(candidate: Candidate): void { +function unlink(candidate: Candidate): void { if (candidate.previous !== undefined) candidate.previous.next = candidate.next if (candidate.next !== undefined) candidate.next.previous = candidate.previous } + +function isLeftFlanking(before: string, after: string): boolean { + if (isWhitespace(after)) return false + if (!isPunctuation(after)) return true + return isWhitespace(before) || isPunctuation(before) +} + +function isRightFlanking(before: string, after: string): boolean { + if (isWhitespace(before)) return false + if (!isPunctuation(before)) return true + return isWhitespace(after) || isPunctuation(after) +} + +function isPunctuation(character: string): boolean { + return character !== '' && unicodePunctuation.test(character) +} + +function isWhitespace(character: string): boolean { + return character === '' || isUnicodeWhitespace(character) +} diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index 7182e5c..e3cb889 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -1,5 +1,5 @@ -import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' -import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts' +import { escapesLineClaim, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' +import { delimiterFlags, isWordCharacter, matchEmphasis } from './emphasis-matching.ts' export type EmphasisRole = 'close' | 'open' @@ -15,7 +15,9 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine export type LineContainer = 'heading' | 'paragraph' | 'table-cell' -type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number } +type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number } + +type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; delimiters: EmittedDelimiter[]; length: number; start: number } const delimiters = ['*', '_', '`', '~'] @@ -23,7 +25,6 @@ const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/ const followsLinkText = /[([:]/ -const unicodePunctuation = /[\p{P}\p{S}]/u export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { return escape(resolveEmphasis(segments), container) @@ -79,18 +80,43 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A } function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined { - const { nodes, runs } = emphasisRuns(segments, placements, output) - for (const run of runs) { - for (const delimiter of run.delimiters) { - if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair] - } - } - const unmatched = unmatchedPair(runs) - return unmatched === undefined ? undefined : nodes[unmatched] + const { nodes, runs } = emittedRuns(segments, placements, output) + const pair = misflanked(runs) ?? unpaired(runs) + return pair === undefined ? undefined : nodes[pair] } -function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } { - const groups: DelimiterGroup[] = [] +function misflanked(runs: readonly EmittedRun[]): number | undefined { + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!(delimiter.closes ? run.canClose : run.canOpen)) return delimiter.pair + } + } + return undefined +} + +function unpaired(runs: readonly EmittedRun[]): number | undefined { + const matched = new Set() + for (const pairing of matchEmphasis(runs)) { + const opened = delimiterAt(pairing.opener, false, pairing.openerOffset, pairing.used) + const closed = delimiterAt(pairing.closer, true, pairing.closerOffset, pairing.used) + if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) + } + // The last opener left unpaired is the innermost: the smallest carry that changes the line. + let innermost: number | undefined + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair + } + } + return innermost +} + +function delimiterAt(run: EmittedRun, closes: boolean, offset: number, width: number): EmittedDelimiter | undefined { + return run.delimiters.find((delimiter) => delimiter.closes === closes && delimiter.offset === offset && delimiter.width === width) +} + +function emittedRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmittedRun[] } { + const runs: EmittedRun[] = [] const nodes: NodeRange[] = [] const open: number[] = [] let cursor = 0 @@ -99,30 +125,33 @@ function emphasisRuns(segments: readonly InlineSegment[], placements: readonly n cursor += segment.text.length if (segment.emphasis === undefined) continue const closes = segment.emphasis === 'close' - const end = start + segment.text.length const pair = closes ? (open.pop() ?? nodes.length) : nodes.length if (!closes) { nodes.push(segment.nodes) open.push(pair) } - const delimiter = { closes, end, pair, start } - const previous = groups[groups.length - 1] - if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) { - previous.delimiters.push(delimiter) - previous.end = end + const width = segment.text.length + const previous = runs[runs.length - 1] + if (previous !== undefined && previous.start + previous.length === start && previous.character === segment.text.charAt(0)) { + previous.delimiters.push({ closes, offset: start - previous.start, pair, width }) + previous.length += width continue } - groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start }) + runs.push({ + canClose: false, + canOpen: false, + character: segment.text.charAt(0), + delimiters: [{ closes, offset: 0, pair, width }], + length: width, + start, + }) } - return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) } -} - -// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. -function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { - const left = isLeftFlanking(before, after) - const right = isRightFlanking(before, after) - if (character !== '_') return { canClose: right, canOpen: left } - return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } + for (const run of runs) { + const flags = delimiterFlags(run.character, charAt(output, run.start - 1), output.charAt(run.start + run.length)) + run.canClose = flags.canClose + run.canOpen = flags.canOpen + } + return { nodes, runs } } function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean { @@ -228,29 +257,8 @@ function runLength(scan: string, index: number): number { return length } -function isLeftFlanking(before: string, after: string): boolean { - if (isWhitespace(after)) return false - if (!isPunctuation(after)) return true - return isWhitespace(before) || isPunctuation(before) -} -function isRightFlanking(before: string, after: string): boolean { - if (isWhitespace(before)) return false - if (!isPunctuation(before)) return true - return isWhitespace(after) || isPunctuation(after) -} -function isPunctuation(character: string): boolean { - return character !== '' && unicodePunctuation.test(character) -} - -function isWhitespace(character: string): boolean { - return character === '' || isUnicodeWhitespace(character) -} - -function isWordCharacter(character: string): boolean { - return character !== '' && !isUnicodeWhitespace(character) && !unicodePunctuation.test(character) -} function charAt(text: string, index: number): string { return index < 0 ? '' : text.charAt(index) diff --git a/src/markdown-inline.ts b/src/markdown-inline.ts index ae4f211..d6b2fa6 100644 --- a/src/markdown-inline.ts +++ b/src/markdown-inline.ts @@ -48,7 +48,7 @@ export function tryImageLine(alt: string | undefined, href: string, path: Conver return attempt.ok ? attempt.value.line : undefined } -// A demand names a run no spelling holds, and a carried node joins no run, so every pass carries at least one more node. +// A carried node joins no run, so every pass carries at least one more node. function emitLine(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath): Result { const carried = new Set() for (;;) { diff --git a/todo.md b/todo.md index 76b767d..9039eb1 100644 --- a/todo.md +++ b/todo.md @@ -162,7 +162,10 @@ detail is settled at its own milestone. stay above all of it — the vocabulary a string-typed attribute grammar needs, which is why HTML will want them too, not a markdown spelling. Both node tables are a second copy of `spec/flavour.md`'s prose with no drift guard, and a mistyped attribute name degrades into a - false refusal no test catches. + false refusal no test catches. The parser reuses `emphasis-matching.ts` whole and lands it + beside the grammar module: `delimiterFlags` and `matchEmphasis` take CommonMark's own run + vocabulary rather than the emitter's, so no second `process_emphasis` exists to drift from + the first. - [ ] **4 — Round-trip property tests** over the corpus, both ways — the thing that proves 2 and 3. Editor-normal (§2) gets its implementation here — `toEditorNormal(doc)` and the equality the round-trip asserts, which over normalized input is the canonical serializer's compact @@ -171,7 +174,10 @@ detail is settled at its own milestone. lifts the branch floor §10 keeps below 100 for exactly those halves. Generators emit editor-normal ADF (§2). Real sanitized ADF from live Atlassian APIs lands here too (§10), in `corpus/real-payloads/`: an ADF→markdown→ADF check with no expected - markdown, the payloads supplied by the maintainer. + markdown, the payloads supplied by the maintainer. This subsumes 2e5's collision property — + a document that round-trips proves no other document shares its spelling — so decide here + whether that gate stays as the parser-free, faster-failing signal or goes; the half holding + no fixture duplicates is hygiene rather than a round-trip claim, and stays either way. - [ ] **5 — Release pipeline, ship `0.1.0`.** Publish-on-version-change (§9), `NPM_TOKEN` secret, the repo made public first (§6). The `ConvertErrorCode` freeze (§8) is checkable here: every `corpus/unspellable/` document is a decision or a deferred trigger this file names, so the