From e4a18dec43fd99e073692510f7b39a41f84084ae Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 27 Aug 2026 12:42:46 +0200 Subject: [PATCH] Model CommonMark's emphasis matching, and escape the delimiter that only closes --- AGENTS.md | 3 + .../combinations/carve-out-strike.md | 4 +- .../combinations/combined-containers.json | 161 +++++++++ .../combinations/combined-containers.md | 39 +++ .../combinations/combined-page.json | 312 ++++++++++++++++++ .../round-trip/combinations/combined-page.md | 20 ++ .../combinations/mark-closing-delimiter.json | 56 ++++ .../combinations/mark-closing-delimiter.md | 5 + .../combinations/mark-run-matching.json | 87 +++++ .../combinations/mark-run-matching.md | 3 + .../commonmark-subset/text-escaping.md | 2 +- spec/flavour.md | 11 +- src/adf-to-markdown.test.ts | 40 ++- src/corpus.test.ts | 19 ++ src/emphasis-matching.ts | 100 ++++++ src/markdown-escaping.ts | 64 ++-- todo.md | 18 +- 17 files changed, 899 insertions(+), 45 deletions(-) create mode 100644 corpus/round-trip/combinations/combined-containers.json create mode 100644 corpus/round-trip/combinations/combined-containers.md create mode 100644 corpus/round-trip/combinations/combined-page.json create mode 100644 corpus/round-trip/combinations/combined-page.md create mode 100644 corpus/round-trip/combinations/mark-closing-delimiter.json create mode 100644 corpus/round-trip/combinations/mark-closing-delimiter.md create mode 100644 corpus/round-trip/combinations/mark-run-matching.json create mode 100644 corpus/round-trip/combinations/mark-run-matching.md create mode 100644 src/emphasis-matching.ts diff --git a/AGENTS.md b/AGENTS.md index a5df455..63e8d3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga the spot. - Only the hard break's inline segment holds a raw newline โ€” every other spelling escapes one or refuses it โ€” which is how the whitespace carry finds a line edge. +- Emphasis is spelled against CommonMark's matching, never flanking alone: a delimiter run in text + escapes wherever CommonMark could open or close with it, leaving the emitter's own delimiters the + only ones in play, and a pair that matching hands to another delimiter rides the carry instead. - A readable spelling tried ahead of a general one โ€” the image, the pipe table, a pipe cell โ€” returns `string | undefined`, never a `Result`: any failure is the fallback signal, and the general form owns the refusal. Refusing there refuses a document the general form spells. diff --git a/corpus/round-trip/combinations/carve-out-strike.md b/corpus/round-trip/combinations/carve-out-strike.md index a5765d7..62af9c9 100644 --- a/corpus/round-trip/combinations/carve-out-strike.md +++ b/corpus/round-trip/combinations/carve-out-strike.md @@ -1,6 +1,6 @@ -\~~not strike~~ +\~~not strike\~~ -~~gone~~ but \~~kept~~ +~~gone~~ but \~~kept\~~ A ~ B ~ C diff --git a/corpus/round-trip/combinations/combined-containers.json b/corpus/round-trip/combinations/combined-containers.json new file mode 100644 index 0000000..c44e475 --- /dev/null +++ b/corpus/round-trip/combinations/combined-containers.json @@ -0,0 +1,161 @@ +{ + "content": [ + { + "content": [ + { + "attrs": { + "width": 50 + }, + "content": [ + { + "attrs": { + "panelType": "info" + }, + "content": [ + { + "content": [ + { + "text": "Check the collation before importing.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "panel" + } + ], + "type": "layoutColumn" + }, + { + "attrs": { + "width": 50 + }, + "content": [ + { + "attrs": { + "localId": "0198f3a2-7c41-7f2e-9b3a-4d8e2c1a6b90" + }, + "content": [ + { + "attrs": { + "localId": "0198f3a2-8d52-70b1-8c4f-5e9f3d2b7ca1", + "state": "DONE" + }, + "content": [ + { + "text": "Write the spec", + "type": "text" + } + ], + "type": "taskItem" + }, + { + "attrs": { + "localId": "0198f3a2-9e63-7d80-a15b-6fa04e3c8db2", + "state": "TODO" + }, + "content": [ + { + "text": "Ship it", + "type": "text" + } + ], + "type": "taskItem" + } + ], + "type": "taskList" + } + ], + "type": "layoutColumn" + } + ], + "type": "layoutSection" + }, + { + "attrs": { + "layout": "center", + "width": 50 + }, + "content": [ + { + "attrs": { + "collection": "MediaServicesSample", + "id": "4478e39c-cf9b-41d1-ba92-68589487cd75", + "type": "file" + }, + "type": "media" + }, + { + "content": [ + { + "text": "The moon, at night.", + "type": "text" + } + ], + "type": "caption" + } + ], + "type": "mediaSingle" + }, + { + "attrs": { + "title": "Full build log" + }, + "content": [ + { + "attrs": { + "isNumberColumnEnabled": true + }, + "content": [ + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Step", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "attrs": { + "background": "#deebff" + }, + "content": [ + { + "content": [ + { + "text": "Compile", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + } + ], + "type": "table" + } + ], + "type": "expand" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/combined-containers.md b/corpus/round-trip/combinations/combined-containers.md new file mode 100644 index 0000000..890497f --- /dev/null +++ b/corpus/round-trip/combinations/combined-containers.md @@ -0,0 +1,39 @@ +::::::layoutSection +::::layoutColumn {width=50} +:::panel info +Check the collation before importing. +::: +:::: +:::::layoutColumn {width=50} +::::taskList {localId=0198f3a2-7c41-7f2e-9b3a-4d8e2c1a6b90} +:::taskItem DONE {localId=0198f3a2-8d52-70b1-8c4f-5e9f3d2b7ca1} +Write the spec +::: +:::taskItem TODO {localId=0198f3a2-9e63-7d80-a15b-6fa04e3c8db2} +Ship it +::: +:::: +::::: +:::::: + +::::mediaSingle {layout=center width=50} +::media {collection=MediaServicesSample id=4478e39c-cf9b-41d1-ba92-68589487cd75 type=file} +:::caption +The moon, at night. +::: +:::: + +::::::expand {title="Full build log"} +:::::table {isNumberColumnEnabled=true} +::::tableRow +:::tableHeader +Step +::: +:::: +::::tableRow +:::tableCell {background="#deebff"} +Compile +::: +:::: +::::: +:::::: diff --git a/corpus/round-trip/combinations/combined-page.json b/corpus/round-trip/combinations/combined-page.json new file mode 100644 index 0000000..f9341e3 --- /dev/null +++ b/corpus/round-trip/combinations/combined-page.json @@ -0,0 +1,312 @@ +{ + "content": [ + { + "attrs": { + "level": 1 + }, + "content": [ + { + "text": "Release 2.4", + "type": "text" + } + ], + "type": "heading" + }, + { + "content": [ + { + "text": "Shipped ", + "type": "text" + }, + { + "attrs": { + "shortName": ":rocket:", + "text": "๐Ÿš€" + }, + "type": "emoji" + }, + { + "text": " on ", + "type": "text" + }, + { + "attrs": { + "timestamp": "1756080000000" + }, + "type": "date" + }, + { + "text": " โ€” ", + "type": "text" + }, + { + "attrs": { + "id": "01a032c3-7a7c-775f-a730-2d79351338b4", + "text": "@Mikael" + }, + "type": "mention" + }, + { + "text": " owns the rollout, status ", + "type": "text" + }, + { + "attrs": { + "color": "yellow", + "text": "In review" + }, + "type": "status" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Part", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + }, + { + "content": [ + { + "content": [ + { + "text": "Qty", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Bolt M8", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + }, + { + "content": [ + { + "content": [ + { + "text": "40", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "marks": [ + { + "attrs": { + "href": "https://example.com/washer" + }, + "type": "link" + } + ], + "text": "Washer", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + }, + { + "content": [ + { + "content": [ + { + "text": "12", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + } + ], + "type": "table" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Torque the ", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "M8 bolt", + "type": "text" + }, + { + "text": " to ", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "25 Nm", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + }, + { + "content": [ + { + "content": [ + { + "text": "Check the collation:", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "marks": [ + { + "type": "code" + } + ], + "text": "mysqldump --default-character-set=utf8mb4", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + }, + { + "content": [ + { + "content": [ + { + "text": "Rolled back once, see the ", + "type": "text" + }, + { + "marks": [ + { + "type": "underline" + } + ], + "text": "postmortem", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "blockquote" + }, + { + "type": "rule" + }, + { + "attrs": { + "panelType": "warning" + }, + "content": [ + { + "content": [ + { + "text": "Do not skip the pre-flight.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "panel" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/combined-page.md b/corpus/round-trip/combinations/combined-page.md new file mode 100644 index 0000000..17439fd --- /dev/null +++ b/corpus/round-trip/combinations/combined-page.md @@ -0,0 +1,20 @@ +# Release 2.4 + +Shipped :emoji[๐Ÿš€]{shortName=":rocket:"} on :date{timestamp=1756080000000} โ€” :mention[@Mikael]{id=01a032c3-7a7c-775f-a730-2d79351338b4} owns the rollout, status :status[In review]{color=yellow}. + +| Part | Qty | +| --- | --- | +| Bolt M8 | 40 | +| [Washer](https://example.com/washer) | 12 | + +- Torque the **M8 bolt** to _25 Nm_. +- Check the collation: + - `mysqldump --default-character-set=utf8mb4` + +> Rolled back once, see the :underline[postmortem]. + +--- + +:::panel warning +Do not skip the pre-flight. +::: diff --git a/corpus/round-trip/combinations/mark-closing-delimiter.json b/corpus/round-trip/combinations/mark-closing-delimiter.json new file mode 100644 index 0000000..700f954 --- /dev/null +++ b/corpus/round-trip/combinations/mark-closing-delimiter.json @@ -0,0 +1,56 @@ +{ + "content": [ + { + "content": [ + { + "text": "un", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "a* b", + "type": "text" + }, + { + "text": "istic", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "strike" + } + ], + "text": "a~~ b", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "em" + } + ], + "text": "a_ b", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/mark-closing-delimiter.md b/corpus/round-trip/combinations/mark-closing-delimiter.md new file mode 100644 index 0000000..17a0f36 --- /dev/null +++ b/corpus/round-trip/combinations/mark-closing-delimiter.md @@ -0,0 +1,5 @@ +un*a\* b*istic + +~~a\~~ b~~ + +_a\_ b_ diff --git a/corpus/round-trip/combinations/mark-run-matching.json b/corpus/round-trip/combinations/mark-run-matching.json new file mode 100644 index 0000000..cb35ec6 --- /dev/null +++ b/corpus/round-trip/combinations/mark-run-matching.json @@ -0,0 +1,87 @@ +{ + "content": [ + { + "content": [ + { + "text": "un", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "a", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + }, + { + "type": "strong" + } + ], + "text": "b", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "c", + "type": "text" + }, + { + "text": "istic", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "strong" + }, + { + "type": "em" + } + ], + "text": "a", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "b", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + }, + { + "type": "em" + } + ], + "text": "c", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/mark-run-matching.md b/corpus/round-trip/combinations/mark-run-matching.md new file mode 100644 index 0000000..5c9aa25 --- /dev/null +++ b/corpus/round-trip/combinations/mark-run-matching.md @@ -0,0 +1,3 @@ +un:adf{json="{\"marks\":[{\"type\":\"em\"}],\"text\":\"a\",\"type\":\"text\"}"}:adf{json="{\"marks\":[{\"type\":\"em\"},{\"type\":\"strong\"}],\"text\":\"b\",\"type\":\"text\"}"}**c**istic + +***a*b**:adf{json="{\"marks\":[{\"type\":\"strong\"},{\"type\":\"em\"}],\"text\":\"c\",\"type\":\"text\"}"} diff --git a/corpus/round-trip/commonmark-subset/text-escaping.md b/corpus/round-trip/commonmark-subset/text-escaping.md index c30602b..c1ecc79 100644 --- a/corpus/round-trip/commonmark-subset/text-escaping.md +++ b/corpus/round-trip/commonmark-subset/text-escaping.md @@ -6,4 +6,4 @@ snake_case_name -\*not emphasis* +\*not emphasis\* diff --git a/spec/flavour.md b/spec/flavour.md index bb1da93..0106017 100644 --- a/spec/flavour.md +++ b/spec/flavour.md @@ -39,9 +39,10 @@ normalizes to it through the round-trip. - Paragraphs on one line โ€” no soft wrapping; a soft line break in input becomes a single space. - Entity references in input decode to their characters; output backslash-escapes only where text would otherwise parse as syntax, scanning the assembled line rather than each text node: escape - the leading delimiter of a construct that would otherwise open, re-scan from there, and - repeat โ€” with the opener literal the closer parses as text, so `*not emphasis*` is - `\*not emphasis*`, one backslash. + the leading delimiter of a construct that would otherwise open, re-scan from there, and repeat. + An emphasis delimiter run in text escapes where CommonMark can open **or** close with it, so + `*not emphasis*` is `\*not emphasis\*` โ€” no delimiter the emitter did not write reaches the + matching below, which is what lets the emitter decide its own pairings. - Blocks separated by one blank line at document level, inside a blockquote and between CommonMark blocks; two directive blocks inside a container take none. No trailing whitespace outside a code block's @@ -390,7 +391,9 @@ An inline node whose marks no nesting spells โ€” a mark type not listed here, an spelling does not list, a value that is not the spelling's type, an attribute the spelling needs and the mark lacks, an order putting a code span outside another mark, `code` over anything but a text node or over text holding a newline, or a spelling CommonMark's flanking rules cannot open or -close where the run sits (`un**-real**istic`) โ€” rides the inline carry whole. A value the spelling +close where the run sits (`un**-real**istic`), or one CommonMark's matching pairs elsewhere โ€” the +intra-word `*` runs together with a neighbouring `**`, and the multiple-of-3 rule can leave the +merged run's pairing to another delimiter โ€” rides the inline carry whole. A value the spelling holds but CommonMark cannot write โ€” a link destination or title โ€” is a named error instead. An opaque carry inside a mark spelling is a named error in input: the carry restores its node exactly, marks included (AGENTS.md ยง3). diff --git a/src/adf-to-markdown.test.ts b/src/adf-to-markdown.test.ts index 21d00ab..1977135 100644 --- a/src/adf-to-markdown.test.ts +++ b/src/adf-to-markdown.test.ts @@ -197,12 +197,12 @@ test('escapes only text that would otherwise open a construct', () => { assert.equal(emitted(':::panel info'), '\\:::panel info\n') assert.equal(emitted('10:30 tomorrow'), '10:30 tomorrow\n') assert.equal(emitted('[a](b)'), '\\[a](b)\n') - assert.equal(emitted('**bold**'), '\\*\\*bold**\n') + assert.equal(emitted('**bold**'), '\\*\\*bold\\*\\*\n') assert.equal(emitted('a `x` b'), 'a \\`x` b\n') - assert.equal(emitted('~~struck~~'), '\\~~struck~~\n') - assert.equal(emitted('a \\* b'), 'a \\\\* b\n') + assert.equal(emitted('~~struck~~'), '\\~~struck\\~~\n') + assert.equal(emitted('a \\* b'), 'a \\\\\\* b\n') assert.equal(emitted('1. not a list'), '1\\. not a list\n') - assert.equal(emitted('*"quoted"*'), '\\*"quoted"*\n') + assert.equal(emitted('*"quoted"*'), '\\*"quoted"\\*\n') assert.equal(emitted('x"_y"'), 'x"\\_y"\n') }) @@ -297,6 +297,38 @@ test('escapes a literal delimiter that would merge with an emitted one', () => { assert.equal(emitted(marked('x', { attrs: { href: 'https://example.com/' }, type: 'link' }), { text: '{}', type: 'text' }), '[x](https://example.com/){}\n') }) +test('escapes a literal delimiter run that flanks either way', () => { + const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) + const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) + assert.equal(emitted({ text: 'un', type: 'text' }, marked('a* b', { type: 'em' }), { text: 'istic', type: 'text' }), 'un*a\\* b*istic\n') + assert.equal(emitted(marked('a~~ b', { type: 'strike' })), '~~a\\~~ b~~\n') + assert.equal(emitted(marked('a_ b', { type: 'em' })), '_a\\_ b_\n') + assert.equal(emitted({ text: 'a* b', type: 'text' }), 'a\\* b\n') + assert.equal(emitted({ text: '2 * 3', type: 'text' }), '2 * 3\n') + assert.equal(emitted({ text: 'snake_case_name', type: 'text' }), 'snake_case_name\n') +}) + +test('carries a mark run CommonMark matching pairs elsewhere', () => { + const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) + const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) + const em: AdfMark = { type: 'em' } + const strong: AdfMark = { type: 'strong' } + assert.equal( + emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', strong), { text: 'istic', type: 'text' }), + 'un:adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"}],\\"text\\":\\"a\\",\\"type\\":\\"text\\"}"}' + + ':adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"},{\\"type\\":\\"strong\\"}],\\"text\\":\\"b\\",\\"type\\":\\"text\\"}"}**c**istic\n', + ) + assert.equal( + emitted(marked('a', strong, em), marked('b', strong), marked('c', strong, em)), + '***a*b**:adf{json="{\\"marks\\":[{\\"type\\":\\"strong\\"},{\\"type\\":\\"em\\"}],\\"text\\":\\"c\\",\\"type\\":\\"text\\"}"}\n', + ) + assert.equal(emitted({ text: 'un', type: 'text' }, marked('a', em, strong), { text: 'istic', type: 'text' }), 'un***a***istic\n') + assert.equal( + emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', em), { text: 'istic', type: 'text' }), + 'un*a**b**c*istic\n', + ) +}) + test('escapes a hyphen underline a hard break would expose', () => { const line = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text: 'foo', type: 'text' }, { type: 'hardBreak' }, { text, type: 'text' })))) assert.equal(line('--'), 'foo\\\n\\--\n') diff --git a/src/corpus.test.ts b/src/corpus.test.ts index 33e87d8..60ff98d 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -66,6 +66,25 @@ for (const directory of emittingDirectories) { } } +// One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it. +test('no two round-trip documents share one spelling', () => { + const documents = new Map() + const spellings = new Map() + for (const directory of emittingDirectories) { + for (const name of fixtureNames(directory, '.md')) { + const fixture = `${directory}/${name}` + const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8')) + assert.ok(isJsonValue(parsed)) + const document = serializeCanonicalJson(parsed, 'compact') + const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8') + assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`) + assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`) + documents.set(document, fixture) + spellings.set(markdown, fixture) + } + } +}) + // spec/flavour.md, Directives: the container fence rule, checked against the emitted bytes. function fenceNestingFault(markdown: string): string | undefined { const open: number[] = [] diff --git a/src/emphasis-matching.ts b/src/emphasis-matching.ts new file mode 100644 index 0000000..2b55059 --- /dev/null +++ b/src/emphasis-matching.ts @@ -0,0 +1,100 @@ +export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number } + +export type EmphasisRun = { + canClose: boolean + canOpen: boolean + character: string + delimiters: readonly EmphasisDelimiter[] + end: number + start: number +} + +type Candidate = { + head: number + next: Candidate | undefined + original: number + previous: Candidate | undefined + remaining: number + run: EmphasisRun + tail: number +} + +// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document. +export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined { + const matched = matchDelimiters(runs) + // The last opener left unpaired is the innermost: the smallest carry that changes the line. + let innermost: number | undefined + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair + } + } + return innermost +} + +function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { + const matched = new Set() + const bottoms = new Map() + let closer = candidates(runs) + while (closer !== undefined) { + if (!closer.run.canClose) { + closer = closer.next + continue + } + const key = `${closer.run.character}${closer.run.canOpen}${closer.original % 3}` + const bottom = bottoms.get(key) + let opener = closer.previous + while (opener !== undefined && opener !== bottom && !pairs(opener, closer)) opener = opener.previous + if (opener === undefined || opener === bottom) { + bottoms.set(key, closer.previous) + const following = closer.next + if (!closer.run.canOpen) unlink(closer) + closer = following + continue + } + const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1 + record(matched, opener, closer, used) + opener.remaining -= used + opener.tail -= used + closer.head += used + closer.remaining -= used + opener.next = closer + closer.previous = opener + if (opener.remaining === 0) unlink(opener) + if (closer.remaining > 0) continue + const following = closer.next + unlink(closer) + closer = following + } + return matched +} + +function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { + let first: Candidate | undefined + let previous: Candidate | undefined + for (const run of runs) { + const length = run.end - run.start + const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end } + if (previous === undefined) first = candidate + else previous.next = candidate + previous = candidate + } + return first +} + +function pairs(opener: Candidate, closer: Candidate): boolean { + if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false + const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0 + return !odd +} + +function record(matched: Set, opener: Candidate, closer: Candidate, used: number): void { + const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail) + const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used) + if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) +} + +function unlink(candidate: Candidate): void { + if (candidate.previous !== undefined) candidate.previous.next = candidate.next + if (candidate.next !== undefined) candidate.next.previous = candidate.previous +} diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index e5e8342..7182e5c 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -1,4 +1,5 @@ import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' +import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts' export type EmphasisRole = 'close' | 'open' @@ -14,7 +15,7 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine export type LineContainer = 'heading' | 'paragraph' | 'table-cell' -type DelimiterRun = { character: string; closeNodes: NodeRange | undefined; end: number; openNodes: NodeRange | undefined; start: number } +type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number } const delimiters = ['*', '_', '`', '~'] @@ -78,17 +79,20 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A } function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined { - for (const run of delimiterRuns(segments, placements)) { - const before = charAt(output, run.start - 1) - const after = output.charAt(run.end) - if (run.openNodes !== undefined && !isLeftFlanking(before, after)) return run.openNodes - if (run.closeNodes !== undefined && !isRightFlanking(before, after)) return run.closeNodes + const { nodes, runs } = emphasisRuns(segments, placements, output) + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair] + } } - return undefined + const unmatched = unmatchedPair(runs) + return unmatched === undefined ? undefined : nodes[unmatched] } -function delimiterRuns(segments: readonly InlineSegment[], placements: readonly number[]): DelimiterRun[] { - const runs: DelimiterRun[] = [] +function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } { + const groups: DelimiterGroup[] = [] + const nodes: NodeRange[] = [] + const open: number[] = [] let cursor = 0 for (const segment of segments) { const start = placements[cursor] ?? 0 @@ -96,22 +100,29 @@ function delimiterRuns(segments: readonly InlineSegment[], placements: readonly if (segment.emphasis === undefined) continue const closes = segment.emphasis === 'close' const end = start + segment.text.length - const previous = runs[runs.length - 1] + const pair = closes ? (open.pop() ?? nodes.length) : nodes.length + if (!closes) { + nodes.push(segment.nodes) + open.push(pair) + } + const delimiter = { closes, end, pair, start } + const previous = groups[groups.length - 1] if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) { - previous.closeNodes = previous.closeNodes ?? (closes ? segment.nodes : undefined) + previous.delimiters.push(delimiter) previous.end = end - previous.openNodes = previous.openNodes ?? (closes ? undefined : segment.nodes) continue } - runs.push({ - character: segment.text.charAt(0), - closeNodes: closes ? segment.nodes : undefined, - end, - openNodes: closes ? undefined : segment.nodes, - start, - }) + groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start }) } - return runs + return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) } +} + +// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. +function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { + const left = isLeftFlanking(before, after) + const right = isRightFlanking(before, after) + if (character !== '_') return { canClose: right, canOpen: left } + return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } } function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean { @@ -177,7 +188,7 @@ function claimsCharacter( if (character === ':') return inlineDirectiveOpener.test(rest) if (character === '[') return opensLink(scan, escapings, index) if (character === '`') return opensCodeSpan(scan, index, escaped) - if (character === '*' || character === '_' || character === '~') return opensEmphasis(scan, index, escaped) + if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped) return false } @@ -196,16 +207,13 @@ function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet return new RegExp('(?): boolean { +function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet): boolean { if (!startsRun(scan, index, escaped)) return false const character = scan.charAt(index) const length = runLength(scan, index) - const before = index === 0 ? '' : scan.charAt(index - 1) - const after = scan.charAt(index + length) - if (character === '~') return length === 2 && isLeftFlanking(before, after) - if (!isLeftFlanking(before, after)) return false - if (character === '*') return true - return !isRightFlanking(before, after) || isPunctuation(before) + if (character === '~' && length !== 2) return false + const flags = delimiterFlags(character, index === 0 ? '' : scan.charAt(index - 1), scan.charAt(index + length)) + return flags.canClose || flags.canOpen } function startsRun(scan: string, index: number, escaped: ReadonlySet): boolean { diff --git a/todo.md b/todo.md index e0ed60a..76b767d 100644 --- a/todo.md +++ b/todo.md @@ -110,14 +110,20 @@ detail is settled at its own milestone. added to that list is the odd one out: `unspellableMark` finds it after assembly and names a mark type against the line's path, so the failing run needs identifying before the carry can replace the refusal `mark-inside-word` pinned. - - [ ] **2e5 โ€” Combined documents and the collision property.** Documents combining nodes rather + - [x] **2e5 โ€” Combined documents and the collision property.** Documents combining nodes rather than isolating one, and the gate's collision property: no two corpus documents may emit the same bytes โ€” one spelling for two documents is a round-trip break no parser can undo, - and it is provable without one. It also settles the emitter's one known approximation: - delimiter flanking is exact, but CommonMark's *matching* โ€” the multiple-of-3 rule and the - way a run splits across several openers โ€” is not modelled. No reachable violation has been - found by hand; the property test is what decides it, and 2e1's `carve-out-strike` pins a - second backslash only flanking-without-matching emits. + and it is provable without one. + **Settled** (the maintainer, 2026-08-27): the approximation this item inherited โ€” flanking + exact, CommonMark's *matching* unmodelled โ€” had two round-trip breaks reachable by hand, + so the emitter now models the matching. `process_emphasis` runs over the runs the emitter + wrote (`emphasis-matching.ts`) and a pair it hands to another delimiter rides the carry, + which is what the multiple-of-3 rule did to the em in `un*a**b*****c**istic`. A delimiter + run in text now escapes wherever CommonMark could open or close with it, not only open: + one that could only close stole the spelling around it (`un*a* b*istic`), and escaping + both ways keeps every delimiter the emitter did not write out of the matching. The + canonical form gained a backslash where a run only closes โ€” `\*not emphasis\*`, and + 2e1's `carve-out-strike` a third and fourth. - [ ] **2f โ€” The attributes CommonMark cannot hold.** 1d's settled answer: the block nodes CommonMark spells โ€” `blockquote`, `bulletList`, `codeBlock`, `heading`, `listItem`, `orderedList`, `paragraph`, `rule` โ€” get directive sections in `spec/flavour.md` carrying