From e4a18dec43fd99e073692510f7b39a41f84084ae Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 27 Aug 2026 12:42:46 +0200 Subject: [PATCH 1/3] Model CommonMark's emphasis matching, and escape the delimiter that only closes --- AGENTS.md | 3 + .../combinations/carve-out-strike.md | 4 +- .../combinations/combined-containers.json | 161 +++++++++ .../combinations/combined-containers.md | 39 +++ .../combinations/combined-page.json | 312 ++++++++++++++++++ .../round-trip/combinations/combined-page.md | 20 ++ .../combinations/mark-closing-delimiter.json | 56 ++++ .../combinations/mark-closing-delimiter.md | 5 + .../combinations/mark-run-matching.json | 87 +++++ .../combinations/mark-run-matching.md | 3 + .../commonmark-subset/text-escaping.md | 2 +- spec/flavour.md | 11 +- src/adf-to-markdown.test.ts | 40 ++- src/corpus.test.ts | 19 ++ src/emphasis-matching.ts | 100 ++++++ src/markdown-escaping.ts | 64 ++-- todo.md | 18 +- 17 files changed, 899 insertions(+), 45 deletions(-) create mode 100644 corpus/round-trip/combinations/combined-containers.json create mode 100644 corpus/round-trip/combinations/combined-containers.md create mode 100644 corpus/round-trip/combinations/combined-page.json create mode 100644 corpus/round-trip/combinations/combined-page.md create mode 100644 corpus/round-trip/combinations/mark-closing-delimiter.json create mode 100644 corpus/round-trip/combinations/mark-closing-delimiter.md create mode 100644 corpus/round-trip/combinations/mark-run-matching.json create mode 100644 corpus/round-trip/combinations/mark-run-matching.md create mode 100644 src/emphasis-matching.ts diff --git a/AGENTS.md b/AGENTS.md index a5df455..63e8d3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga the spot. - Only the hard break's inline segment holds a raw newline β€” every other spelling escapes one or refuses it β€” which is how the whitespace carry finds a line edge. +- Emphasis is spelled against CommonMark's matching, never flanking alone: a delimiter run in text + escapes wherever CommonMark could open or close with it, leaving the emitter's own delimiters the + only ones in play, and a pair that matching hands to another delimiter rides the carry instead. - A readable spelling tried ahead of a general one β€” the image, the pipe table, a pipe cell β€” returns `string | undefined`, never a `Result`: any failure is the fallback signal, and the general form owns the refusal. Refusing there refuses a document the general form spells. diff --git a/corpus/round-trip/combinations/carve-out-strike.md b/corpus/round-trip/combinations/carve-out-strike.md index a5765d7..62af9c9 100644 --- a/corpus/round-trip/combinations/carve-out-strike.md +++ b/corpus/round-trip/combinations/carve-out-strike.md @@ -1,6 +1,6 @@ -\~~not strike~~ +\~~not strike\~~ -~~gone~~ but \~~kept~~ +~~gone~~ but \~~kept\~~ A ~ B ~ C diff --git a/corpus/round-trip/combinations/combined-containers.json b/corpus/round-trip/combinations/combined-containers.json new file mode 100644 index 0000000..c44e475 --- /dev/null +++ b/corpus/round-trip/combinations/combined-containers.json @@ -0,0 +1,161 @@ +{ + "content": [ + { + "content": [ + { + "attrs": { + "width": 50 + }, + "content": [ + { + "attrs": { + "panelType": "info" + }, + "content": [ + { + "content": [ + { + "text": "Check the collation before importing.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "panel" + } + ], + "type": "layoutColumn" + }, + { + "attrs": { + "width": 50 + }, + "content": [ + { + "attrs": { + "localId": "0198f3a2-7c41-7f2e-9b3a-4d8e2c1a6b90" + }, + "content": [ + { + "attrs": { + "localId": "0198f3a2-8d52-70b1-8c4f-5e9f3d2b7ca1", + "state": "DONE" + }, + "content": [ + { + "text": "Write the spec", + "type": "text" + } + ], + "type": "taskItem" + }, + { + "attrs": { + "localId": "0198f3a2-9e63-7d80-a15b-6fa04e3c8db2", + "state": "TODO" + }, + "content": [ + { + "text": "Ship it", + "type": "text" + } + ], + "type": "taskItem" + } + ], + "type": "taskList" + } + ], + "type": "layoutColumn" + } + ], + "type": "layoutSection" + }, + { + "attrs": { + "layout": "center", + "width": 50 + }, + "content": [ + { + "attrs": { + "collection": "MediaServicesSample", + "id": "4478e39c-cf9b-41d1-ba92-68589487cd75", + "type": "file" + }, + "type": "media" + }, + { + "content": [ + { + "text": "The moon, at night.", + "type": "text" + } + ], + "type": "caption" + } + ], + "type": "mediaSingle" + }, + { + "attrs": { + "title": "Full build log" + }, + "content": [ + { + "attrs": { + "isNumberColumnEnabled": true + }, + "content": [ + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Step", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "attrs": { + "background": "#deebff" + }, + "content": [ + { + "content": [ + { + "text": "Compile", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + } + ], + "type": "table" + } + ], + "type": "expand" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/combined-containers.md b/corpus/round-trip/combinations/combined-containers.md new file mode 100644 index 0000000..890497f --- /dev/null +++ b/corpus/round-trip/combinations/combined-containers.md @@ -0,0 +1,39 @@ +::::::layoutSection +::::layoutColumn {width=50} +:::panel info +Check the collation before importing. +::: +:::: +:::::layoutColumn {width=50} +::::taskList {localId=0198f3a2-7c41-7f2e-9b3a-4d8e2c1a6b90} +:::taskItem DONE {localId=0198f3a2-8d52-70b1-8c4f-5e9f3d2b7ca1} +Write the spec +::: +:::taskItem TODO {localId=0198f3a2-9e63-7d80-a15b-6fa04e3c8db2} +Ship it +::: +:::: +::::: +:::::: + +::::mediaSingle {layout=center width=50} +::media {collection=MediaServicesSample id=4478e39c-cf9b-41d1-ba92-68589487cd75 type=file} +:::caption +The moon, at night. +::: +:::: + +::::::expand {title="Full build log"} +:::::table {isNumberColumnEnabled=true} +::::tableRow +:::tableHeader +Step +::: +:::: +::::tableRow +:::tableCell {background="#deebff"} +Compile +::: +:::: +::::: +:::::: diff --git a/corpus/round-trip/combinations/combined-page.json b/corpus/round-trip/combinations/combined-page.json new file mode 100644 index 0000000..f9341e3 --- /dev/null +++ b/corpus/round-trip/combinations/combined-page.json @@ -0,0 +1,312 @@ +{ + "content": [ + { + "attrs": { + "level": 1 + }, + "content": [ + { + "text": "Release 2.4", + "type": "text" + } + ], + "type": "heading" + }, + { + "content": [ + { + "text": "Shipped ", + "type": "text" + }, + { + "attrs": { + "shortName": ":rocket:", + "text": "πŸš€" + }, + "type": "emoji" + }, + { + "text": " on ", + "type": "text" + }, + { + "attrs": { + "timestamp": "1756080000000" + }, + "type": "date" + }, + { + "text": " β€” ", + "type": "text" + }, + { + "attrs": { + "id": "01a032c3-7a7c-775f-a730-2d79351338b4", + "text": "@Mikael" + }, + "type": "mention" + }, + { + "text": " owns the rollout, status ", + "type": "text" + }, + { + "attrs": { + "color": "yellow", + "text": "In review" + }, + "type": "status" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Part", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + }, + { + "content": [ + { + "content": [ + { + "text": "Qty", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Bolt M8", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + }, + { + "content": [ + { + "content": [ + { + "text": "40", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "marks": [ + { + "attrs": { + "href": "https://example.com/washer" + }, + "type": "link" + } + ], + "text": "Washer", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + }, + { + "content": [ + { + "content": [ + { + "text": "12", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableCell" + } + ], + "type": "tableRow" + } + ], + "type": "table" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "text": "Torque the ", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "M8 bolt", + "type": "text" + }, + { + "text": " to ", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "25 Nm", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + }, + { + "content": [ + { + "content": [ + { + "text": "Check the collation:", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "content": [ + { + "content": [ + { + "marks": [ + { + "type": "code" + } + ], + "text": "mysqldump --default-character-set=utf8mb4", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + } + ], + "type": "listItem" + } + ], + "type": "bulletList" + }, + { + "content": [ + { + "content": [ + { + "text": "Rolled back once, see the ", + "type": "text" + }, + { + "marks": [ + { + "type": "underline" + } + ], + "text": "postmortem", + "type": "text" + }, + { + "text": ".", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "blockquote" + }, + { + "type": "rule" + }, + { + "attrs": { + "panelType": "warning" + }, + "content": [ + { + "content": [ + { + "text": "Do not skip the pre-flight.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "panel" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/combined-page.md b/corpus/round-trip/combinations/combined-page.md new file mode 100644 index 0000000..17439fd --- /dev/null +++ b/corpus/round-trip/combinations/combined-page.md @@ -0,0 +1,20 @@ +# Release 2.4 + +Shipped :emoji[πŸš€]{shortName=":rocket:"} on :date{timestamp=1756080000000} β€” :mention[@Mikael]{id=01a032c3-7a7c-775f-a730-2d79351338b4} owns the rollout, status :status[In review]{color=yellow}. + +| Part | Qty | +| --- | --- | +| Bolt M8 | 40 | +| [Washer](https://example.com/washer) | 12 | + +- Torque the **M8 bolt** to _25 Nm_. +- Check the collation: + - `mysqldump --default-character-set=utf8mb4` + +> Rolled back once, see the :underline[postmortem]. + +--- + +:::panel warning +Do not skip the pre-flight. +::: diff --git a/corpus/round-trip/combinations/mark-closing-delimiter.json b/corpus/round-trip/combinations/mark-closing-delimiter.json new file mode 100644 index 0000000..700f954 --- /dev/null +++ b/corpus/round-trip/combinations/mark-closing-delimiter.json @@ -0,0 +1,56 @@ +{ + "content": [ + { + "content": [ + { + "text": "un", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "a* b", + "type": "text" + }, + { + "text": "istic", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "strike" + } + ], + "text": "a~~ b", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "em" + } + ], + "text": "a_ b", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/mark-closing-delimiter.md b/corpus/round-trip/combinations/mark-closing-delimiter.md new file mode 100644 index 0000000..17a0f36 --- /dev/null +++ b/corpus/round-trip/combinations/mark-closing-delimiter.md @@ -0,0 +1,5 @@ +un*a\* b*istic + +~~a\~~ b~~ + +_a\_ b_ diff --git a/corpus/round-trip/combinations/mark-run-matching.json b/corpus/round-trip/combinations/mark-run-matching.json new file mode 100644 index 0000000..cb35ec6 --- /dev/null +++ b/corpus/round-trip/combinations/mark-run-matching.json @@ -0,0 +1,87 @@ +{ + "content": [ + { + "content": [ + { + "text": "un", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + } + ], + "text": "a", + "type": "text" + }, + { + "marks": [ + { + "type": "em" + }, + { + "type": "strong" + } + ], + "text": "b", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "c", + "type": "text" + }, + { + "text": "istic", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "strong" + }, + { + "type": "em" + } + ], + "text": "a", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + } + ], + "text": "b", + "type": "text" + }, + { + "marks": [ + { + "type": "strong" + }, + { + "type": "em" + } + ], + "text": "c", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/mark-run-matching.md b/corpus/round-trip/combinations/mark-run-matching.md new file mode 100644 index 0000000..5c9aa25 --- /dev/null +++ b/corpus/round-trip/combinations/mark-run-matching.md @@ -0,0 +1,3 @@ +un:adf{json="{\"marks\":[{\"type\":\"em\"}],\"text\":\"a\",\"type\":\"text\"}"}:adf{json="{\"marks\":[{\"type\":\"em\"},{\"type\":\"strong\"}],\"text\":\"b\",\"type\":\"text\"}"}**c**istic + +***a*b**:adf{json="{\"marks\":[{\"type\":\"strong\"},{\"type\":\"em\"}],\"text\":\"c\",\"type\":\"text\"}"} diff --git a/corpus/round-trip/commonmark-subset/text-escaping.md b/corpus/round-trip/commonmark-subset/text-escaping.md index c30602b..c1ecc79 100644 --- a/corpus/round-trip/commonmark-subset/text-escaping.md +++ b/corpus/round-trip/commonmark-subset/text-escaping.md @@ -6,4 +6,4 @@ snake_case_name -\*not emphasis* +\*not emphasis\* diff --git a/spec/flavour.md b/spec/flavour.md index bb1da93..0106017 100644 --- a/spec/flavour.md +++ b/spec/flavour.md @@ -39,9 +39,10 @@ normalizes to it through the round-trip. - Paragraphs on one line β€” no soft wrapping; a soft line break in input becomes a single space. - Entity references in input decode to their characters; output backslash-escapes only where text would otherwise parse as syntax, scanning the assembled line rather than each text node: escape - the leading delimiter of a construct that would otherwise open, re-scan from there, and - repeat β€” with the opener literal the closer parses as text, so `*not emphasis*` is - `\*not emphasis*`, one backslash. + the leading delimiter of a construct that would otherwise open, re-scan from there, and repeat. + An emphasis delimiter run in text escapes where CommonMark can open **or** close with it, so + `*not emphasis*` is `\*not emphasis\*` β€” no delimiter the emitter did not write reaches the + matching below, which is what lets the emitter decide its own pairings. - Blocks separated by one blank line at document level, inside a blockquote and between CommonMark blocks; two directive blocks inside a container take none. No trailing whitespace outside a code block's @@ -390,7 +391,9 @@ An inline node whose marks no nesting spells β€” a mark type not listed here, an spelling does not list, a value that is not the spelling's type, an attribute the spelling needs and the mark lacks, an order putting a code span outside another mark, `code` over anything but a text node or over text holding a newline, or a spelling CommonMark's flanking rules cannot open or -close where the run sits (`un**-real**istic`) β€” rides the inline carry whole. A value the spelling +close where the run sits (`un**-real**istic`), or one CommonMark's matching pairs elsewhere β€” the +intra-word `*` runs together with a neighbouring `**`, and the multiple-of-3 rule can leave the +merged run's pairing to another delimiter β€” rides the inline carry whole. A value the spelling holds but CommonMark cannot write β€” a link destination or title β€” is a named error instead. An opaque carry inside a mark spelling is a named error in input: the carry restores its node exactly, marks included (AGENTS.md Β§3). diff --git a/src/adf-to-markdown.test.ts b/src/adf-to-markdown.test.ts index 21d00ab..1977135 100644 --- a/src/adf-to-markdown.test.ts +++ b/src/adf-to-markdown.test.ts @@ -197,12 +197,12 @@ test('escapes only text that would otherwise open a construct', () => { assert.equal(emitted(':::panel info'), '\\:::panel info\n') assert.equal(emitted('10:30 tomorrow'), '10:30 tomorrow\n') assert.equal(emitted('[a](b)'), '\\[a](b)\n') - assert.equal(emitted('**bold**'), '\\*\\*bold**\n') + assert.equal(emitted('**bold**'), '\\*\\*bold\\*\\*\n') assert.equal(emitted('a `x` b'), 'a \\`x` b\n') - assert.equal(emitted('~~struck~~'), '\\~~struck~~\n') - assert.equal(emitted('a \\* b'), 'a \\\\* b\n') + assert.equal(emitted('~~struck~~'), '\\~~struck\\~~\n') + assert.equal(emitted('a \\* b'), 'a \\\\\\* b\n') assert.equal(emitted('1. not a list'), '1\\. not a list\n') - assert.equal(emitted('*"quoted"*'), '\\*"quoted"*\n') + assert.equal(emitted('*"quoted"*'), '\\*"quoted"\\*\n') assert.equal(emitted('x"_y"'), 'x"\\_y"\n') }) @@ -297,6 +297,38 @@ test('escapes a literal delimiter that would merge with an emitted one', () => { assert.equal(emitted(marked('x', { attrs: { href: 'https://example.com/' }, type: 'link' }), { text: '{}', type: 'text' }), '[x](https://example.com/){}\n') }) +test('escapes a literal delimiter run that flanks either way', () => { + const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) + const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) + assert.equal(emitted({ text: 'un', type: 'text' }, marked('a* b', { type: 'em' }), { text: 'istic', type: 'text' }), 'un*a\\* b*istic\n') + assert.equal(emitted(marked('a~~ b', { type: 'strike' })), '~~a\\~~ b~~\n') + assert.equal(emitted(marked('a_ b', { type: 'em' })), '_a\\_ b_\n') + assert.equal(emitted({ text: 'a* b', type: 'text' }), 'a\\* b\n') + assert.equal(emitted({ text: '2 * 3', type: 'text' }), '2 * 3\n') + assert.equal(emitted({ text: 'snake_case_name', type: 'text' }), 'snake_case_name\n') +}) + +test('carries a mark run CommonMark matching pairs elsewhere', () => { + const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) + const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) + const em: AdfMark = { type: 'em' } + const strong: AdfMark = { type: 'strong' } + assert.equal( + emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', strong), { text: 'istic', type: 'text' }), + 'un:adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"}],\\"text\\":\\"a\\",\\"type\\":\\"text\\"}"}' + + ':adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"},{\\"type\\":\\"strong\\"}],\\"text\\":\\"b\\",\\"type\\":\\"text\\"}"}**c**istic\n', + ) + assert.equal( + emitted(marked('a', strong, em), marked('b', strong), marked('c', strong, em)), + '***a*b**:adf{json="{\\"marks\\":[{\\"type\\":\\"strong\\"},{\\"type\\":\\"em\\"}],\\"text\\":\\"c\\",\\"type\\":\\"text\\"}"}\n', + ) + assert.equal(emitted({ text: 'un', type: 'text' }, marked('a', em, strong), { text: 'istic', type: 'text' }), 'un***a***istic\n') + assert.equal( + emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', em), { text: 'istic', type: 'text' }), + 'un*a**b**c*istic\n', + ) +}) + test('escapes a hyphen underline a hard break would expose', () => { const line = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text: 'foo', type: 'text' }, { type: 'hardBreak' }, { text, type: 'text' })))) assert.equal(line('--'), 'foo\\\n\\--\n') diff --git a/src/corpus.test.ts b/src/corpus.test.ts index 33e87d8..60ff98d 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -66,6 +66,25 @@ for (const directory of emittingDirectories) { } } +// One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it. +test('no two round-trip documents share one spelling', () => { + const documents = new Map() + const spellings = new Map() + for (const directory of emittingDirectories) { + for (const name of fixtureNames(directory, '.md')) { + const fixture = `${directory}/${name}` + const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8')) + assert.ok(isJsonValue(parsed)) + const document = serializeCanonicalJson(parsed, 'compact') + const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8') + assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`) + assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`) + documents.set(document, fixture) + spellings.set(markdown, fixture) + } + } +}) + // spec/flavour.md, Directives: the container fence rule, checked against the emitted bytes. function fenceNestingFault(markdown: string): string | undefined { const open: number[] = [] diff --git a/src/emphasis-matching.ts b/src/emphasis-matching.ts new file mode 100644 index 0000000..2b55059 --- /dev/null +++ b/src/emphasis-matching.ts @@ -0,0 +1,100 @@ +export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number } + +export type EmphasisRun = { + canClose: boolean + canOpen: boolean + character: string + delimiters: readonly EmphasisDelimiter[] + end: number + start: number +} + +type Candidate = { + head: number + next: Candidate | undefined + original: number + previous: Candidate | undefined + remaining: number + run: EmphasisRun + tail: number +} + +// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document. +export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined { + const matched = matchDelimiters(runs) + // The last opener left unpaired is the innermost: the smallest carry that changes the line. + let innermost: number | undefined + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair + } + } + return innermost +} + +function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { + const matched = new Set() + const bottoms = new Map() + let closer = candidates(runs) + while (closer !== undefined) { + if (!closer.run.canClose) { + closer = closer.next + continue + } + const key = `${closer.run.character}${closer.run.canOpen}${closer.original % 3}` + const bottom = bottoms.get(key) + let opener = closer.previous + while (opener !== undefined && opener !== bottom && !pairs(opener, closer)) opener = opener.previous + if (opener === undefined || opener === bottom) { + bottoms.set(key, closer.previous) + const following = closer.next + if (!closer.run.canOpen) unlink(closer) + closer = following + continue + } + const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1 + record(matched, opener, closer, used) + opener.remaining -= used + opener.tail -= used + closer.head += used + closer.remaining -= used + opener.next = closer + closer.previous = opener + if (opener.remaining === 0) unlink(opener) + if (closer.remaining > 0) continue + const following = closer.next + unlink(closer) + closer = following + } + return matched +} + +function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { + let first: Candidate | undefined + let previous: Candidate | undefined + for (const run of runs) { + const length = run.end - run.start + const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end } + if (previous === undefined) first = candidate + else previous.next = candidate + previous = candidate + } + return first +} + +function pairs(opener: Candidate, closer: Candidate): boolean { + if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false + const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0 + return !odd +} + +function record(matched: Set, opener: Candidate, closer: Candidate, used: number): void { + const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail) + const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used) + if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) +} + +function unlink(candidate: Candidate): void { + if (candidate.previous !== undefined) candidate.previous.next = candidate.next + if (candidate.next !== undefined) candidate.next.previous = candidate.previous +} diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index e5e8342..7182e5c 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -1,4 +1,5 @@ import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' +import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts' export type EmphasisRole = 'close' | 'open' @@ -14,7 +15,7 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine export type LineContainer = 'heading' | 'paragraph' | 'table-cell' -type DelimiterRun = { character: string; closeNodes: NodeRange | undefined; end: number; openNodes: NodeRange | undefined; start: number } +type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number } const delimiters = ['*', '_', '`', '~'] @@ -78,17 +79,20 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A } function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined { - for (const run of delimiterRuns(segments, placements)) { - const before = charAt(output, run.start - 1) - const after = output.charAt(run.end) - if (run.openNodes !== undefined && !isLeftFlanking(before, after)) return run.openNodes - if (run.closeNodes !== undefined && !isRightFlanking(before, after)) return run.closeNodes + const { nodes, runs } = emphasisRuns(segments, placements, output) + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair] + } } - return undefined + const unmatched = unmatchedPair(runs) + return unmatched === undefined ? undefined : nodes[unmatched] } -function delimiterRuns(segments: readonly InlineSegment[], placements: readonly number[]): DelimiterRun[] { - const runs: DelimiterRun[] = [] +function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } { + const groups: DelimiterGroup[] = [] + const nodes: NodeRange[] = [] + const open: number[] = [] let cursor = 0 for (const segment of segments) { const start = placements[cursor] ?? 0 @@ -96,22 +100,29 @@ function delimiterRuns(segments: readonly InlineSegment[], placements: readonly if (segment.emphasis === undefined) continue const closes = segment.emphasis === 'close' const end = start + segment.text.length - const previous = runs[runs.length - 1] + const pair = closes ? (open.pop() ?? nodes.length) : nodes.length + if (!closes) { + nodes.push(segment.nodes) + open.push(pair) + } + const delimiter = { closes, end, pair, start } + const previous = groups[groups.length - 1] if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) { - previous.closeNodes = previous.closeNodes ?? (closes ? segment.nodes : undefined) + previous.delimiters.push(delimiter) previous.end = end - previous.openNodes = previous.openNodes ?? (closes ? undefined : segment.nodes) continue } - runs.push({ - character: segment.text.charAt(0), - closeNodes: closes ? segment.nodes : undefined, - end, - openNodes: closes ? undefined : segment.nodes, - start, - }) + groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start }) } - return runs + return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) } +} + +// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. +function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { + const left = isLeftFlanking(before, after) + const right = isRightFlanking(before, after) + if (character !== '_') return { canClose: right, canOpen: left } + return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } } function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean { @@ -177,7 +188,7 @@ function claimsCharacter( if (character === ':') return inlineDirectiveOpener.test(rest) if (character === '[') return opensLink(scan, escapings, index) if (character === '`') return opensCodeSpan(scan, index, escaped) - if (character === '*' || character === '_' || character === '~') return opensEmphasis(scan, index, escaped) + if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped) return false } @@ -196,16 +207,13 @@ function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet return new RegExp('(?): boolean { +function claimsEmphasis(scan: string, index: number, escaped: ReadonlySet): boolean { if (!startsRun(scan, index, escaped)) return false const character = scan.charAt(index) const length = runLength(scan, index) - const before = index === 0 ? '' : scan.charAt(index - 1) - const after = scan.charAt(index + length) - if (character === '~') return length === 2 && isLeftFlanking(before, after) - if (!isLeftFlanking(before, after)) return false - if (character === '*') return true - return !isRightFlanking(before, after) || isPunctuation(before) + if (character === '~' && length !== 2) return false + const flags = delimiterFlags(character, index === 0 ? '' : scan.charAt(index - 1), scan.charAt(index + length)) + return flags.canClose || flags.canOpen } function startsRun(scan: string, index: number, escaped: ReadonlySet): boolean { diff --git a/todo.md b/todo.md index e0ed60a..76b767d 100644 --- a/todo.md +++ b/todo.md @@ -110,14 +110,20 @@ detail is settled at its own milestone. added to that list is the odd one out: `unspellableMark` finds it after assembly and names a mark type against the line's path, so the failing run needs identifying before the carry can replace the refusal `mark-inside-word` pinned. - - [ ] **2e5 β€” Combined documents and the collision property.** Documents combining nodes rather + - [x] **2e5 β€” Combined documents and the collision property.** Documents combining nodes rather than isolating one, and the gate's collision property: no two corpus documents may emit the same bytes β€” one spelling for two documents is a round-trip break no parser can undo, - and it is provable without one. It also settles the emitter's one known approximation: - delimiter flanking is exact, but CommonMark's *matching* β€” the multiple-of-3 rule and the - way a run splits across several openers β€” is not modelled. No reachable violation has been - found by hand; the property test is what decides it, and 2e1's `carve-out-strike` pins a - second backslash only flanking-without-matching emits. + and it is provable without one. + **Settled** (the maintainer, 2026-08-27): the approximation this item inherited β€” flanking + exact, CommonMark's *matching* unmodelled β€” had two round-trip breaks reachable by hand, + so the emitter now models the matching. `process_emphasis` runs over the runs the emitter + wrote (`emphasis-matching.ts`) and a pair it hands to another delimiter rides the carry, + which is what the multiple-of-3 rule did to the em in `un*a**b*****c**istic`. A delimiter + run in text now escapes wherever CommonMark could open or close with it, not only open: + one that could only close stole the spelling around it (`un*a* b*istic`), and escaping + both ways keeps every delimiter the emitter did not write out of the matching. The + canonical form gained a backslash where a run only closes β€” `\*not emphasis\*`, and + 2e1's `carve-out-strike` a third and fourth. - [ ] **2f β€” The attributes CommonMark cannot hold.** 1d's settled answer: the block nodes CommonMark spells β€” `blockquote`, `bulletList`, `codeBlock`, `heading`, `listItem`, `orderedList`, `paragraph`, `rule` β€” get directive sections in `spec/flavour.md` carrying From a8bea25b7c2fc9d27d02ac56fbe1b895cdac07df Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 27 Aug 2026 12:53:15 +0200 Subject: [PATCH 2/3] Give CommonMark's emphasis section one home, and split the collision property --- AGENTS.md | 2 + src/corpus.test.ts | 39 +++++++++----- src/emphasis-matching.ts | 102 +++++++++++++++++++++--------------- src/markdown-escaping.ts | 110 +++++++++++++++++++++------------------ src/markdown-inline.ts | 2 +- todo.md | 10 +++- 6 files changed, 155 insertions(+), 110 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 63e8d3c..aa1e4bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,6 +122,8 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - Emphasis is spelled against CommonMark's matching, never flanking alone: a delimiter run in text escapes wherever CommonMark could open or close with it, leaving the emitter's own delimiters the only ones in play, and a pair that matching hands to another delimiter rides the carry instead. + `matchEmphasis` is a line-for-line transcription of the reference `process_emphasis` and stays one + function: split into named steps it drifts from the algorithm whose fidelity is the whole point. - A readable spelling tried ahead of a general one β€” the image, the pipe table, a pipe cell β€” returns `string | undefined`, never a `Result`: any failure is the fallback signal, and the general form owns the refusal. Refusing there refuses a document the general form spells. diff --git a/src/corpus.test.ts b/src/corpus.test.ts index 60ff98d..c729a68 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -66,22 +66,33 @@ for (const directory of emittingDirectories) { } } +function roundTripFixtures(): { name: string; path: string }[] { + return emittingDirectories.flatMap((directory) => + fixtureNames(directory, '.json').map((name) => ({ name: `${directory}/${name}`, path: join(roundTripRoot, directory, `${name}.json`) })), + ) +} + // One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it. -test('no two round-trip documents share one spelling', () => { - const documents = new Map() +test('no two round-trip documents share one markdown spelling', () => { const spellings = new Map() - for (const directory of emittingDirectories) { - for (const name of fixtureNames(directory, '.md')) { - const fixture = `${directory}/${name}` - const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8')) - assert.ok(isJsonValue(parsed)) - const document = serializeCanonicalJson(parsed, 'compact') - const markdown = readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8') - assert.equal(documents.get(document), undefined, `${fixture} repeats the document ${documents.get(document)} holds`) - assert.equal(spellings.get(markdown), undefined, `${fixture} and ${spellings.get(markdown)} share one markdown spelling`) - documents.set(document, fixture) - spellings.set(markdown, fixture) - } + for (const fixture of roundTripFixtures()) { + const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8')) + assert.ok(isAdfDocument(parsed), `${fixture.name} is not an ADF document`) + const result = adfToMarkdown(parsed) + assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`) + assert.equal(spellings.get(result.value), undefined, `${fixture.name} and ${spellings.get(result.value)} share one markdown spelling`) + spellings.set(result.value, fixture.name) + } +}) + +test('no round-trip fixture repeats the document another holds', () => { + const documents = new Map() + for (const fixture of roundTripFixtures()) { + const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8')) + assert.ok(isJsonValue(parsed)) + const document = serializeCanonicalJson(parsed, 'compact') + assert.equal(documents.get(document), undefined, `${fixture.name} repeats the document ${documents.get(document)} holds`) + documents.set(document, fixture.name) } }) diff --git a/src/emphasis-matching.ts b/src/emphasis-matching.ts index 2b55059..8792ce0 100644 --- a/src/emphasis-matching.ts +++ b/src/emphasis-matching.ts @@ -1,40 +1,37 @@ -export type EmphasisDelimiter = { closes: boolean; end: number; pair: number; start: number } +import { isUnicodeWhitespace } from './commonmark-grammar.ts' -export type EmphasisRun = { - canClose: boolean - canOpen: boolean - character: string - delimiters: readonly EmphasisDelimiter[] - end: number - start: number -} +export type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } -type Candidate = { +export type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } + +type Candidate = { head: number - next: Candidate | undefined + next: Candidate | undefined original: number - previous: Candidate | undefined + previous: Candidate | undefined remaining: number - run: EmphasisRun + run: Run tail: number } -// Flanking decides which delimiters may pair; matching decides which do, and a pair it leaves unpaired reads back as another document. -export function unmatchedPair(runs: readonly EmphasisRun[]): number | undefined { - const matched = matchDelimiters(runs) - // The last opener left unpaired is the innermost: the smallest carry that changes the line. - let innermost: number | undefined - for (const run of runs) { - for (const delimiter of run.delimiters) { - if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair - } - } - return innermost +const unicodePunctuation = /[\p{P}\p{S}]/u + +// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. +export function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { + const left = isLeftFlanking(before, after) + const right = isRightFlanking(before, after) + if (character !== '_') return { canClose: right, canOpen: left } + return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } } -function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { - const matched = new Set() - const bottoms = new Map() +export function isWordCharacter(character: string): boolean { + return character !== '' && !isWhitespace(character) && !isPunctuation(character) +} + +// Flanking decides which delimiters may pair; this decides which ones do, and a pair it leaves out reads back as another document. +export function matchEmphasis(runs: readonly Run[]): EmphasisPairing[] { + const pairings: EmphasisPairing[] = [] + const bottoms = new Map | undefined>() let closer = candidates(runs) while (closer !== undefined) { if (!closer.run.canClose) { @@ -53,9 +50,9 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { continue } const used = closer.remaining >= 2 && opener.remaining >= 2 ? 2 : 1 - record(matched, opener, closer, used) opener.remaining -= used opener.tail -= used + pairings.push({ closer: closer.run, closerOffset: closer.head, opener: opener.run, openerOffset: opener.tail, used }) closer.head += used closer.remaining -= used opener.next = closer @@ -66,15 +63,22 @@ function matchDelimiters(runs: readonly EmphasisRun[]): ReadonlySet { unlink(closer) closer = following } - return matched + return pairings } -function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { - let first: Candidate | undefined - let previous: Candidate | undefined +function candidates(runs: readonly Run[]): Candidate | undefined { + let first: Candidate | undefined + let previous: Candidate | undefined for (const run of runs) { - const length = run.end - run.start - const candidate: Candidate = { head: run.start, next: undefined, original: length, previous, remaining: length, run, tail: run.end } + const candidate: Candidate = { + head: 0, + next: undefined, + original: run.length, + previous, + remaining: run.length, + run, + tail: run.length, + } if (previous === undefined) first = candidate else previous.next = candidate previous = candidate @@ -82,19 +86,33 @@ function candidates(runs: readonly EmphasisRun[]): Candidate | undefined { return first } -function pairs(opener: Candidate, closer: Candidate): boolean { +function pairs(opener: Candidate, closer: Candidate): boolean { if (!opener.run.canOpen || opener.run.character !== closer.run.character) return false const odd = (closer.run.canOpen || opener.run.canClose) && closer.original % 3 !== 0 && (opener.original + closer.original) % 3 === 0 return !odd } -function record(matched: Set, opener: Candidate, closer: Candidate, used: number): void { - const opened = opener.run.delimiters.find((delimiter) => !delimiter.closes && delimiter.start === opener.tail - used && delimiter.end === opener.tail) - const closed = closer.run.delimiters.find((delimiter) => delimiter.closes && delimiter.start === closer.head && delimiter.end === closer.head + used) - if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) -} - -function unlink(candidate: Candidate): void { +function unlink(candidate: Candidate): void { if (candidate.previous !== undefined) candidate.previous.next = candidate.next if (candidate.next !== undefined) candidate.next.previous = candidate.previous } + +function isLeftFlanking(before: string, after: string): boolean { + if (isWhitespace(after)) return false + if (!isPunctuation(after)) return true + return isWhitespace(before) || isPunctuation(before) +} + +function isRightFlanking(before: string, after: string): boolean { + if (isWhitespace(before)) return false + if (!isPunctuation(before)) return true + return isWhitespace(after) || isPunctuation(after) +} + +function isPunctuation(character: string): boolean { + return character !== '' && unicodePunctuation.test(character) +} + +function isWhitespace(character: string): boolean { + return character === '' || isUnicodeWhitespace(character) +} diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index 7182e5c..e3cb889 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -1,5 +1,5 @@ -import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' -import { unmatchedPair, type EmphasisDelimiter, type EmphasisRun } from './emphasis-matching.ts' +import { escapesLineClaim, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts' +import { delimiterFlags, isWordCharacter, matchEmphasis } from './emphasis-matching.ts' export type EmphasisRole = 'close' | 'open' @@ -15,7 +15,9 @@ export type AssembledLine = { line: string; unspellableRun: NodeRange | undefine export type LineContainer = 'heading' | 'paragraph' | 'table-cell' -type DelimiterGroup = { character: string; delimiters: EmphasisDelimiter[]; end: number; start: number } +type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number } + +type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; delimiters: EmittedDelimiter[]; length: number; start: number } const delimiters = ['*', '_', '`', '~'] @@ -23,7 +25,6 @@ const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/ const followsLinkText = /[([:]/ -const unicodePunctuation = /[\p{P}\p{S}]/u export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { return escape(resolveEmphasis(segments), container) @@ -79,18 +80,43 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A } function unspellableRun(segments: readonly InlineSegment[], output: string, placements: readonly number[]): NodeRange | undefined { - const { nodes, runs } = emphasisRuns(segments, placements, output) - for (const run of runs) { - for (const delimiter of run.delimiters) { - if (!(delimiter.closes ? run.canClose : run.canOpen)) return nodes[delimiter.pair] - } - } - const unmatched = unmatchedPair(runs) - return unmatched === undefined ? undefined : nodes[unmatched] + const { nodes, runs } = emittedRuns(segments, placements, output) + const pair = misflanked(runs) ?? unpaired(runs) + return pair === undefined ? undefined : nodes[pair] } -function emphasisRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmphasisRun[] } { - const groups: DelimiterGroup[] = [] +function misflanked(runs: readonly EmittedRun[]): number | undefined { + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!(delimiter.closes ? run.canClose : run.canOpen)) return delimiter.pair + } + } + return undefined +} + +function unpaired(runs: readonly EmittedRun[]): number | undefined { + const matched = new Set() + for (const pairing of matchEmphasis(runs)) { + const opened = delimiterAt(pairing.opener, false, pairing.openerOffset, pairing.used) + const closed = delimiterAt(pairing.closer, true, pairing.closerOffset, pairing.used) + if (opened !== undefined && closed !== undefined && opened.pair === closed.pair) matched.add(opened.pair) + } + // The last opener left unpaired is the innermost: the smallest carry that changes the line. + let innermost: number | undefined + for (const run of runs) { + for (const delimiter of run.delimiters) { + if (!delimiter.closes && !matched.has(delimiter.pair)) innermost = delimiter.pair + } + } + return innermost +} + +function delimiterAt(run: EmittedRun, closes: boolean, offset: number, width: number): EmittedDelimiter | undefined { + return run.delimiters.find((delimiter) => delimiter.closes === closes && delimiter.offset === offset && delimiter.width === width) +} + +function emittedRuns(segments: readonly InlineSegment[], placements: readonly number[], output: string): { nodes: NodeRange[]; runs: EmittedRun[] } { + const runs: EmittedRun[] = [] const nodes: NodeRange[] = [] const open: number[] = [] let cursor = 0 @@ -99,30 +125,33 @@ function emphasisRuns(segments: readonly InlineSegment[], placements: readonly n cursor += segment.text.length if (segment.emphasis === undefined) continue const closes = segment.emphasis === 'close' - const end = start + segment.text.length const pair = closes ? (open.pop() ?? nodes.length) : nodes.length if (!closes) { nodes.push(segment.nodes) open.push(pair) } - const delimiter = { closes, end, pair, start } - const previous = groups[groups.length - 1] - if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) { - previous.delimiters.push(delimiter) - previous.end = end + const width = segment.text.length + const previous = runs[runs.length - 1] + if (previous !== undefined && previous.start + previous.length === start && previous.character === segment.text.charAt(0)) { + previous.delimiters.push({ closes, offset: start - previous.start, pair, width }) + previous.length += width continue } - groups.push({ character: segment.text.charAt(0), delimiters: [delimiter], end, start }) + runs.push({ + canClose: false, + canOpen: false, + character: segment.text.charAt(0), + delimiters: [{ closes, offset: 0, pair, width }], + length: width, + start, + }) } - return { nodes, runs: groups.map((group) => ({ ...group, ...delimiterFlags(group.character, charAt(output, group.start - 1), output.charAt(group.end)) })) } -} - -// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. -function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { - const left = isLeftFlanking(before, after) - const right = isRightFlanking(before, after) - if (character !== '_') return { canClose: right, canOpen: left } - return { canClose: right && (!left || isPunctuation(after)), canOpen: left && (!right || isPunctuation(before)) } + for (const run of runs) { + const flags = delimiterFlags(run.character, charAt(output, run.start - 1), output.charAt(run.start + run.length)) + run.canClose = flags.canClose + run.canOpen = flags.canOpen + } + return { nodes, runs } } function mergesWithSyntax(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean { @@ -228,29 +257,8 @@ function runLength(scan: string, index: number): number { return length } -function isLeftFlanking(before: string, after: string): boolean { - if (isWhitespace(after)) return false - if (!isPunctuation(after)) return true - return isWhitespace(before) || isPunctuation(before) -} -function isRightFlanking(before: string, after: string): boolean { - if (isWhitespace(before)) return false - if (!isPunctuation(before)) return true - return isWhitespace(after) || isPunctuation(after) -} -function isPunctuation(character: string): boolean { - return character !== '' && unicodePunctuation.test(character) -} - -function isWhitespace(character: string): boolean { - return character === '' || isUnicodeWhitespace(character) -} - -function isWordCharacter(character: string): boolean { - return character !== '' && !isUnicodeWhitespace(character) && !unicodePunctuation.test(character) -} function charAt(text: string, index: number): string { return index < 0 ? '' : text.charAt(index) diff --git a/src/markdown-inline.ts b/src/markdown-inline.ts index ae4f211..d6b2fa6 100644 --- a/src/markdown-inline.ts +++ b/src/markdown-inline.ts @@ -48,7 +48,7 @@ export function tryImageLine(alt: string | undefined, href: string, path: Conver return attempt.ok ? attempt.value.line : undefined } -// A demand names a run no spelling holds, and a carried node joins no run, so every pass carries at least one more node. +// A carried node joins no run, so every pass carries at least one more node. function emitLine(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath): Result { const carried = new Set() for (;;) { diff --git a/todo.md b/todo.md index 76b767d..9039eb1 100644 --- a/todo.md +++ b/todo.md @@ -162,7 +162,10 @@ detail is settled at its own milestone. stay above all of it β€” the vocabulary a string-typed attribute grammar needs, which is why HTML will want them too, not a markdown spelling. Both node tables are a second copy of `spec/flavour.md`'s prose with no drift guard, and a mistyped attribute name degrades into a - false refusal no test catches. + false refusal no test catches. The parser reuses `emphasis-matching.ts` whole and lands it + beside the grammar module: `delimiterFlags` and `matchEmphasis` take CommonMark's own run + vocabulary rather than the emitter's, so no second `process_emphasis` exists to drift from + the first. - [ ] **4 β€” Round-trip property tests** over the corpus, both ways β€” the thing that proves 2 and 3. Editor-normal (Β§2) gets its implementation here β€” `toEditorNormal(doc)` and the equality the round-trip asserts, which over normalized input is the canonical serializer's compact @@ -171,7 +174,10 @@ detail is settled at its own milestone. lifts the branch floor Β§10 keeps below 100 for exactly those halves. Generators emit editor-normal ADF (Β§2). Real sanitized ADF from live Atlassian APIs lands here too (Β§10), in `corpus/real-payloads/`: an ADFβ†’markdownβ†’ADF check with no expected - markdown, the payloads supplied by the maintainer. + markdown, the payloads supplied by the maintainer. This subsumes 2e5's collision property β€” + a document that round-trips proves no other document shares its spelling β€” so decide here + whether that gate stays as the parser-free, faster-failing signal or goes; the half holding + no fixture duplicates is hygiene rather than a round-trip claim, and stays either way. - [ ] **5 β€” Release pipeline, ship `0.1.0`.** Publish-on-version-change (Β§9), `NPM_TOKEN` secret, the repo made public first (Β§6). The `ConvertErrorCode` freeze (Β§8) is checkable here: every `corpus/unspellable/` document is a decision or a deferred trigger this file names, so the From 1108d36d1f237e38d21e81a67e1ea8a9e5416716 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 27 Aug 2026 13:17:08 +0200 Subject: [PATCH 3/3] Answer the review: one home per contract, and cover the matching's last branch --- AGENTS.md | 5 +- .../combinations/mark-run-matching.json | 62 +++++++++++++++++++ .../combinations/mark-run-matching.md | 2 + src/adf-to-markdown.test.ts | 25 ++------ src/corpus.test.ts | 3 +- src/emphasis-matching.ts | 6 +- src/markdown-escaping.ts | 3 - 7 files changed, 76 insertions(+), 30 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index aa1e4bb..08a92e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,8 +122,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga - Emphasis is spelled against CommonMark's matching, never flanking alone: a delimiter run in text escapes wherever CommonMark could open or close with it, leaving the emitter's own delimiters the only ones in play, and a pair that matching hands to another delimiter rides the carry instead. - `matchEmphasis` is a line-for-line transcription of the reference `process_emphasis` and stays one - function: split into named steps it drifts from the algorithm whose fidelity is the whole point. + `matchEmphasis` transcribes the reference `process_emphasis` line for line, and its closer walk and + opener search stay whole: broken into named steps they drift from the algorithm being faithful is + the whole point of. - A readable spelling tried ahead of a general one β€” the image, the pipe table, a pipe cell β€” returns `string | undefined`, never a `Result`: any failure is the fallback signal, and the general form owns the refusal. Refusing there refuses a document the general form spells. diff --git a/corpus/round-trip/combinations/mark-run-matching.json b/corpus/round-trip/combinations/mark-run-matching.json index cb35ec6..179aece 100644 --- a/corpus/round-trip/combinations/mark-run-matching.json +++ b/corpus/round-trip/combinations/mark-run-matching.json @@ -80,6 +80,68 @@ } ], "type": "paragraph" + }, + { + "content": [ + { + "marks": [ + { + "type": "strike" + } + ], + "text": "un", + "type": "text" + }, + { + "marks": [ + { + "type": "strike" + }, + { + "type": "em" + } + ], + "text": "a", + "type": "text" + }, + { + "marks": [ + { + "type": "strike" + }, + { + "type": "em" + }, + { + "type": "strong" + } + ], + "text": "b", + "type": "text" + }, + { + "marks": [ + { + "type": "strike" + }, + { + "type": "strong" + } + ], + "text": "c", + "type": "text" + }, + { + "marks": [ + { + "type": "strike" + } + ], + "text": "istic", + "type": "text" + } + ], + "type": "paragraph" } ], "type": "doc", diff --git a/corpus/round-trip/combinations/mark-run-matching.md b/corpus/round-trip/combinations/mark-run-matching.md index 5c9aa25..bc30a65 100644 --- a/corpus/round-trip/combinations/mark-run-matching.md +++ b/corpus/round-trip/combinations/mark-run-matching.md @@ -1,3 +1,5 @@ un:adf{json="{\"marks\":[{\"type\":\"em\"}],\"text\":\"a\",\"type\":\"text\"}"}:adf{json="{\"marks\":[{\"type\":\"em\"},{\"type\":\"strong\"}],\"text\":\"b\",\"type\":\"text\"}"}**c**istic ***a*b**:adf{json="{\"marks\":[{\"type\":\"strong\"},{\"type\":\"em\"}],\"text\":\"c\",\"type\":\"text\"}"} + +~~un~~:adf{json="{\"marks\":[{\"type\":\"strike\"},{\"type\":\"em\"}],\"text\":\"a\",\"type\":\"text\"}"}:adf{json="{\"marks\":[{\"type\":\"strike\"},{\"type\":\"em\"},{\"type\":\"strong\"}],\"text\":\"b\",\"type\":\"text\"}"}~~**c**istic~~ diff --git a/src/adf-to-markdown.test.ts b/src/adf-to-markdown.test.ts index 1977135..6a7f150 100644 --- a/src/adf-to-markdown.test.ts +++ b/src/adf-to-markdown.test.ts @@ -297,32 +297,19 @@ test('escapes a literal delimiter that would merge with an emitted one', () => { assert.equal(emitted(marked('x', { attrs: { href: 'https://example.com/' }, type: 'link' }), { text: '{}', type: 'text' }), '[x](https://example.com/){}\n') }) -test('escapes a literal delimiter run that flanks either way', () => { - const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) - const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) - assert.equal(emitted({ text: 'un', type: 'text' }, marked('a* b', { type: 'em' }), { text: 'istic', type: 'text' }), 'un*a\\* b*istic\n') - assert.equal(emitted(marked('a~~ b', { type: 'strike' })), '~~a\\~~ b~~\n') - assert.equal(emitted(marked('a_ b', { type: 'em' })), '_a\\_ b_\n') - assert.equal(emitted({ text: 'a* b', type: 'text' }), 'a\\* b\n') - assert.equal(emitted({ text: '2 * 3', type: 'text' }), '2 * 3\n') - assert.equal(emitted({ text: 'snake_case_name', type: 'text' }), 'snake_case_name\n') +test('escapes a literal delimiter run that only closes', () => { + const emitted = (text: string): string => markdown(adfToMarkdown(document(paragraph({ text, type: 'text' })))) + assert.equal(emitted('a* b'), 'a\\* b\n') + assert.equal(emitted('2 * 3'), '2 * 3\n') }) -test('carries a mark run CommonMark matching pairs elsewhere', () => { +test("spells a mark run CommonMark's matching pairs as written", () => { const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' }) const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content)))) const em: AdfMark = { type: 'em' } const strong: AdfMark = { type: 'strong' } - assert.equal( - emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', strong), { text: 'istic', type: 'text' }), - 'un:adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"}],\\"text\\":\\"a\\",\\"type\\":\\"text\\"}"}' + - ':adf{json="{\\"marks\\":[{\\"type\\":\\"em\\"},{\\"type\\":\\"strong\\"}],\\"text\\":\\"b\\",\\"type\\":\\"text\\"}"}**c**istic\n', - ) - assert.equal( - emitted(marked('a', strong, em), marked('b', strong), marked('c', strong, em)), - '***a*b**:adf{json="{\\"marks\\":[{\\"type\\":\\"strong\\"},{\\"type\\":\\"em\\"}],\\"text\\":\\"c\\",\\"type\\":\\"text\\"}"}\n', - ) assert.equal(emitted({ text: 'un', type: 'text' }, marked('a', em, strong), { text: 'istic', type: 'text' }), 'un***a***istic\n') + assert.equal(emitted({ text: 're', type: 'text' }, marked('structure', strong), { text: ' the code', type: 'text' }), 're**structure** the code\n') assert.equal( emitted({ text: 'un', type: 'text' }, marked('a', em), marked('b', em, strong), marked('c', em), { text: 'istic', type: 'text' }), 'un*a**b**c*istic\n', diff --git a/src/corpus.test.ts b/src/corpus.test.ts index c729a68..64f4251 100644 --- a/src/corpus.test.ts +++ b/src/corpus.test.ts @@ -72,7 +72,6 @@ function roundTripFixtures(): { name: string; path: string }[] { ) } -// One spelling for two documents is a round-trip break no parser can undo, and no parser is needed to see it. test('no two round-trip documents share one markdown spelling', () => { const spellings = new Map() for (const fixture of roundTripFixtures()) { @@ -89,7 +88,7 @@ test('no round-trip fixture repeats the document another holds', () => { const documents = new Map() for (const fixture of roundTripFixtures()) { const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8')) - assert.ok(isJsonValue(parsed)) + assert.ok(isJsonValue(parsed), `${fixture.name} does not hold a JSON value`) const document = serializeCanonicalJson(parsed, 'compact') assert.equal(documents.get(document), undefined, `${fixture.name} repeats the document ${documents.get(document)} holds`) documents.set(document, fixture.name) diff --git a/src/emphasis-matching.ts b/src/emphasis-matching.ts index 8792ce0..a5897d0 100644 --- a/src/emphasis-matching.ts +++ b/src/emphasis-matching.ts @@ -1,8 +1,8 @@ import { isUnicodeWhitespace } from './commonmark-grammar.ts' -export type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } +type DelimiterRun = { canClose: boolean; canOpen: boolean; character: string; length: number } -export type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } +type EmphasisPairing = { closer: Run; closerOffset: number; opener: Run; openerOffset: number; used: number } type Candidate = { head: number @@ -16,7 +16,6 @@ type Candidate = { const unicodePunctuation = /[\p{P}\p{S}]/u -// spec/flavour.md, Canonical form: CommonMark's own can-open and can-close, which `~` follows too. export function delimiterFlags(character: string, before: string, after: string): { canClose: boolean; canOpen: boolean } { const left = isLeftFlanking(before, after) const right = isRightFlanking(before, after) @@ -28,7 +27,6 @@ export function isWordCharacter(character: string): boolean { return character !== '' && !isWhitespace(character) && !isPunctuation(character) } -// Flanking decides which delimiters may pair; this decides which ones do, and a pair it leaves out reads back as another document. export function matchEmphasis(runs: readonly Run[]): EmphasisPairing[] { const pairings: EmphasisPairing[] = [] const bottoms = new Map | undefined>() diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index e3cb889..40ad3ec 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -257,9 +257,6 @@ function runLength(scan: string, index: number): number { return length } - - - function charAt(text: string, index: number): string { return index < 0 ? '' : text.charAt(index) }