diff --git a/corpus/round-trip/block-nodes/image-boundary.md b/corpus/round-trip/block-nodes/image-boundary.md index 0672446..5b2963f 100644 --- a/corpus/round-trip/block-nodes/image-boundary.md +++ b/corpus/round-trip/block-nodes/image-boundary.md @@ -22,5 +22,5 @@ Taken from Kiruna. ::: :::mediaSingle {layout=center} -::media {type=external url="https://example.com/plan.png?a=1&b=2"} +::media {type=external url="https://example.com/plan.png?a=1\u0026amp;b=2"} ::: diff --git a/corpus/round-trip/combinations/attribute-canonical.json b/corpus/round-trip/combinations/attribute-canonical.json new file mode 100644 index 0000000..32fc6f1 --- /dev/null +++ b/corpus/round-trip/combinations/attribute-canonical.json @@ -0,0 +1,60 @@ +{ + "content": [ + { + "attrs": { + "isNumberColumnEnabled": true, + "layout": "wide", + "width": 760.5 + }, + "content": [ + { + "content": [ + { + "attrs": { + "colspan": 2, + "colwidth": [ + 340, + 420 + ] + }, + "content": [ + { + "content": [ + { + "text": "Assembly", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "tableHeader" + } + ], + "type": "tableRow" + } + ], + "type": "table" + }, + { + "attrs": { + "localId": "01a0-33_z", + "title": "a\nb\tc\"d\\e" + }, + "content": [ + { + "content": [ + { + "text": "Notes.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "expand" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/round-trip/combinations/attribute-canonical.md b/corpus/round-trip/combinations/attribute-canonical.md new file mode 100644 index 0000000..d49fbbe --- /dev/null +++ b/corpus/round-trip/combinations/attribute-canonical.md @@ -0,0 +1,11 @@ +:::::table {isNumberColumnEnabled=true layout=wide width="760.5"} +::::tableRow +:::tableHeader {colspan=2 colwidth="[340,420]"} +Assembly +::: +:::: +::::: + +:::expand {localId=01a0-33_z title="a\nb\tc\"d\\e"} +Notes. +::: diff --git a/corpus/round-trip/combinations/attribute-escapes.json b/corpus/round-trip/combinations/attribute-escapes.json new file mode 100644 index 0000000..3ec83d2 --- /dev/null +++ b/corpus/round-trip/combinations/attribute-escapes.json @@ -0,0 +1,56 @@ +{ + "content": [ + { + "content": [ + { + "text": "a `b ", + "type": "text" + }, + { + "attrs": { + "url": "c`d" + }, + "type": "inlineCard" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "attrs": { + "url": "https://example.com/s?q=a&b { assert.equal(code(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, marked('~a', { type: 'strike' }))))), 'unspellable-mark') assert.equal(emitted({ text: '`', type: 'text' }, marked('x', { type: 'code' })), '\\``x`\n') assert.equal(emitted(marked('x', { type: 'code' }), { text: '`', type: 'text' }), '`x`\\`\n') + assert.equal(emitted({ text: '`a', type: 'text' }, marked('b', { type: 'code' })), '\\`a`b`\n') + assert.equal( + emitted({ text: '' }, type: 'inlineCard' }), + '\\ table(row(cell('tableHeader', { content: [{ marks: [mark], text: 'l', type: 'text' }], type: 'paragraph' }))) assert.ok(directive(adfToMarkdown(marked({ attrs: { href: 'https://example.com/?x|y' }, type: 'link' })))) assert.ok(directive(adfToMarkdown(marked({ attrs: { href: 'https://example.com/', title: 'a|b' }, type: 'link' })))) - const fallsBack = (node: AdfNode): boolean => directive(adfToMarkdown(table(row(cell('tableHeader', { content: [node], type: 'paragraph' }))))) - assert.ok(fallsBack({ marks: [{ type: 'code' }], text: 'a|b', type: 'text' })) - assert.ok(fallsBack({ attrs: { style: 'a|b' }, type: 'status' })) - assert.ok(fallsBack({ attrs: { url: 'https://example.com/?a|b' }, type: 'blockCard' })) + const codeSpan: AdfNode = { marks: [{ type: 'code' }], text: 'a|b', type: 'text' } + assert.ok(directive(adfToMarkdown(table(row(cell('tableHeader', { content: [codeSpan], type: 'paragraph' })))))) assert.equal(markdown(adfToMarkdown(marked({ attrs: { href: 'https://example.com/x' }, type: 'link' }))), '| [l](https://example.com/x) |\n| --- |\n') }) @@ -427,6 +432,7 @@ test('spells an inline node as a directive with its content slot and attributes' assert.equal(emitted({ attrs: { id: '1f389', text: 'a]b' }, type: 'emoji' }), ':emoji[a\\]b]{id=1f389}\n') assert.equal(emitted({ attrs: { data: { url: 'https://example.com/' } }, type: 'inlineCard' }), ':inlineCard{data="{\\"url\\":\\"https://example.com/\\"}"}\n') assert.equal(emitted({ attrs: { height: 24 }, type: 'mediaInline' }), ':mediaInline{height=24}\n') + assert.equal(emitted({ attrs: { url: 'a`b&c { diff --git a/src/directive-attributes.ts b/src/directive-attributes.ts index c4d8d1b..88992c2 100644 --- a/src/directive-attributes.ts +++ b/src/directive-attributes.ts @@ -9,6 +9,9 @@ export type SpelledPairs = { fault: AttributeFault; pairs?: undefined } | { faul const bareToken = /^[A-Za-z0-9_-]+$/ +// spec/flavour.md, Attributes. +const quotedEscapes = /[&<`|]/g + export function isBareToken(text: string): boolean { return bareToken.test(text) } @@ -53,5 +56,5 @@ export function spellStringAttribute(text: string): string { } function quote(text: string): string { - return JSON.stringify(text) + return JSON.stringify(text).replace(quotedEscapes, (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`) } diff --git a/src/markdown-escaping.ts b/src/markdown-escaping.ts index ff0820a..21cf506 100644 --- a/src/markdown-escaping.ts +++ b/src/markdown-escaping.ts @@ -2,7 +2,7 @@ import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEn export type EmphasisRole = 'close' | 'open' -export type InlineEscaping = 'attribute' | 'backslash' | 'bracketed' | 'none' +export type InlineEscaping = 'backslash' | 'bracketed' | 'none' export type InlineSegment = | { emphasis: EmphasisRole; escaping: 'none'; mark: string; text: string } @@ -19,7 +19,7 @@ const delimiters = ['*', '_', '`', '~'] const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/ -const linkOpener = /\](?=[([:])/ +const followsLinkText = /[([:]/ const unicodePunctuation = /[\p{P}\p{S}]/u export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): AssembledLine { @@ -65,7 +65,7 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A for (let index = 0; index < scan.length; index += 1) { const escaping = escapings[index] const escapable = escaping === 'backslash' || escaping === 'bracketed' - if (escapable && (mergesWithSyntax(scan, escapings, index) || opensConstruct(scan, index, escaping === 'bracketed', container, escaped))) { + if (escapable && (mergesWithSyntax(scan, escapings, index) || opensConstruct(scan, escapings, index, escaping === 'bracketed', container, escaped))) { output += '\\' escaped.add(index) } @@ -128,13 +128,20 @@ function touchesSyntax(scan: string, escapings: readonly (InlineEscaping | undef } function isSyntax(escaping: InlineEscaping | undefined): boolean { - return escaping === 'attribute' || escaping === 'none' + return escaping === 'none' } -function opensConstruct(scan: string, index: number, inBrackets: boolean, container: LineContainer, escaped: ReadonlySet): boolean { +function opensConstruct( + scan: string, + escapings: readonly (InlineEscaping | undefined)[], + index: number, + inBrackets: boolean, + container: LineContainer, + escaped: ReadonlySet, +): boolean { if (container === 'heading' && closesHeading(scan, index)) return true if (container === 'paragraph' && claimsLineStart(scan, index)) return true - return claimsCharacter(scan, index, inBrackets, container, escaped) + return claimsCharacter(scan, escapings, index, inBrackets, container, escaped) } function claimsLineStart(scan: string, index: number): boolean { @@ -150,7 +157,14 @@ function closesHeading(scan: string, index: number): boolean { return index === 0 || /[ \t]/.test(scan.charAt(index - 1)) } -function claimsCharacter(scan: string, index: number, inBrackets: boolean, container: LineContainer, escaped: ReadonlySet): boolean { +function claimsCharacter( + scan: string, + escapings: readonly (InlineEscaping | undefined)[], + index: number, + inBrackets: boolean, + container: LineContainer, + escaped: ReadonlySet, +): boolean { const character = scan.charAt(index) const rest = scan.slice(index) if (inBrackets && (character === '[' || character === ']')) return true @@ -159,12 +173,21 @@ function claimsCharacter(scan: string, index: number, inBrackets: boolean, conta if (character === '&') return startsEntityReference(rest) if (character === '<') return opensBracketedAutolink(rest) || htmlConstructs.some((construct) => construct.test(rest)) if (character === ':') return inlineDirectiveOpener.test(rest) - if (character === '[') return linkOpener.test(rest) + if (character === '[') return opensLink(scan, escapings, index) if (character === '`') return opensCodeSpan(scan, index, escaped) if (character === '*' || character === '_' || character === '~') return opensEmphasis(scan, index, escaped) return false } +// A `]` the emitter spelled sits inside a construct that binds before link text does. +function opensLink(scan: string, escapings: readonly (InlineEscaping | undefined)[], index: number): boolean { + for (let cursor = index + 1; cursor < scan.length; cursor += 1) { + if (scan.charAt(cursor) !== ']' || isSyntax(escapings[cursor])) continue + if (followsLinkText.test(scan.charAt(cursor + 1))) return true + } + return false +} + function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet): boolean { if (!startsRun(scan, index, escaped)) return false const length = runLength(scan, index) diff --git a/src/markdown-inline.ts b/src/markdown-inline.ts index 78f2256..6065e71 100644 --- a/src/markdown-inline.ts +++ b/src/markdown-inline.ts @@ -92,7 +92,7 @@ function carryEdges(segment: InlineSegment, leading: boolean, trailing: boolean) } function carriedText(text: string): InlineSegment { - return { escaping: 'attribute', text: spellLeafDirective('text', spellAttributes([['text', spellStringAttribute(text)]])) } + return syntax(spellLeafDirective('text', spellAttributes([['text', spellStringAttribute(text)]]))) } function spellLeafDirective(name: string, attributes: string): string { diff --git a/todo.md b/todo.md index 7e74443..52b09a5 100644 --- a/todo.md +++ b/todo.md @@ -72,12 +72,11 @@ detail is settled at its own milestone. the emitter's refusals a corpus home while the directories grow: `corpus/unspellable/`, a `.json` beside the `ConvertErrorCode` it must return, the emitter half of `corpus/errors/`. - [x] **2c — Inline nodes and marks.** `inline-nodes/` green. `InlineSegment` splits into its - two axes — escapability (`attribute` for `:text{text="…"}`, `backslash`, `bracketed`, - `none`) and the emphasis role. A lone surrogate in a text node emits verbatim and becomes - U+FFFD on any UTF-8 encode, a §2 break plain text still holds open — attribute values - already escape it. The pipe form's fallback reads the emitted segments rather than naming - the nodes whose attribute values spell a pipe as syntax, so 2e's `\u007c` narrows it in - one place. + two axes — escapability (`backslash`, `bracketed`, `none`) and the emphasis role. A lone + surrogate in a text node emits verbatim and becomes U+FFFD on any UTF-8 encode, a §2 break + plain text still holds open — attribute values already escape it. The pipe form's fallback + reads the emitted segments rather than naming the nodes whose attribute values spell a pipe + as syntax, so 2e's `\u007c` narrows it in one place. - [x] **2d — The opaque carry** (§3). Fixtures and emitter together, into `corpus/round-trip/opaque-carry/`: an unknown node in both positions, the reserved `adf` info string, and the `codeBlock` whose language is `adf` — carried whole ahead of the @@ -96,24 +95,21 @@ detail is settled at its own milestone. `inline-directives.ts`, which holds four of the nine marks while the rest are branch literals in the emitter — and the parser (3) needs every name to make `:em[x]` the named error `spec/flavour.md` promises. - - [ ] **2e3 — Attribute canonicalization and the quoted value's escape.** Attribute - canonicalization, a pipe cell's whitespace edges, and `\u007c` for a `|` inside a quoted - attribute value. The rule is wider than the pipe: a quoted value spells `` ` ``, `&` and - `<` raw today, so a backtick pair in an `inlineCard`'s `data`, a `status`'s `style` or the - inline carry's `json` reads back as a code span, an entity or raw HTML inside the value. - One escape settles all four, and the pick decides whether a directive or a code span wins - where they overlap (3). `escaping: 'attribute'` earns its keep at that rule or collapses - into `none`: nothing the escaper does tells the two apart today, since a carried segment - holds only spaces, tabs and newlines. + - [x] **2e3 — Attribute canonicalization and the quoted value's escape.** + **Settled** (the maintainer, 2026-08-26): a quoted attribute value escapes `` ` ``, `&`, + `<` and `|` as `\u0060`, `\u0026`, `\u003c` and `\u007c`, in every directive, block and + inline alike — the constructs those four open all bind at or before a directive does, and + nothing else reaches into `{attrs}`. Emitted attributes being inert leaves 3 free to keep + CommonMark's own precedence between a directive and a code span, and collapsed + `escaping: 'attribute'` into `none`. The escaper's link-opener scan skips emitted syntax + to match: a `](` inside a directive escapes no text `[`. - [ ] **2e4 — The carry's fallback triggers.** `spec/flavour.md` carries a node its section cannot spell — an attrs key no section lists, a value that is not the section's type, an arg slot holding no bare token, marks no nesting spells — where the emitter still refuses, - which leaves the refusals a container's own spelling owns. The carry spells - `escaping: 'none'`, which is what makes `tryPipeCell` refuse the pipe form for a carry - whose JSON holds a pipe. The flanking trigger 2e2 added to that list is the odd one out: - `unspellableMark` finds it after assembly and names a mark type against the line's path, - so the failing run needs identifying before the carry can replace the refusal - `corpus/unspellable/mark-inside-word` pins. + which leaves the refusals a container's own spelling owns. The flanking trigger 2e2 + added to that list is the odd one out: `unspellableMark` finds it after assembly and + names a mark type against the line's path, so the failing run needs identifying before + the carry can replace the refusal `corpus/unspellable/mark-inside-word` pins. - [ ] **2e5 — Combined documents and the collision property.** Documents combining nodes rather than isolating one, and the gate's collision property: no two corpus documents may emit the same bytes — one spelling for two documents is a round-trip break no parser can undo, @@ -142,14 +138,14 @@ detail is settled at its own milestone. every raw-HTML construct in input is an error result. The CommonMark spec suite runs against it from here (§10). The parser owes `~` the same `can_open`/`can_close` the emitter assumes — CommonMark flanking, as for `*` — which `spec/flavour.md` does not yet pin, and - the precedence between a directive and the constructs a raw attribute value opens inside it — - a code span, an entity, raw HTML — which one directive alone already reaches until 2e's - escape lands. Whether a claimed line interrupts a paragraph is pinned for the plain case and - unstated for the lazy one: a closing fence on the line after a blockquote's open paragraph is - lazy continuation in CommonMark, which would swallow the fence and leave the container - unclosed. 2e1's `closing-fence-line` orders its blockquote away from that edge meanwhile. - `src/` gets its hierarchy at the same split — `adf/`, - `markdown/`, `html/`, the grammar module shared inside `markdown/` — while the rename is + the precedence in input between a directive and a code span, an entity or raw HTML written + raw inside its attributes — 2e3's escape keeps emitted output clear of that collision, so the + pick binds input alone. Whether a claimed line interrupts a paragraph is pinned for the plain + case and unstated for the lazy one: a closing fence on the line after a blockquote's open + paragraph is lazy continuation in CommonMark, which would swallow the fence and leave the + container unclosed. 2e1's `closing-fence-line` orders its blockquote away from that edge + meanwhile. `src/` gets its hierarchy at the same split — `adf/`, `markdown/`, `html/`, the + grammar module shared inside `markdown/` — while the rename is still mechanical. Three files do not move whole: `block-directives.ts` and `inline-directives.ts` each hold a node table milestones 6-7 need in `adf/` beside a markdown spelling that belongs in `markdown/`, and `directive-attributes.ts` fuses the