3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-21 07:32:58 +01:00

Spell a line in memory linear in its length, so a paragraph up to V8's longest string converts within Node's default heap

This commit is contained in:
2026-10-13 11:48:18 +02:00
committed by Lillem4n
parent 7bc69355c3
commit 03b4ab2e44
5 changed files with 97 additions and 55 deletions
+4
View File
@@ -91,6 +91,10 @@
list lines ran the call out of memory. list lines ran the call out of memory.
- Fix the markdown readers past the longest string or array the JavaScript engine builds: they - Fix the markdown readers past the longest string or array the JavaScript engine builds: they
return `unsupported-node-shape` where they threw. return `unsupported-node-shape` where they threw.
- Fix a paragraph of tens of millions of characters: `adfToLosslessMarkdown` and
`adfToPortableMarkdown` write one up to 536870888 characters within Node's default heap, where 80
million characters ran out of memory and 20 million `*` threw `RangeError`. `losslessMarkdownToAdf`
no longer throws on such a paragraph inside a directive.
- Fix deeply nested compact JSON in an `adf:` fence: `losslessMarkdownToAdf` refuses it as - Fix deeply nested compact JSON in an `adf:` fence: `losslessMarkdownToAdf` refuses it as
`unsupported-node-shape`, as it refuses any compact JSON there, where about a megabyte of it threw. `unsupported-node-shape`, as it refuses any compact JSON there, where about a megabyte of it threw.
- Add `adfToPortableMarkdown` and `portableMarkdownToAdf`, a lossy pair converting ADF to and from - Add `adfToPortableMarkdown` and `portableMarkdownToAdf`, a lossy pair converting ADF to and from
@@ -70,6 +70,12 @@ test('refuses text past V8\'s longest string', () => {
assert.equal(markdown(result), tooLong) assert.equal(markdown(result), tooLong)
}) })
// One escape past the most entries V8's Set holds.
test('escapes more characters in a line than a Set holds', () => {
const stars = '*'.repeat(2 ** 24 + 1)
assert.equal(markdown(adfToLosslessMarkdown(document(paragraph({ text: stars, type: 'text' })))), `${'\\*'.repeat(2 ** 24 + 1)}\n`)
})
test('refuses a document version the markdown cannot carry', () => { test('refuses a document version the markdown cannot carry', () => {
assert.equal(code(adfToLosslessMarkdown({ type: 'doc', version: 2 })), 'unsupported-document-version') assert.equal(code(adfToLosslessMarkdown({ type: 'doc', version: 2 })), 'unsupported-document-version')
}) })
+78 -43
View File
@@ -32,15 +32,20 @@ type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width:
type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; delimiters: EmittedDelimiter[]; length: number; start: number } type EmittedRun = { canClose: boolean; canOpen: boolean; character: string; delimiters: EmittedDelimiter[]; length: number; start: number }
// One byte a character, never an array element or Set entry: a line may run to hundreds of millions of characters.
type Escapings = (index: number) => InlineEscaping | undefined
type EscapeWalk = { type EscapeWalk = {
readonly claims: Claims readonly claims: Claims
readonly container: LineContainer readonly container: LineContainer
readonly escapings: readonly InlineEscaping[] readonly escapings: Escapings
readonly headingCloser: number | undefined readonly headingCloser: number | undefined
readonly linkClose: number readonly linkClose: number
readonly scan: string readonly scan: string
} }
const escapingCodes: readonly InlineEscaping[] = ['backslash', 'bracketed', 'bracketed-link-target', 'none']
const followsLinkText = /[([]/ const followsLinkText = /[([]/
const runCharacters = '*_`~' const runCharacters = '*_`~'
@@ -80,16 +85,19 @@ function resolveEmphasis(segments: readonly InlineSegment[]): InlineSegment[] {
function escape(segments: readonly InlineSegment[], container: LineContainer, claims: Claims): AssembledLine { function escape(segments: readonly InlineSegment[], container: LineContainer, claims: Claims): AssembledLine {
const scan = segments.map((segment) => segment.text).join('') const scan = segments.map((segment) => segment.text).join('')
const escapings: InlineEscaping[] = [] const codes = new Uint8Array(scan.length)
for (const segment of segments) for (let index = 0; index < segment.text.length; index += 1) escapings.push(segment.escaping) let start = 0
const escaped = escapeClosedRuns(scan, escapings, escapeClaims(scan, escapings, container, claims)) for (const segment of segments) {
const placements: number[] = [] codes.fill(escapingCodes.indexOf(segment.escaping), start, start + segment.text.length)
let output = '' start += segment.text.length
for (let index = 0; index < scan.length; index += 1) {
if (escaped.has(index)) output += '\\'
placements.push(output.length)
output += scan.charAt(index)
} }
const escapings: Escapings = (index) => {
const code = codes[index]
return code === undefined ? undefined : escapingCodes[code]
}
const escaped = escapeClaims(scan, escapings, container, claims)
escapeClosedRuns(scan, escapings, escaped)
const { output, placements } = backslashed(segments, scan, escaped)
if (container === 'paragraph' && opensLinkDefinition(output)) { if (container === 'paragraph' && opensLinkDefinition(output)) {
if (segments[0]?.nodes !== undefined) return { line: output, openingLinkAsDirective: true, unspellableRuns: [] } if (segments[0]?.nodes !== undefined) return { line: output, openingLinkAsDirective: true, unspellableRuns: [] }
return { line: `\\${output}`, unspellableRuns: unspellableRuns(segments, output, placements) } return { line: `\\${output}`, unspellableRuns: unspellableRuns(segments, output, placements) }
@@ -97,7 +105,37 @@ function escape(segments: readonly InlineSegment[], container: LineContainer, cl
return { line: output, unspellableRuns: unspellableRuns(segments, output, placements) } return { line: output, unspellableRuns: unspellableRuns(segments, output, placements) }
} }
function escapeClaims(scan: string, escapings: readonly InlineEscaping[], container: LineContainer, claims: Claims): ReadonlySet<number> { // The output, and where each segment's first character landed in it.
function backslashed(segments: readonly InlineSegment[], scan: string, escaped: Uint8Array): { output: string; placements: number[] } {
// Joined a few thousand pieces at a time: an array element per escape would cost more than the character it escapes.
const chunks: string[] = []
let pieces: string[] = []
let from = 0
for (let index = escaped.indexOf(1); index !== -1; index = escaped.indexOf(1, index + 1)) {
pieces.push(scan.slice(from, index), '\\')
from = index
if (pieces.length < 4096) continue
chunks.push(pieces.join(''))
pieces = []
}
pieces.push(scan.slice(from))
chunks.push(pieces.join(''))
const placements: number[] = []
let escapes = 0
let next = escaped.indexOf(1)
let start = 0
for (const segment of segments) {
while (next !== -1 && next <= start) {
escapes += 1
next = escaped.indexOf(1, next + 1)
}
placements.push(start + escapes)
start += segment.text.length
}
return { output: chunks.join(''), placements }
}
function escapeClaims(scan: string, escapings: Escapings, container: LineContainer, claims: Claims): Uint8Array {
const walk: EscapeWalk = { const walk: EscapeWalk = {
claims, claims,
container, container,
@@ -106,20 +144,20 @@ function escapeClaims(scan: string, escapings: readonly InlineEscaping[], contai
linkClose: lastLinkClose(scan, escapings), linkClose: lastLinkClose(scan, escapings),
scan, scan,
} }
const escaped = new Set<number>() const escaped = new Uint8Array(scan.length)
let line = scanLine(scan, 0) let line = scanLine(scan, 0)
let index = 0 let index = 0
while (index < scan.length) { while (index < scan.length) {
while (index > line.start + line.text.length) line = scanLine(scan, line.start + line.text.length + 1) while (index > line.start + line.text.length) line = scanLine(scan, line.start + line.text.length + 1)
if (runCharacters.includes(scan.charAt(index))) { if (runCharacters.includes(scan.charAt(index))) {
const end = index + runLength(scan, index) const end = index + runLength(scan, index)
for (const claimed of runEscapes(walk, index, end, line)) escaped.add(claimed) runEscapes(walk, index, end, line, escaped)
index = end index = end
continue continue
} }
const token = readInlineToken(scan, index, claims) const token = readInlineToken(scan, index, claims)
const claimed = claimsCharacter(walk, index, token, line) const claimed = claimsCharacter(walk, index, token, line)
if (claimed) escaped.add(index) if (claimed) escaped[index] = 1
// The reader takes a `==` whole, so its second `=` starts nothing unless the first is escaped. // The reader takes a `==` whole, so its second `=` starts nothing unless the first is escaped.
index += !claimed && token.kind === 'highlight' ? token.width : 1 index += !claimed && token.kind === 'highlight' ? token.width : 1
} }
@@ -127,25 +165,27 @@ function escapeClaims(scan: string, escapings: readonly InlineEscaping[], contai
} }
// A text character beside one of the emitter's delimiters would lengthen that delimiter's run, so every text character in such a run escapes. // A text character beside one of the emitter's delimiters would lengthen that delimiter's run, so every text character in such a run escapes.
function runEscapes(walk: EscapeWalk, start: number, end: number, line: ScanLine): number[] { function runEscapes(walk: EscapeWalk, start: number, end: number, line: ScanLine, escaped: Uint8Array): void {
const { escapings, scan } = walk const { escapings, scan } = walk
const indexes: number[] = [] if (runHolds(escapings, start, end, isSyntax) && runHolds(escapings, start, end, isEscapable)) {
const run = escapings.slice(start, end) for (let index = start; index < end; index += 1) if (isEscapable(escapings(index))) escaped[index] = 1
if (run.some(isSyntax) && run.some(isEscapable)) { return
for (let index = start; index < end; index += 1) if (isEscapable(escapings[index])) indexes.push(index)
return indexes
} }
for (let index = start; index < end; index += 1) { for (let index = start; index < end; index += 1) {
// A backtick run escapes whole: a bare rest would be a shorter backtick string, which can close a span an earlier run opens. // A backtick run escapes whole: a bare rest would be a shorter backtick string, which can close a span an earlier run opens.
const claimed = (index > start && scan.charAt(index) === '`') || claimsRunRest(walk, index, end, line) const claimed = (index > start && scan.charAt(index) === '`') || claimsRunRest(walk, index, end, line)
if (!claimed) break if (!claimed) break
indexes.push(index) escaped[index] = 1
} }
return indexes }
function runHolds(escapings: Escapings, start: number, end: number, kind: (escaping: InlineEscaping | undefined) => boolean): boolean {
for (let index = start; index < end; index += 1) if (kind(escapings(index))) return true
return false
} }
function claimsRunRest({ claims, container, escapings, scan }: EscapeWalk, index: number, end: number, line: ScanLine): boolean { function claimsRunRest({ claims, container, escapings, scan }: EscapeWalk, index: number, end: number, line: ScanLine): boolean {
const escaping = escapings[index] const escaping = escapings(index)
if (escaping === 'bracketed-link-target') return readInlineToken(scan, index, claims).kind === 'code-span' if (escaping === 'bracketed-link-target') return readInlineToken(scan, index, claims).kind === 'code-span'
if (!isEscapable(escaping)) return false if (!isEscapable(escaping)) return false
if (claimsLineStart(line, index, container)) return true if (claimsLineStart(line, index, container)) return true
@@ -154,7 +194,7 @@ function claimsRunRest({ claims, container, escapings, scan }: EscapeWalk, index
} }
function claimsCharacter({ container, escapings, headingCloser, linkClose, scan }: EscapeWalk, index: number, token: InlineToken, line: ScanLine): boolean { function claimsCharacter({ container, escapings, headingCloser, linkClose, scan }: EscapeWalk, index: number, token: InlineToken, line: ScanLine): boolean {
const escaping = escapings[index] const escaping = escapings(index)
if (escaping === 'bracketed-link-target') return token.kind === 'directive' if (escaping === 'bracketed-link-target') return token.kind === 'directive'
if (!isEscapable(escaping)) return false if (!isEscapable(escaping)) return false
if (index === headingCloser || claimsLineStart(line, index, container)) return true if (index === headingCloser || claimsLineStart(line, index, container)) return true
@@ -169,7 +209,7 @@ function claimsCharacter({ container, escapings, headingCloser, linkClose, scan
case 'html': case 'html':
return true return true
case 'bracket': case 'bracket':
if (token.image) return isSyntax(escapings[index + 1]) if (token.image) return isSyntax(escapings(index + 1))
return escaping === 'bracketed' || index < linkClose return escaping === 'bracketed' || index < linkClose
case 'bracket-close': case 'bracket-close':
return escaping === 'bracketed' return escaping === 'bracketed'
@@ -180,14 +220,14 @@ function claimsCharacter({ container, escapings, headingCloser, linkClose, scan
case 'text': { case 'text': {
const character = scan.charAt(index) const character = scan.charAt(index)
if (character === '|') return container === 'table-cell' if (character === '|') return container === 'table-cell'
return character === '{' && scan.charAt(index - 1) === ']' && isSyntax(escapings[index - 1]) return character === '{' && scan.charAt(index - 1) === ']' && isSyntax(escapings(index - 1))
} }
} }
} }
// CommonMark reads no escape inside a code span, so a backtick string left beside an escape still closes a span that an earlier bare run of its length opens. // CommonMark reads no escape inside a code span, so a backtick string left beside an escape still closes a span that an earlier bare run of its length opens.
function escapeClosedRuns(scan: string, escapings: readonly InlineEscaping[], claimed: ReadonlySet<number>): ReadonlySet<number> { // The walk runs backwards and escapes only the run it just read, so every index it reads still holds the claims alone.
const escaped = new Set(claimed) function escapeClosedRuns(scan: string, escapings: Escapings, escaped: Uint8Array): void {
// The lengths of the backtick strings escapes leave: a bare run of one of these lengths would open a span one of them closes, so it is escaped too. // The lengths of the backtick strings escapes leave: a bare run of one of these lengths would open a span one of them closes, so it is escaped too.
const formed = new Set<number>() const formed = new Set<number>()
let end = scan.length - 1 let end = scan.length - 1
@@ -200,19 +240,18 @@ function escapeClosedRuns(scan: string, escapings: readonly InlineEscaping[], cl
while (scan.charAt(start - 1) === '`') start -= 1 while (scan.charAt(start - 1) === '`') start -= 1
let segmentEnd = end let segmentEnd = end
for (let index = end; index > start; index -= 1) { for (let index = end; index > start; index -= 1) {
if (!claimed.has(index)) continue if (escaped[index] !== 1) continue
formed.add(segmentEnd - index + 1) formed.add(segmentEnd - index + 1)
segmentEnd = index - 1 segmentEnd = index - 1
} }
if (segmentEnd !== end || claimed.has(start)) formed.add(segmentEnd - start + 1) if (segmentEnd !== end || escaped[start] === 1) formed.add(segmentEnd - start + 1)
else if (escapings[start] !== 'none' && formed.has(end - start + 1)) { else if (escapings(start) !== 'none' && formed.has(end - start + 1)) {
for (let index = start; index <= end; index += 1) escaped.add(index) escaped.fill(1, start, end + 1)
// Each escaped backtick then stands alone, a string of length one. // Each escaped backtick then stands alone, a string of length one.
formed.add(1) formed.add(1)
} }
end = start - 1 end = start - 1
} }
return escaped
} }
// One emphasis run, the innermost, or every highlight run the line cannot spell. // One emphasis run, the innermost, or every highlight run the line cannot spell.
@@ -226,10 +265,8 @@ function unspellableRuns(segments: readonly InlineSegment[], output: string, pla
// No `==` in text can open or close, and highlights never nest, so a pair reads back where each delimiter flanks. // No `==` in text can open or close, and highlights never nest, so a pair reads back where each delimiter flanks.
function unreadHighlights(segments: readonly InlineSegment[], output: string, placements: readonly number[]): MarkRun[] { function unreadHighlights(segments: readonly InlineSegment[], output: string, placements: readonly number[]): MarkRun[] {
const unread: MarkRun[] = [] const unread: MarkRun[] = []
let cursor = 0 for (const [index, segment] of segments.entries()) {
for (const segment of segments) { const start = placements[index] ?? 0
const start = placements[cursor] ?? 0
cursor += segment.text.length
if (segment.highlight === undefined) continue if (segment.highlight === undefined) continue
const flanking = highlightFlanking(output, start) const flanking = highlightFlanking(output, start)
const flanks = segment.highlight === 'open' ? flanking.opens : flanking.closes const flanks = segment.highlight === 'open' ? flanking.opens : flanking.closes
@@ -272,10 +309,8 @@ function emittedRuns(segments: readonly InlineSegment[], placements: readonly nu
const runs: EmittedRun[] = [] const runs: EmittedRun[] = []
const nodes: MarkRun[] = [] const nodes: MarkRun[] = []
const open: number[] = [] const open: number[] = []
let cursor = 0 for (const [index, segment] of segments.entries()) {
for (const segment of segments) { const start = placements[index] ?? 0
const start = placements[cursor] ?? 0
cursor += segment.text.length
if (segment.emphasis === undefined) continue if (segment.emphasis === undefined) continue
const closes = segment.emphasis === 'close' const closes = segment.emphasis === 'close'
const pair = closes ? (open.pop() ?? nodes.length) : nodes.length const pair = closes ? (open.pop() ?? nodes.length) : nodes.length
@@ -336,9 +371,9 @@ function scanLine(scan: string, start: number): ScanLine {
} }
// A `]` the emitter spelled sits inside a construct that binds before link text does. // A `]` the emitter spelled sits inside a construct that binds before link text does.
function lastLinkClose(scan: string, escapings: readonly (InlineEscaping | undefined)[]): number { function lastLinkClose(scan: string, escapings: Escapings): number {
for (let cursor = scan.length - 1; cursor >= 0; cursor -= 1) { for (let cursor = scan.length - 1; cursor >= 0; cursor -= 1) {
if (scan.charAt(cursor) !== ']' || isSyntax(escapings[cursor])) continue if (scan.charAt(cursor) !== ']' || isSyntax(escapings(cursor))) continue
if (followsLinkText.test(scan.charAt(cursor + 1))) return cursor if (followsLinkText.test(scan.charAt(cursor + 1))) return cursor
} }
return -1 return -1
@@ -341,6 +341,15 @@ test('refuses the directive form of a blockquote opening with a code span', () =
assert.equal(content(losslessMarkdownToAdf('!adf:blockquote\n` `` `\n!adf:/blockquote\n')), 'unsupported-node-shape: blockquote takes the CommonMark spelling, not the directive form') assert.equal(content(losslessMarkdownToAdf('!adf:blockquote\n` `` `\n!adf:/blockquote\n')), 'unsupported-node-shape: blockquote takes the CommonMark spelling, not the directive form')
}) })
// Some 1000 characters short of V8's longest string, which the quote's `> ` on each of its 600 lines passes.
test('refuses a blockquote directive whose CommonMark spelling would pass V8\'s longest string', () => {
const fence = `\`\`\`\n${`${'a'.repeat(894_782)}\n`.repeat(300)}\`\`\`\n`
assert.equal(
content(losslessMarkdownToAdf(`!adf:blockquote\n${fence}\n${fence}!adf:/blockquote\n`)),
'unsupported-node-shape: the markdown for this document, or for a directive\'s plain CommonMark, would be longer than 536870888 characters, the longest this library writes',
)
})
test('names the directive name no node reads back to', () => { test('names the directive name no node reads back to', () => {
assert.equal(code(losslessMarkdownToAdf('!adf:widget info\nx\n!adf:/widget\n')), 'unknown-directive-name') assert.equal(code(losslessMarkdownToAdf('!adf:widget info\nx\n!adf:/widget\n')), 'unknown-directive-name')
assert.equal(content(losslessMarkdownToAdf('!adf:widget\n')), 'unknown-directive-name: the directive name widget reads back to no node; \\!adf: keeps the prefix literal') assert.equal(content(losslessMarkdownToAdf('!adf:widget\n')), 'unknown-directive-name: the directive name widget reads back to no node; \\!adf: keeps the prefix literal')
-12
View File
@@ -25,7 +25,6 @@
| ID | Release | Exempt | Item | R | S | A | G | Goals | Score | | ID | Release | Exempt | Item | R | S | A | G | Goals | Score |
|---|---|---|---|---|---|---|---|---|---| |---|---|---|---|---|---|---|---|---|---|
| 108 | 0.2.0 | defect | **Spell a line in memory linear in its length, so a paragraph shorter than V8's longest string converts within Node's default heap.** | 4 | 6 | 5 | 8 | 1, 9 | 20.5 |
| 103 | 0.2.0 | | **Fold `unspellable-whitespace` into `unsupported-node-shape`, so a line ending and a null character in a content slot take one code.** | 2 | 2 | 5 | 6 | 1 | 19.5 | | 103 | 0.2.0 | | **Fold `unspellable-whitespace` into `unsupported-node-shape`, so a line ending and a null character in a content slot take one code.** | 2 | 2 | 5 | 6 | 1 | 19.5 |
| 84 | 0.2.0 | | **Read GFM's pipe table without leading pipes, and with alignment colons, in `portableMarkdownToAdf`, dropping the alignment.** | 3 | 4 | 7 | 7 | 4, 7 | 19.4 | | 84 | 0.2.0 | | **Read GFM's pipe table without leading pipes, and with alignment colons, in `portableMarkdownToAdf`, dropping the alignment.** | 3 | 4 | 7 | 7 | 4, 7 | 19.4 |
| 52 | 0.2.0 | | **Spell `colwidth` as a comma list, `colwidth="340,420"`.** | 3 | 3 | 5 | 6 | 6 | 11.5 | | 52 | 0.2.0 | | **Spell `colwidth` as a comma list, `colwidth="340,420"`.** | 3 | 3 | 5 | 6 | 6 | 11.5 |
@@ -75,17 +74,6 @@
## Details ## Details
### 108. Spell a line in memory linear in its length, so a paragraph shorter than V8's longest string converts within Node's default heap.
The line escaper keeps an array element per character twice (`escapings`, `placements` in `escape`,
`line-escaping.ts`) and a `Set` entry per escaped character, so `adfToLosslessMarkdown` holds about
50 bytes a character: one 20-million-character text node took 1.2 GB on Node 24, and one of 80
million exhausted its 2.2 GB default heap. Twenty million `*` throw `RangeError: Set maximum size
exceeded`, which `withinEngineLimits` lets through. `losslessMarkdownToAdf` reaches both through a
directive's CommonMark ask, and reading 537 MB of markdown, a `!adf:bulletList` of two code blocks
just under the length bound, took 3.4 GB on Bun, too much for the gate to test its length refusal:
add that test with this fix. Found while shipping item 106, 2026-10-11.
### 103. Fold `unspellable-whitespace` into `unsupported-node-shape`, so a line ending and a null character in a content slot take one code. ### 103. Fold `unspellable-whitespace` into `unsupported-node-shape`, so a line ending and a null character in a content slot take one code.
`slotFault` in `directive-syntax.ts` gives a newline in an `emoji`, `mention` or `status` content `slotFault` in `directive-syntax.ts` gives a newline in an `emoji`, `mention` or `status` content