3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-12 11:48:29 +01:00

Answer architecture round 1: every container joining several walks sums their projections before it joins, the portable reduction refuses a list whose spelling is refused, and heldTextLength moves beside textLength

This commit is contained in:
2026-10-12 17:10:56 +02:00
committed by Lillem4n
parent e3aa790ec8
commit 2b5d22cbc6
10 changed files with 101 additions and 41 deletions
+7 -4
View File
@@ -667,10 +667,13 @@ is the shortest of the engines the library claims.
Every writer, and the reader's CommonMark ask, refuses markdown longer than 2^29 − 24 characters
(`longestOutput`) before building it, so the same document converts alike everywhere and no
amplification — an indent on every line of a deep subtree, the carry's two-space JSON, text one
string shared by many nodes — builds a string no engine was asked for. Each block counts its lines
once with the indent its containers will prefix them with, so the bound trips at the block that
passes it, before any container copies it. The engines' own throws past their longest string or
array stay refused as `unsupported-node-shape` by `withinEngineLimits`, behind this bound.
string shared by many nodes — builds a string no engine was asked for. Each walk of blocks projects
their length with the indent their containers will prefix every line with, and a container joining
several walks — a list's items, a table's rows — sums their projections before it joins them, so the
bound trips at the block that passes it, before any container copies it. The projection never
undercounts: a list giving way to the directive form counts its items' marker indent, so markdown a
few characters a line under the bound may be refused. The engines' own throws past their longest
string or array stay refused as `unsupported-node-shape` by `withinEngineLimits`, behind this bound.
## Nothing recurses unbounded
+11
View File
@@ -86,6 +86,17 @@ export function textLength(node: AdfNode): number {
return node.text?.length ?? 0
}
// A line spells every character of its nodes' text at least once, so this much text cannot fit.
export function heldTextLength(nodes: readonly AdfNode[]): number {
let length = 0
const pending = [...nodes]
for (let node = pending.pop(); node !== undefined; node = pending.pop()) {
length += textLength(node)
for (const child of nodeContent(node)) pending.push(child)
}
return length
}
export function mergeAdjacentText(nodes: readonly AdfNode[], joins: (previous: AdfNode, node: AdfNode) => boolean): AdfNode[] {
const merged: AdfNode[] = []
for (const node of nodes) {
+15
View File
@@ -69,6 +69,21 @@ test('refuses a directive whose opener and body together would pass V8\'s longes
assert.equal(markdown(adfToLosslessMarkdown(document(expand))), tooLong)
})
test('refuses a list whose items together would pass V8\'s longest string, before joining them', () => {
const item: AdfNode = { content: [{ attrs: {}, content: [{ text: 'a'.repeat(2 ** 28), type: 'text' }], type: 'codeBlock' }], type: 'listItem' }
const result = adfToLosslessMarkdown(document({ content: [item, item], type: 'bulletList' }))
assert.equal(markdown(result), tooLong)
assert.deepEqual(path(result), ['content', 0, 'content', 1])
})
test('refuses a pipe table whose rows their containers would prefix past V8\'s longest string', () => {
const row: AdfNode = { content: [{ content: [{ type: 'paragraph' }], type: 'tableCell' }], type: 'tableRow' }
const header: AdfNode = { content: [{ content: [{ type: 'paragraph' }], type: 'tableHeader' }], type: 'tableRow' }
let quote: AdfNode = { content: [header, ...Array.from({ length: 1_100_000 }, () => row)], type: 'table' }
for (let level = 0; level < 245; level += 1) quote = { content: [quote], type: 'blockquote' }
assert.equal(markdown(adfToLosslessMarkdown(document(quote))), tooLong)
})
test('refuses text past V8\'s longest string before spelling it', () => {
const long = 'a'.repeat(2 ** 28)
const result = adfToLosslessMarkdown(document(paragraph({ text: long, type: 'text' }, { text: long, type: 'text' }, { text: long, type: 'text' })))
+37 -13
View File
@@ -24,7 +24,8 @@ type Delimiters = readonly [string, string]
type EmittedBlock = SpelledBlock & { headroom: number }
type KeptSpelling = { block: EmittedBlock | undefined; depth: number }
type PlacedBlock = SpelledBlock & { node: AdfNode }
type PlacedBlocks = { blocks: readonly PlacedBlock[]; headroom: number }
// projected: the blocks' length once joined and their containers prefix their lines; it never undercounts.
type PlacedBlocks = { blocks: readonly PlacedBlock[]; headroom: number; projected: number }
// CommonMark starts a new list where the marker changes, so every second list in a run of one kind takes `alternateText`.
type SpelledBlock = { spelling: 'commonmark' | 'directive'; text: string } | { alternateText: string; spelling: 'list'; text: string }
// Keyed by reference: only a caller building one object per position (the parse, the portable reduction) passes one; a consumer's document may share a node.
@@ -38,6 +39,7 @@ const bulletWidth = 2
export const largestListMarker = 999999999
// Bare because tryList admits no item carrying attributes, marks or text.
const listItemOpener = spellDirectiveOpener('listItem', undefined, '')
const listItemCloser = spellDirectiveCloser('listItem')
const orderedDelimiters: Delimiters = ['.', ')']
export function adfToLosslessMarkdown(document: AdfDocument): Result<string> {
@@ -59,17 +61,21 @@ function walkBlocks(nodes: readonly AdfNode[], path: ConvertErrorPath, depth: nu
let headroom = largestNesting - depth
if (headroom < 0) return tooDeep(path)
const blocks: PlacedBlock[] = []
// Each block's length once its containers prefix its lines, and the two characters parting it from the next.
let projected = 0
for (const [index, node] of nodes.entries()) {
const block = emitBlock(node, [...path, 'content', index], depth, writing)
if (!block.ok) return block
projected += prefixedLength(block.value.text, writing.indent) + 2
projected += (index === 0 ? 0 : separatorLength(writing)) + prefixedLength(block.value.text, writing.indent) + 1
if (projected > longestOutput) return tooLong([...path, 'content', index])
headroom = Math.min(headroom, block.value.headroom)
blocks.push({ ...block.value, node })
}
return success({ blocks, headroom })
return success({ blocks, headroom, projected })
}
// A block counts the newline ending it; a blank line between two is prefixed as their lines are.
function separatorLength(writing: Writing): number {
return 1 + writing.indent
}
function prefixedLength(text: string, indent: number): number {
@@ -155,7 +161,10 @@ function spellReadableBlock(node: AdfNode, path: ConvertErrorPath, depth: number
if (node.type === 'mediaSingle') return readableText(tryImage(node))
if (node.type === 'paragraph') return tryParagraph(node, path, writing.flavour)
if (node.type === 'rule') return readableText(tryRule(node))
if (node.type === 'table') return readableText(tryPipeTable(node, path, writing.flavour))
if (node.type === 'table') {
const table = tryPipeTable(node, path, writing)
return table === undefined || !table.ok ? table : success(commonMarkText(table.value))
}
return undefined
}
@@ -171,8 +180,10 @@ function spellPortableBlock(node: AdfNode, path: ConvertErrorPath, depth: number
}
function quotedUnder(head: string, node: AdfNode, path: ConvertErrorPath, depth: number, writing: Writing): Result<EmittedBlock> {
const inner = walkBlocks(nodeContent(node), path, depth + 1, indented(writing, quoteWidth))
const quotedWriting = indented(writing, quoteWidth)
const inner = walkBlocks(nodeContent(node), path, depth + 1, quotedWriting)
if (!inner.ok) return inner
if (prefixedLength(head, quotedWriting.indent) + 1 + separatorLength(quotedWriting) + inner.value.projected > longestOutput) return tooLong(path)
const body = joinBlocks(inner.value.blocks, 'document')
return success(commonMarkText(quoted(body === '' ? head : `${head}\n\n${body}`), inner.value.headroom))
}
@@ -188,6 +199,8 @@ function quoted(text: string): string {
function tryTaskList(node: AdfNode, path: ConvertErrorPath, depth: number, writing: Writing): Result<EmittedBlock> {
const items: PlacedBlock[][] = []
// Each walk alone stayed within the bound, so the join is checked before it builds.
let projected = 0
let headroom = largestNesting - depth - 1
// A child other than a task nests in the task before it.
for (const [index, child] of nodeContent(node).entries()) {
@@ -195,6 +208,8 @@ function tryTaskList(node: AdfNode, path: ConvertErrorPath, depth: number, writi
const itemWriting = indented(writing, bulletWidth)
const walk = task ? taskBlocks(child, [...path, 'content', index], depth + 1, itemWriting) : placedBlock(child, [...path, 'content', index], depth + 1, itemWriting)
if (!walk.ok) return walk
projected += (index === 0 ? 0 : separatorLength(writing)) + walk.value.projected
if (projected > longestOutput) return tooLong([...path, 'content', index])
headroom = Math.min(headroom, walk.value.headroom)
const previous = items.at(-1)
if (task || previous === undefined) items.push([...walk.value.blocks])
@@ -212,7 +227,7 @@ function tryTaskList(node: AdfNode, path: ConvertErrorPath, depth: number, writi
function placedBlock(node: AdfNode, path: ConvertErrorPath, depth: number, writing: Writing): Result<PlacedBlocks> {
const block = emitBlock(node, path, depth, writing)
return block.ok ? success({ blocks: [{ ...block.value, node }], headroom: block.value.headroom }) : block
return block.ok ? success({ blocks: [{ ...block.value, node }], headroom: block.value.headroom, projected: prefixedLength(block.value.text, writing.indent) + 1 }) : block
}
// The marker leads the first paragraph, or stands as one where the blocks open with another.
@@ -223,13 +238,15 @@ function taskBlocks(task: AdfNode, path: ConvertErrorPath, depth: number, writin
const content = nodeContent(task)
const line = content.length === 0 ? success('') : emitInlineLine(content, 'paragraph', path, writing.flavour)
if (!line.ok) return line
return success({ blocks: [{ ...markerBlock, text: line.value === '' ? marker : `${marker} ${line.value}` }], headroom: Number.POSITIVE_INFINITY })
const text = line.value === '' ? marker : `${marker} ${line.value}`
return success({ blocks: [{ ...markerBlock, text }], headroom: Number.POSITIVE_INFINITY, projected: prefixedLength(text, writing.indent) + 1 })
}
const walk = walkBlocks(nodeContent(task), path, depth, writing)
if (!walk.ok) return walk
const [first, ...rest] = walk.value.blocks
const blocks = first?.node.type === 'paragraph' ? [{ ...first, text: `${marker} ${first.text}` }, ...rest] : [markerBlock, ...walk.value.blocks]
return success({ blocks, headroom: walk.value.headroom })
// The marker either leads the first line or stands as a block of its own.
return success({ blocks, headroom: walk.value.headroom, projected: walk.value.projected + marker.length + separatorLength(writing) + writing.indent + 1 })
}
function readableText(text: string | undefined): Result<EmittedBlock> | undefined {
@@ -268,9 +285,8 @@ function emitDirectiveBody(node: AdfNode, model: BlockNodeModel, opener: string,
}
const walk = walkBody()
if (!walk.ok) return walk
const body = joinBlocks(walk.value.blocks, 'directive')
if (opener.length + body.length > longestOutput) return tooLong(path)
return success(directivePair(node, opener, body, walk.value.headroom))
if (opener.length + walk.value.projected > longestOutput) return tooLong(path)
return success(directivePair(node, opener, joinBlocks(walk.value.blocks, 'directive'), walk.value.headroom))
}
function tryBlockquote(node: AdfNode, path: ConvertErrorPath, depth: number, writing: Writing): Result<EmittedBlock> | undefined {
@@ -334,6 +350,8 @@ function tryList(node: AdfNode, path: ConvertErrorPath, depth: number, writing:
if (start === undefined || items.length === 0) return undefined
if (items.some((item) => item.type !== 'listItem' || !holdsOnlyAttributes(item, []))) return undefined
const walked: WalkedItem[] = []
// Each walk alone stayed within the bound, so the join is checked before it builds; the directive form adds an opener and a closer line to each item.
let projected = 0
let headroom = Number.POSITIVE_INFINITY
const itemWriting = indented(writing, ordered ? `${start + items.length - 1}. `.length : bulletWidth)
for (const [offset, item] of items.entries()) {
@@ -341,6 +359,8 @@ function tryList(node: AdfNode, path: ConvertErrorPath, depth: number, writing:
if (!walk.ok) return walk
headroom = Math.min(headroom, walk.value.headroom)
walked.push({ node: item, walk: walk.value })
projected += (offset === 0 ? 0 : separatorLength(writing)) + walk.value.projected + directiveItemLength(writing)
if (projected > longestOutput) return tooLong([...path, 'content', offset])
}
const inners = walked.map((item) => {
const inner = joinBlocks(item.walk.blocks, 'list-item')
@@ -351,7 +371,7 @@ function tryList(node: AdfNode, path: ConvertErrorPath, depth: number, writing:
if (list !== undefined) return success(list)
// The directive form spends a level the walk did not count.
if (headroom < 1) return tooDeep(path)
return emitDirectiveBlock(node, ordered ? blockNodes.orderedList : blockNodes.bulletList, path, depth, () => success({ blocks: directiveItems(walked), headroom: headroom - 1 }))
return emitDirectiveBlock(node, ordered ? blockNodes.orderedList : blockNodes.bulletList, path, depth, () => success({ blocks: directiveItems(walked), headroom: headroom - 1, projected }))
}
function spelledList(inners: readonly string[], number: (offset: number) => string, [usual, alternate]: Delimiters, headroom: number): EmittedBlock | undefined {
@@ -370,6 +390,10 @@ function listLines(inners: readonly string[], number: (offset: number) => string
return lines.join('\n')
}
function directiveItemLength(writing: Writing): number {
return listItemOpener.length + listItemCloser.length + 2 * (1 + writing.indent)
}
function directiveItems(items: readonly WalkedItem[]): PlacedBlock[] {
return items.map((item) => ({ ...directivePair(item.node, listItemOpener, joinBlocks(item.walk.blocks, 'directive')), node: item.node }))
}
+1 -12
View File
@@ -10,7 +10,7 @@ import { escapeUnbalanced, spellDestination, spellLinkTarget } from '../commonma
import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
import { flavourClaims, highlightDelimiter } from '../portable/conventions.ts'
import { holdsNullCharacter, trimTrailingSpace } from '../commonmark/grammar.ts'
import { identicalMark, isSpellableText, nodeAttrs, nodeContent, nodeMarks, textLength } from '../../adf/document.ts'
import { heldTextLength, identicalMark, isSpellableText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
import { inlineNodeModel } from '../../adf/inline-nodes.ts'
import { joinsWhenRead, textBreakSpelling } from '../adjacent-text.ts'
import { largestNesting } from '../../nesting.ts'
@@ -101,17 +101,6 @@ function emitLine(nodes: readonly AdfNode[], container: LineContainer, path: Con
}
}
// The line spells every character of its text at least once, so this much text cannot fit.
export function heldTextLength(nodes: readonly AdfNode[]): number {
let length = 0
const pending = [...nodes]
for (let node = pending.pop(); node !== undefined; node = pending.pop()) {
length += textLength(node)
for (const child of nodeContent(node)) pending.push(child)
}
return length
}
function takeFallback(fallbacks: LineFallbacks, fallback: NodeRange | 'opening-link', path: ConvertErrorPath): Result<null> {
if (fallback === 'opening-link') {
if (fallbacks.openingLinkAsDirective) return failure('unsupported-node-shape', 'an opening link spelled as a directive still reads as a link definition, so the line has no spelling left', path)
+10 -3
View File
@@ -1,14 +1,18 @@
import type { AdfNode } from '../../adf/document.ts'
import type { ConvertErrorPath } from '../../result.ts'
import type { WrittenFlavour } from '../portable/conventions.ts'
import { holdsOnlyAttributes, nodeContent } from '../../adf/document.ts'
import { longestOutput, tooLong } from '../../output-length.ts'
import { spellPipeDelimiter, spellPipeRow } from '../pipe-table-syntax.ts'
import { success, type ConvertErrorPath, type Result } from '../../result.ts'
import { tryPipeCell } from './inline-line.ts'
export function tryPipeTable(node: AdfNode, path: ConvertErrorPath, flavour: WrittenFlavour): string | undefined {
// indent: the characters the containers around the table prefix each of its lines with.
export function tryPipeTable(node: AdfNode, path: ConvertErrorPath, { flavour, indent }: { flavour: WrittenFlavour; indent: number }): Result<string> | undefined {
const rows = pipeRows(node)
if (rows === undefined) return undefined
const lines: string[] = []
// A row spells ` | ` beside each cell, and a newline and the indent after it.
let projected = 0
for (const [rowIndex, row] of rows.entries()) {
const cells: string[] = []
for (const [cellIndex, paragraph] of row.entries()) {
@@ -16,11 +20,14 @@ export function tryPipeTable(node: AdfNode, path: ConvertErrorPath, flavour: Wri
const line = content.length === 0 ? '' : tryPipeCell(content, [...path, 'content', rowIndex, 'content', cellIndex, 'content', 0], flavour)
if (line === undefined) return undefined
cells.push(line)
projected += line.length + 3
}
projected += 2 + indent + (rowIndex === 0 ? spellPipeDelimiter(cells.length).length + 1 + indent : 0)
if (projected > longestOutput) return tooLong([...path, 'content', rowIndex])
lines.push(spellPipeRow(cells))
if (rowIndex === 0) lines.push(spellPipeDelimiter(cells.length))
}
return lines.join('\n')
return success(lines.join('\n'))
}
function pipeRows(node: AdfNode): AdfNode[][] | undefined {
@@ -76,6 +76,12 @@ test('refuses a paragraph whose text would pass V8\'s longest string, joining no
assert.equal(portableDocument(document(merged)), 'unsupported-node-shape at /content/0')
})
test('refuses an expand whose title and body together would pass V8\'s longest string', () => {
let quote: AdfNode = node('expand', { title: 'a'.repeat(1_000_000) }, { attrs: {}, content: [text('a\n'.repeat(1_108_000))], type: 'codeBlock' })
for (let level = 0; level < 240; level += 1) quote = { content: [quote], type: 'blockquote' }
assert.equal(portableDocument(document(quote)), `unsupported-node-shape at /${Array.from({ length: 241 }, () => 'content/0').join('/')}`)
})
test('refuses nesting past 500 levels wherever the reduction walks', () => {
const lowest = (bottom: AdfNode): string => {
let deep = bottom
@@ -117,14 +117,16 @@ function concatenated(results: readonly Result<AdfNode[]>[]): Result<AdfNode[]>
return success(blocks)
}
// A list still taking the directive form gives way to its items' blocks.
// A list still taking the directive form gives way to its items' blocks; one whose spelling is refused, too deep or too long, refuses the reduction (the maintainer, 2026-10-12).
function portableSequence(blocks: readonly AdfNode[], reduction: Reduction): Result<AdfNode[]> {
const pending = blocks.filter((block) => block.type !== 'paragraph' || nodeContent(block).length > 0).reverse()
const sequence: AdfNode[] = []
for (let next = pending.pop(); next !== undefined; next = pending.pop()) {
const block = numberedPastMarkers(next)
const listed = block.type === 'bulletList' || block.type === 'orderedList'
if (!listed || (block === next && commonMarkSpelling(block, reduction.path, reduction.depth, { flavour: 'portable', indent: 0, memo: reduction.memo })?.ok === true)) {
const spelled = listed && block === next ? commonMarkSpelling(block, reduction.path, reduction.depth, { flavour: 'portable', indent: 0, memo: reduction.memo }) : undefined
if (spelled !== undefined && !spelled.ok) return spelled
if (!listed || spelled !== undefined) {
sequence.push(block)
continue
}
+2 -2
View File
@@ -6,8 +6,8 @@ import { failure, success, type ConvertErrorPath, type Result } from '../../resu
import { joinsWhenEditorNormal, sameMarkWhenEditorNormal } from './editor-normal.ts'
import { largestNesting } from '../../nesting.ts'
import { longestOutput, tooLong } from '../../output-length.ts'
import { mergeAdjacentText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
import { heldTextLength, portableLineFallback, type PortableLineFallback } from '../emit/inline-line.ts'
import { heldTextLength, mergeAdjacentText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
import { portableLineFallback, type PortableLineFallback } from '../emit/inline-line.ts'
import { spellDestination, spellLinkTarget } from '../commonmark/link-syntax.ts'
const highlight = 'backgroundColor'
+8 -5
View File
@@ -80,7 +80,8 @@ The line escaper keeps an array element per character twice (`escapings`, `place
`line-escaping.ts`) and a `Set` entry per escaped character, so `adfToLosslessMarkdown` holds about
50 bytes a character: one 20-million-character text node took 1.2 GB on Node 24, and one of 80
million exhausted its 2.2 GB default heap. Twenty million `*` throw `RangeError: Set maximum size
exceeded`, which `withinEngineLimits` lets through. Found while shipping item 106, 2026-10-11.
exceeded`, which `withinEngineLimits` lets through. `losslessMarkdownToAdf` reaches both through a
directive's CommonMark ask. Found while shipping item 106, 2026-10-11.
### 103. Fold `unspellable-whitespace` into `unsupported-node-shape`, so a line ending and a null character in a content slot take one code.
@@ -208,9 +209,11 @@ the reason the project read worse.
### 109. Spell nested lists and quotes in time linear in the markdown.
Each level of a nested list or quote re-indents its whole text, so `losslessMarkdownToAdf` on 3 MB
of lazy lines in a list nested 490 levels inside `!adf:blockquote` spends about 5 seconds
re-spelling it, the writers the same on its document. Found while shipping item 106, 2026-10-11.
Each level of a nested list or quote re-indents its whole text, so the work is the depth times the
text. `losslessMarkdownToAdf` on 3 MB of lazy lines in a list nested 490 levels inside
`!adf:blockquote` spends about 5 seconds re-spelling it. The same 2 MB split over two items of a
list nested 480 levels took 11 minutes to refuse on Node 24, every item fitting the length bound
alone. The writers do the same on such a document. Found while shipping item 106, 2026-10-11.
### 7. Ship HTML: `adfToHtml`, `htmlToAdf`, and markdown to and from HTML composed through ADF, each call named by its flavour.
@@ -272,7 +275,7 @@ document's whole tree. Give the node's path, and name its type and the attribute
### 68. Carry a non-text inline node holding `content` or `text`, a block holding `text`, and a leaf block holding `content`.
`refuseContentAndText` (`emit/inline-line.ts`) and `emitDirectiveBlock` (`emit/adf-to-markdown.ts`)
`emitLeaf` (`emit/inline-line.ts`) and `emitDirectiveBlock` (`emit/adf-to-markdown.ts`)
refuse these with `unsupported-node-shape`, though `isAdfDocument` accepts them and the carry
round-trips them; `docs/decisions.md` §The code list gives a cause the carry answers no code. Drop
them from the emitting half of the README's `unsupported-node-shape` row. Found by the README-goals