diff --git a/docs/reading-cost.md b/docs/reading-cost.md index ccf3060..6a631c8 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -516,3 +516,20 @@ Read most (opens and searches): 436866 new input tokens before the first commit. ## Reading cost reset, 2026-10-28T23:28:20Z + +## Reading cost, chunk a45f84ed8e32, finished 2026-10-29T11:27:46Z + +96824 new input tokens before the first commit. + +Read most (opens and searches): + +- `todo.md` 13 +- `README.md` 4 +- `docs/decisions.md` 4 +- `docs/reading-cost.md` 4 +- `AGENTS.md` 1 +- `CHANGELOG.md` 1 +- `MIGRATION.md` 1 +- `ci.sh` 1 +- `docker-runner.sh` 1 +- `package.json` 1 diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 92446b0..17925af 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -289,6 +289,16 @@ test('carries a code mark over anything but text', () => { ) }) +test('carries every inline node of a line that only the carry spells', () => { + const odd: AdfNode = { attrs: { id: 'a', odd: 'b' }, type: 'mention' } + const carriedOdd = '!adf:carry{json="{\\"attrs\\":{\\"id\\":\\"a\\",\\"odd\\":\\"b\\"},\\"type\\":\\"mention\\"}"}' + const unlinked = '!adf:carry{json="{\\"marks\\":[{\\"type\\":\\"link\\"}],\\"text\\":\\"x\\",\\"type\\":\\"text\\"}"}' + assert.equal( + markdown(adfToLosslessMarkdown(document(paragraph(odd, { text: ' ', type: 'text' }, { marks: [{ type: 'link' }], text: 'x', type: 'text' }, { text: ' ', type: 'text' }, odd)))), + `${carriedOdd} ${unlinked} ${carriedOdd}\n`, + ) +}) + test('carries an inline node holding a mark Atlassian\'s schema withholds from it', () => { const carried = (node: AdfNode): string => markdown(adfToLosslessMarkdown(document(paragraph(node)))) assert.equal(carried({ attrs: { id: 'b1c2', text: '@A' }, marks: [{ type: 'strong' }], type: 'mention' }), '!adf:carry{json="{\\"attrs\\":{\\"id\\":\\"b1c2\\",\\"text\\":\\"@A\\"},\\"marks\\":[{\\"type\\":\\"strong\\"}],\\"type\\":\\"mention\\"}"}\n') @@ -392,6 +402,12 @@ test('carries a mark spelling that cannot open or close where it sits', () => { 'x!adf:carry{json="{\\"marks\\":[{\\"type\\":\\"strong\\"}],\\"text\\":\\".a\\",\\"type\\":\\"text\\"}"}\n', ) assert.equal(emitted({ text: 'un ', type: 'text' }, marked('-real', strong), { text: ' istic', type: 'text' }), 'un **-real** istic\n') + const carriedR = '!adf:carry{json="{\\"marks\\":[{\\"type\\":\\"strong\\"}],\\"text\\":\\"-r\\",\\"type\\":\\"text\\"}"}' + assert.equal(emitted({ text: 'un', type: 'text' }, marked('-r', strong), { text: 'un', type: 'text' }, marked('-r', strong), { text: 'un', type: 'text' }), `un${carriedR}un${carriedR}un\n`) + assert.equal( + emitted({ text: 'x', type: 'text' }, marked('[', { type: 'em' }, { type: 'strike' }), marked('a', { type: 'em' })), + 'x!adf:carry{json="{\\"marks\\":[{\\"type\\":\\"em\\"},{\\"type\\":\\"strike\\"}],\\"text\\":\\"[\\",\\"type\\":\\"text\\"}"}_a_\n', + ) assert.equal(emitted(marked('a.', strong)), '**a.**\n') assert.equal(emitted({ text: 'x', type: 'text' }, marked('a', strong), { text: 'y', type: 'text' }), 'x**a**y\n') const em: AdfMark = { type: 'em' } diff --git a/src/markdown/emit/inline-line.ts b/src/markdown/emit/inline-line.ts index c198da7..150832f 100644 --- a/src/markdown/emit/inline-line.ts +++ b/src/markdown/emit/inline-line.ts @@ -22,7 +22,8 @@ import { spellTextDirective } from '../directive/text.ts' type EmittedLine = { line: string; openingLinkAsDirective: boolean; segments: InlineSegment[] } -type Emission = { carry: NodeRange; segments?: undefined } | { carry?: undefined; segments: InlineSegment[] } +// Every carry one walk finds, taken together: carrying a node changes no other carry's range. +type Emission = { carries: [NodeRange, ...NodeRange[]]; segments?: undefined } | { carries?: undefined; segments: InlineSegment[] } type InlineContext = { atBlockEnd: boolean @@ -37,7 +38,7 @@ type InlineContext = { type InlineRun = { index: number; kind: 'marked'; mark: AdfMark; nodes: AdfNode[] } | { index: number; kind: 'plain'; node: AdfNode } -type LineAttempt = { fallback: NodeRange | 'opening-link'; line?: undefined } | { fallback?: undefined; line: string } +type LineAttempt = { fallback: readonly NodeRange[] | 'opening-link'; line?: undefined } | { fallback?: undefined; line: string } type LineFallbacks = { carried: Set; flavour: WrittenFlavour; openingLinkAsDirective: boolean } @@ -58,7 +59,7 @@ export function openingLinkTakesDirective(nodes: readonly AdfNode[], path: Conve export function portableLineFallback(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath): Result { const emission = lineSegments(nodes, container, path, withheldMarkFallbacks(nodes, 'portable')) if (!emission.ok) return emission - if (emission.value.carry !== undefined) return failure('unsupported-node-shape', 'an inline node on a portable line has no spelling but the carry', path) + if (emission.value.carries !== undefined) return failure('unsupported-node-shape', 'an inline node on a portable line has no spelling but the carry', path) const verdict = lineVerdict(emission.value.segments, container, 'portable') return success(verdict.kind === 'line' ? undefined : verdict) } @@ -89,8 +90,8 @@ function emitLine(nodes: readonly AdfNode[], container: LineContainer, path: Con for (;;) { const emission = lineSegments(nodes, container, path, fallbacks) if (!emission.ok) return emission - if (emission.value.carry !== undefined) { - const taken = takeFallback(fallbacks, emission.value.carry, path) + if (emission.value.carries !== undefined) { + const taken = takeFallback(fallbacks, emission.value.carries, path) if (!taken.ok) return taken continue } @@ -113,14 +114,14 @@ function withheldMarkFallbacks(nodes: readonly AdfNode[], flavour: WrittenFlavou return { carried, flavour, openingLinkAsDirective: false } } -function takeFallback(fallbacks: LineFallbacks, fallback: NodeRange | 'opening-link', path: ConvertErrorPath): Result { +function takeFallback(fallbacks: LineFallbacks, fallback: readonly NodeRange[] | 'opening-link', path: ConvertErrorPath): Result { if (fallback === 'opening-link') { if (fallbacks.openingLinkAsDirective) return failure('unsupported-node-shape', 'an opening link spelled as a directive still reads as a link definition, so the line has no spelling left', path) fallbacks.openingLinkAsDirective = true return success(null) } const before = fallbacks.carried.size - for (let index = fallback.first; index <= fallback.last; index += 1) fallbacks.carried.add(index) + for (const range of fallback) for (let index = range.first; index <= range.last; index += 1) fallbacks.carried.add(index) if (fallbacks.carried.size === before) return failure('unsupported-node-shape', 'a carry took no inline node the line had not carried, so the line has no spelling left', path) return success(null) } @@ -129,14 +130,14 @@ function lineSegments(nodes: readonly AdfNode[], container: LineContainer, path: const context: InlineContext = { atBlockEnd: true, bracketed: false, ...fallbacks, container, path, spansLines: container === 'paragraph' } const emission = emitRun(nodes, 0, 0, context) if (!emission.ok) return emission - if (emission.value.carry !== undefined) return emission + if (emission.value.carries !== undefined) return emission return success({ segments: spellEdgeWhitespace(emission.value.segments) }) } function attemptLine(segments: readonly InlineSegment[], container: LineContainer, flavour: WrittenFlavour): LineAttempt { const verdict = lineVerdict(segments, container, flavour) if (verdict.kind === 'opening-link') return { fallback: 'opening-link' } - if (verdict.kind === 'unspellable-run') return { fallback: verdict.runs[0] } + if (verdict.kind === 'unspellable-run') return { fallback: verdict.runs } return { line: verdict.text } } @@ -201,16 +202,18 @@ function emitRun(nodes: readonly AdfNode[], depth: number, firstIndex: number, c return failure('unsupported-nesting-depth', `the marks nest deeper than the ${largestNesting} levels the emitter carries`, context.path) } const runs = inlineRuns(nodes, depth, firstIndex, context.carried) + const carries: NodeRange[] = [] const segments: InlineSegment[] = [] for (const [offset, run] of runs.entries()) { if (takesTextBreak(runs[offset - 1], run, context.carried)) segments.push(syntax(textBreakSpelling)) const runContext = { ...context, atBlockEnd: context.atBlockEnd && offset === runs.length - 1 } const emitted = run.kind === 'plain' ? emitLeaf(run.node, runContext, run.index) : emitMarkedRun(run.nodes, run.mark, depth, run.index, runContext) if (!emitted.ok) return emitted - if (emitted.value.carry !== undefined) return emitted - for (const segment of emitted.value.segments) segments.push(segment) + if (emitted.value.carries !== undefined) for (const carry of emitted.value.carries) carries.push(carry) + else for (const segment of emitted.value.segments) segments.push(segment) } - return success({ segments }) + const [carry, ...others] = carries + return success(carry === undefined ? { segments } : { carries: [carry, ...others] }) } function inlineRuns(nodes: readonly AdfNode[], depth: number, firstIndex: number, carried: ReadonlySet): InlineRun[] { @@ -269,24 +272,24 @@ function emitLeaf(node: AdfNode, context: InlineContext, index: number): Result< function emitHardBreak(node: AdfNode, model: InlineNodeModel, context: InlineContext, index: number): Emission { const attributes = spellInlineNodeAttributes(node, model) - if (attributes === undefined) return { carry: { first: index, last: index } } + if (attributes === undefined) return { carries: [{ first: index, last: index }] } if (attributes === '' && context.spansLines && !context.atBlockEnd) return { segments: [syntax('\\\n')] } return { segments: [syntax(spellInlineLeafDirective('hardBreak', attributes))] } } function emitInlineDirective(node: AdfNode, model: InlineNodeModel, index: number): Emission { const attributes = spellInlineNodeAttributes(node, model) - if (attributes === undefined) return { carry: { first: index, last: index } } + if (attributes === undefined) return { carries: [{ first: index, last: index }] } const slot = model.textAttribute === undefined ? undefined : nodeAttrs(node)[model.textAttribute] if (slot === undefined) return { segments: [syntax(spellInlineLeafDirective(node.type, attributes))] } - if (typeof slot !== 'string' || slotFault(node.type, slot) !== undefined) return { carry: { first: index, last: index } } + if (typeof slot !== 'string' || slotFault(node.type, slot) !== undefined) return { carries: [{ first: index, last: index }] } const content: InlineSegment[] = slot === '' ? [] : [{ escaping: 'bracketed', text: slot }] return { segments: [syntax(spellInlineDirectiveOpener(node.type)), ...content, syntax(`]${attributes}`)] } } // CommonMark reads a raw carriage return as a line ending and a null character as U+FFFD, so neither is written raw. function emitText(node: AdfNode, context: InlineContext, index: number): Emission { - if (!isSpellableText(node) || typeof node.text !== 'string' || node.text === '') return { carry: { first: index, last: index } } + if (!isSpellableText(node) || typeof node.text !== 'string' || node.text === '') return { carries: [{ first: index, last: index }] } const escaping: InlineEscaping = context.bracketed ? 'bracketed' : 'backslash' const parts = node.text.split(/(\n+|\0+|\r)/).filter((part) => part !== '') return { segments: parts.map((part) => (part === '\r' ? syntax(' ') : /^[\n\0]/.test(part) ? textDirectiveSegment(part) : { escaping, text: part })) } @@ -296,23 +299,23 @@ function emitMarkedRun(nodes: readonly AdfNode[], mark: AdfMark, depth: number, const range: NodeRange = { first: index, last: index + nodes.length - 1 } if (mark.type === 'backgroundColor' && flavourClaims[context.flavour].highlights) return emitHighlight(nodes, depth, range, context) const spelling = markSpelling(mark.type) - if (spelling === undefined) return success({ carry: range }) + if (spelling === undefined) return success({ carries: [range] }) const attributes = spellMarkAttributes(mark, spelling) - if (attributes === undefined) return success({ carry: range }) + if (attributes === undefined) return success({ carries: [range] }) if (spelling.kind === 'code') return success(emitCodeSpan(nodes, depth, range)) if (spelling.kind === 'emphasis') return emitEmphasis(nodes, spelling.spelling, depth, range, context) const link = spelling.kind === 'link' ? tryLink(nodes, mark, depth, range, context) : undefined if (link !== undefined) return link const inner = emitRun(nodes, depth + 1, index, { ...context, bracketed: true, spansLines: false }) if (!inner.ok) return inner - if (inner.value.carry !== undefined) return inner + if (inner.value.carries !== undefined) return inner return success({ segments: [syntax(spellInlineDirectiveOpener(mark.type)), ...inner.value.segments, syntax(`]${attributes}`)] }) } function emitEmphasis(nodes: readonly AdfNode[], spelling: string, depth: number, range: NodeRange, context: InlineContext): Result { const inner = emitRun(nodes, depth + 1, range.first, context) if (!inner.ok) return inner - if (inner.value.carry !== undefined) return inner + if (inner.value.carries !== undefined) return inner const spelled = spellEdgeWhitespace(inner.value.segments) return success({ segments: [ @@ -325,7 +328,7 @@ function emitEmphasis(nodes: readonly AdfNode[], spelling: string, depth: number function emitHighlight(nodes: readonly AdfNode[], depth: number, range: NodeRange, context: InlineContext): Result { const inner = emitRun(nodes, depth + 1, range.first, context) - if (!inner.ok || inner.value.carry !== undefined) return inner + if (!inner.ok || inner.value.carries !== undefined) return inner const run = { ...range, depth } return success({ segments: [{ escaping: 'none', highlight: 'open', nodes: run, text: highlightDelimiter }, ...inner.value.segments, { escaping: 'none', highlight: 'close', nodes: run, text: highlightDelimiter }], @@ -337,7 +340,7 @@ function emitCodeSpan(nodes: readonly AdfNode[], depth: number, range: NodeRange const spans: string[] = [] for (const node of nodes) { const { text } = node - if (!isSpellableText(node) || nodeMarks(node).length !== depth + 1 || typeof text !== 'string' || text === '' || /[\n\r]/.test(text) || holdsNullCharacter(text)) return { carry: range } + if (!isSpellableText(node) || nodeMarks(node).length !== depth + 1 || typeof text !== 'string' || text === '' || /[\n\r]/.test(text) || holdsNullCharacter(text)) return { carries: [range] } const fence = '`'.repeat(longestBacktickRun(text) + 1) spans.push(`${fence}${needsPadding(text) ? ` ${text} ` : text}${fence}`) } @@ -351,14 +354,14 @@ function needsPadding(text: string): boolean { function tryLink(nodes: readonly AdfNode[], mark: AdfMark, depth: number, range: NodeRange, context: InlineContext): Result | undefined { const href = linkHref(nodeAttrs(mark)) - if (href === undefined) return success({ carry: range }) + if (href === undefined) return success({ carries: [range] }) const opening = depth === 0 && range.first === 0 && context.openingLinkAsDirective const commonMark = opening ? undefined : commonMarkLink(nodeAttrs(mark), href, nodes, depth + 1, context.bracketed, titleEscapes(context.flavour, context.container)) if (commonMark === undefined) return undefined if (commonMark.form === 'autolink') return success({ segments: [syntax(`<${href}>`)] }) const inner = emitRun(nodes, depth + 1, range.first, { ...context, bracketed: true }) if (!inner.ok) return inner - if (inner.value.carry !== undefined) return inner + if (inner.value.carries !== undefined) return inner const target = context.bracketed ? bracketedLinkTarget(commonMark.target) : syntax(commonMark.target) return success({ segments: [{ escaping: 'none', nodes: range, text: '[' }, ...inner.value.segments, syntax(']('), target, syntax(')')] }) } diff --git a/src/markdown/emit/line-escaping.ts b/src/markdown/emit/line-escaping.ts index 91d6dc0..172242b 100644 --- a/src/markdown/emit/line-escaping.ts +++ b/src/markdown/emit/line-escaping.ts @@ -6,7 +6,7 @@ import { delimiterRunToken, highlightFlanking, inlineSource, readInlineToken } f import { escapesLineClaim, type LinePosition } from '../commonmark/grammar.ts' import { flavourClaims } from '../flavour.ts' import { isBareDelimiterRow } from '../pipe-table-syntax.ts' -import { isWordCharacter, runLength } from '../commonmark/emphasis-matching.ts' +import { delimiterFlags, isWordCharacter, runLength } from '../commonmark/emphasis-matching.ts' import { opensLinkDefinition } from '../commonmark/link-reference-definitions.ts' import { readLineBack } from '../inline-content.ts' @@ -267,8 +267,9 @@ function claimsCharacter({ container, escapings, headingCloser, inline: { text: } } -// Of the runs the line does not read back as written, the first emphasis run to close, else every highlight run: one misread emphasis pair can re-pair the delimiters after it. +// Of the runs the line does not read back as written, the first emphasis run to close and every later one whose own delimiters cannot flank, else every highlight run: one misread emphasis pair can re-pair the delimiters after it. function unspellableRuns(placed: readonly PlacedSegment[], reading: InlineReading, output: string): MarkRun[] { + const misread: MarkRun[] = [] const misflanked: MarkRun[] = [] const openers: { nodes: MarkRun; start: number }[] = [] for (const { segment, start } of placed) { @@ -277,10 +278,26 @@ function unspellableRuns(placed: readonly PlacedSegment[], reading: InlineReadin if (segment.highlight !== undefined && !highlightFlanks(output, start, segment.highlight) && misflanked.at(-1) !== segment.nodes) misflanked.push(segment.nodes) if (segment.emphasis === 'close') { const opener = openers.pop() - if (opener !== undefined && !readAsWritten(reading, opener.start, start, segment.text.length)) return [opener.nodes] + if (opener === undefined) continue + const latest = misread.at(-1) + if (latest === undefined ? !readAsWritten(reading, opener.start, start, segment.text.length) : givesUpWith(latest, opener.nodes, output, opener.start, start, segment.text)) misread.push(opener.nodes) } } - return misflanked + return misread.length > 0 ? misread : misflanked +} + +// A later run gives up with the earlier ones where it holds none of them, so their fallbacks leave its range, and a delimiter of it no other delimiter character touches fails to flank, beside text or its own hard break, which their fallbacks leave too. Runs close in order, so a run holding any earlier one holds the latest. +function givesUpWith(latest: MarkRun, run: MarkRun, output: string, opener: number, closer: number, delimiter: string): boolean { + if (run.first <= latest.first && latest.last <= run.last) return false + const flanks = (start: number, role: DelimiterRole): boolean => { + const before = output.charAt(start - 1) + const after = output.charAt(start + delimiter.length) + const character = delimiter.charAt(0) + if (before === character || after === character) return true + const flags = delimiterFlags(character, before, after) + return role === 'open' ? flags.canOpen : flags.canClose + } + return !flanks(opener, 'open') || !flanks(closer, 'close') } function highlightFlanks(output: string, start: number, role: DelimiterRole): boolean { diff --git a/src/markdown/portable/adf-to-portable-markdown.test.ts b/src/markdown/portable/adf-to-portable-markdown.test.ts index 8ca648e..54ba29c 100644 --- a/src/markdown/portable/adf-to-portable-markdown.test.ts +++ b/src/markdown/portable/adf-to-portable-markdown.test.ts @@ -379,6 +379,8 @@ test('percent-encodes a link href no CommonMark escape writes until one does', ( test('drops the mark of a run CommonMark flanking or matching cannot spell', () => { assert.equal(portable(paragraph(text('un'), text('-real', strong), text('istic'))), 'un-realistic\n') + assert.equal(portable(paragraph(text('un'), text('-r', strong), text('un'), text('-r', strong), text('un'))), 'un-run-run\n') + assert.equal(portable(paragraph(text('0'), text('a', { type: 'strike' }, strong), text('-', strong), text('0'))), '0a-0\n') assert.equal(portable(paragraph(text('a', em), text('b', strong), text('c', em))), '_a_**b**_c_\n') assert.equal(portable(paragraph(text('x'), text('*', em), text('y'))), 'x\\*y\n') const highlight: AdfMark = { type: 'backgroundColor' } diff --git a/src/markdown/portable/inline-reduction.ts b/src/markdown/portable/inline-reduction.ts index 0c73a70..4b0f528 100644 --- a/src/markdown/portable/inline-reduction.ts +++ b/src/markdown/portable/inline-reduction.ts @@ -241,11 +241,18 @@ function withoutFallback(leaves: readonly AdfNode[], fallback: PortableLineFallb } // The runs cover disjoint leaves, so one pass drops them all. -function withoutMarks(leaves: readonly AdfNode[], runs: readonly MarkRun[]): AdfNode[] { +function withoutMarks(leaves: readonly AdfNode[], [first, ...others]: readonly [MarkRun, ...MarkRun[]]): AdfNode[] { const depths = new Map() - for (const run of runs) for (let index = run.first; index <= run.last; index += 1) depths.set(index, run.depth) + for (const run of [first, ...others.filter((run) => dropsWithFirst(leaves, run))]) for (let index = run.first; index <= run.last; index += 1) depths.set(index, run.depth) return leaves.map((leaf, index) => { const depth = depths.get(index) return depth === undefined ? leaf : withMarks(leaf, nodeMarks(leaf).filter((_, held) => held !== depth)) }) } + +// Every misflanked highlight drops at once, a later emphasis run only where dropping its mark merges no run and moves no edge: its mark is its leaves' innermost and no leaf beside it holds a mark that deep. +function dropsWithFirst(leaves: readonly AdfNode[], run: MarkRun): boolean { + if (nodeMarks(leaves[run.first] ?? {})[run.depth]?.type === highlight) return true + const beside = [leaves[run.first - 1], leaves[run.last + 1]].every((leaf) => leaf === undefined || nodeMarks(leaf).length <= run.depth) + return beside && leaves.slice(run.first, run.last + 1).every((leaf) => nodeMarks(leaf).length === run.depth + 1) +}