larvit/adf-codec
3
0
Fork 0
Code Pull Requests Activity

Answer architecture round 1: a code span opening a line spells as CommonMark, the slot refuses a null character, every generated document emits

Browse Source
This commit is contained in:
Lilleman auf Larv
2026-10-04 20:43:46 +02:00
parent 03f18ee17b
commit 7b57c1edc8
24 changed files with 183 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/adf/document.ts
+1 -1
View File
@@ -65,7 +65,7 @@ export function emptyKeys(held: { attrs?: AdfAttributes; content?: AdfNode[]; ma
return keys
}
// A format spells this node as text; anything more rides a carry, save content, which is refused.
// A format spells this node as text; anything more rides a carry.
export function isBareText(node: AdfNode): boolean {
return node.type === 'text' && node.attrs === undefined && node.content === undefined && node.marks?.length !== 0
}
src/conformance/adf-property.test.ts
+2 -2
View File
@@ -34,11 +34,11 @@ function shownText(nodes: readonly AdfNode[]): string[] {
return shown.filter((text) => text !== '')
}
test('a generated document refuses to emit, or its markdown reads back to it', { timeout: propertyTimeout }, () => {
test('a generated document emits markdown that reads back to it', { timeout: propertyTimeout }, () => {
fc.assert(
fc.property(adfDocument, (document) => {
const emitted = adfToMarkdown(document)
if (!emitted.ok) return
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
const read = markdownToAdf(emitted.value)
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
assert.deepEqual(read.value, document, `reading ${JSON.stringify(emitted.value)}`)
src/markdown/commonmark/grammar.ts
-4
View File
@@ -83,10 +83,6 @@ export function backslashEscape(text: string, index: number): string | undefined
return isAsciiPunctuation(escaped) ? escaped : undefined
}
export function claimsLine(line: string, position: LinePosition): boolean {
return escapesLineClaim(line, 0, position) || orderedListOpener.test(line)
}
export function claimsPipeLine(line: string): boolean {
return pipeClaim.test(line)
}
src/markdown/commonmark/link-syntax.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { backslashEscape, decodeTextEscapes, holdsControlCharacter } from './grammar.ts'
import { backslashEscape, decodeTextEscapes, holdsControlCharacter, holdsNullCharacter } from './grammar.ts'
import { holdsEntityReference } from './entity-references.ts'
export type LinkDefinition = { destination: string; title?: string }
@@ -117,5 +117,5 @@ export function escapeUnbalanced(spelling: string, opener: string, closer: strin
}
function spellTitle(title: string): string | undefined {
return /[\n\r\0\\]/.test(title) || holdsEntityReference(title) ? undefined : ` "${title.replaceAll('"', '\\"')}"`
return /[\n\r\\]/.test(title) || holdsNullCharacter(title) || holdsEntityReference(title) ? undefined : ` "${title.replaceAll('"', '\\"')}"`
}
src/markdown/directive-syntax.ts
+5 -5
View File
@@ -1,7 +1,7 @@
import type { AttributeKind, VocabularyPair, VocabularyValue } from '../adf/attribute-vocabulary.ts'
import type { ConvertFault } from '../result.ts'
import type { JsonValue } from '../json-value.ts'
import { backslashEscape } from './commonmark/grammar.ts'
import { backslashEscape, holdsNullCharacter } from './commonmark/grammar.ts'
import { backtickRun, closingBacktickRun } from './commonmark/backtick-runs.ts'
import { isJsonValue, overNested } from '../json-value.ts'
import { largestNesting } from '../nesting.ts'
@@ -100,10 +100,10 @@ export function readSoleStringAttribute(span: DirectiveSpan, key: string): Read<
return { value: spelled.decoded }
}
// An inline directive never spans lines, so no content slot holds a line ending.
export function slotLineEndingFault(type: string, text: string): ConvertFault | undefined {
if (!/[\n\r]/.test(text)) return undefined
return { code: 'unspellable-whitespace', message: `the ${type} content slot holds a newline no inline directive spans` }
// What no content slot holds, which the emitter carries instead: an inline directive never spans lines, and CommonMark reads a null character as U+FFFD.
export function slotFault(type: string, text: string): ConvertFault | undefined {
if (/[\n\r]/.test(text)) return { code: 'unspellable-whitespace', message: `the ${type} content slot holds a newline no inline directive spans` }
return holdsNullCharacter(text) ? unsupportedNodeShape(`the ${type} content slot holds a null character: write the node as !adf:carry`) : undefined
}
export function spellAttributes(pairs: readonly (readonly [string, string])[]): string {
src/markdown/emit/adf-to-markdown.test.ts
+4 -5
View File
@@ -153,11 +153,10 @@ function readsBack(doc: AdfDocument): string {
return spelled
}
test('carries a code span whose backticks would open its line as a fence', () => {
const fence: AdfNode = { marks: [{ type: 'code' }], text: '```', type: 'text' }
const carried = '!adf:carry{json="{\\"marks\\":[{\\"type\\":\\"code\\"}],\\"text\\":\\"\\u0060\\u0060\\u0060\\",\\"type\\":\\"text\\"}"}'
assert.equal(readsBack(document(paragraph(fence))), `${carried}\n`)
assert.equal(readsBack(document(paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, fence, fence))), `a\\\n${carried}\`\`\`\` \`\`\` \`\`\`\`\n`)
test('spells a code span opening a line, whose backticks open no fence', () => {
const fence: AdfNode = { marks: [{ type: 'code' }], text: '``', type: 'text' }
assert.equal(readsBack(document(paragraph(fence))), '``` `` ```\n')
assert.equal(readsBack(document(paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, fence))), 'a\\\n``` `` ```\n')
})
test('escapes the delimiter row a hard break leaves opening a pipe table with no leading pipe', () => {
src/markdown/emit/inline-line.ts
+11 -28
View File
@@ -4,17 +4,17 @@ import type { InlineNodeModel } from '../../adf/inline-nodes.ts'
import type { LineContainer } from '../line-container.ts'
import { assembleInlineLine, isSyntax, type InlineEscaping, type InlineSegment, type MarkRun, type NodeRange } from './line-escaping.ts'
import { carriedInline } from '../opaque-carry.ts'
import { claimsLine, holdsNullCharacter, trimTrailingSpace } from '../commonmark/grammar.ts'
import { holdsNullCharacter, trimTrailingSpace } from '../commonmark/grammar.ts'
import { commonMarkLink, linkHref, markSpelling, spellMarkAttributes } from '../mark-spellings.ts'
import { identicalMark, isBareText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
import { escapeUnbalanced, spellDestination } from '../commonmark/link-syntax.ts'
import { identicalMark, isBareText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
import { highlightDelimiter } from '../plain/conventions.ts'
import { inlineNodeModel } from '../../adf/inline-nodes.ts'
import { joinsWhenRead, textBreakSpelling } from '../adjacent-text.ts'
import { largestNesting } from '../../nesting.ts'
import { longestBacktickRun } from '../commonmark/backtick-runs.ts'
import { slotLineEndingFault, spellInlineDirectiveOpener, spellInlineLeafDirective } from '../directive-syntax.ts'
import { slotFault, spellInlineDirectiveOpener, spellInlineLeafDirective } from '../directive-syntax.ts'
import { spellInlineNodeAttributes } from './inline-directive-spelling.ts'
import { spellTextDirective } from '../text-directive.ts'
@@ -38,7 +38,7 @@ type LineAttempt = { fallback: NodeRange | 'opening-link'; line?: undefined } |
type LineFallbacks = { carried: Set<number>; flavour: Flavour; openingLinkAsDirective: boolean }
export type PlainLineFallback = { kind: 'claimed-line'; line: number; text: string } | { kind: 'opening-link' } | { kind: 'unspellable-run'; runs: [MarkRun, ...MarkRun[]] }
export type PlainLineFallback = { kind: 'opening-link' } | { kind: 'unspellable-run'; runs: [MarkRun, ...MarkRun[]] }
export function emitInlineLine(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath, flavour: Flavour): Result<string> {
const emitted = emitLine(nodes, container, path, flavour)
@@ -119,19 +119,7 @@ function attemptLine(segments: readonly InlineSegment[], container: LineContaine
const verdict = lineVerdict(segments, container, flavour)
if (verdict.kind === 'opening-link') return { fallback: 'opening-link' }
if (verdict.kind === 'unspellable-run') return { fallback: verdict.runs[0] }
if (verdict.kind === 'line') return { line: verdict.text }
return { fallback: lineOpener(segments, verdict.line) }
}
// A code span's backticks are what block parsing claims, and its segment names its node; a segment naming none leaves takeFallback to refuse the line.
function lineOpener(segments: readonly InlineSegment[], line: number): NodeRange {
let breaks = 0
const opener = segments.find((segment) => {
const opens = breaks === line && segment.text !== ''
breaks += segment.text.split('\n').length - 1
return opens
})
return opener?.nodes ?? { first: 0, last: -1 }
return { line: verdict.text }
}
// The fallbacks in the order a line takes them, or the line where it takes none.
@@ -139,10 +127,7 @@ function lineVerdict(segments: readonly InlineSegment[], container: LineContaine
const assembled = assembleInlineLine(segments, container, flavour)
if (assembled.openingLinkAsDirective) return { kind: 'opening-link' }
const [run, ...others] = assembled.unspellableRuns
if (run !== undefined) return { kind: 'unspellable-run', runs: [run, ...others] }
const lines = assembled.line.split('\n')
const claimed = container === 'paragraph' ? lines.findIndex((single, index) => claimsLine(single, index === 0 ? 'first' : 'later')) : -1
return claimed === -1 ? { kind: 'line', text: assembled.line } : { kind: 'claimed-line', line: claimed, text: lines[claimed] ?? '' }
return run === undefined ? { kind: 'line', text: assembled.line } : { kind: 'unspellable-run', runs: [run, ...others] }
}
// spec/flavour.md, Inline nodes.
@@ -269,7 +254,7 @@ function emitInlineDirective(node: AdfNode, model: InlineNodeModel, index: numbe
const slot = model.textAttribute === undefined ? undefined : nodeAttrs(node)[model.textAttribute]
if (slot === undefined) return success({ segments: [syntax(spellInlineLeafDirective(node.type, attributes))] })
if (typeof slot !== 'string') return success({ carry: { first: index, last: index } })
if (slotLineEndingFault(node.type, slot) !== undefined || holdsNullCharacter(slot)) return success({ carry: { first: index, last: index } })
if (slotFault(node.type, slot) !== undefined) return success({ carry: { first: index, last: index } })
const content: InlineSegment[] = slot === '' ? [] : [{ escaping: 'bracketed', text: slot }]
return success({ segments: [syntax(spellInlineDirectiveOpener(node.type)), ...content, syntax(`]${attributes}`)] })
}
@@ -324,16 +309,14 @@ function emitHighlight(nodes: readonly AdfNode[], depth: number, range: NodeRang
// Each node its own span: CommonMark reads two adjacent text nodes in one span back as one.
function emitCodeSpan(nodes: readonly AdfNode[], depth: number, range: NodeRange): Emission {
const segments: InlineSegment[] = []
for (const [offset, node] of nodes.entries()) {
const spans: string[] = []
for (const node of nodes) {
const { text } = node
if (!isBareText(node) || nodeMarks(node).length !== depth + 1 || typeof text !== 'string' || text === '' || /[\n\r]/.test(text) || holdsNullCharacter(text)) return { carry: range }
if (offset > 0) segments.push(syntax(textBreakSpelling))
const fence = '`'.repeat(longestBacktickRun(text) + 1)
const index = range.first + offset
segments.push({ escaping: 'none', nodes: { first: index, last: index }, text: `${fence}${needsPadding(text) ? ` ${text} ` : text}${fence}` })
spans.push(`${fence}${needsPadding(text) ? ` ${text} ` : text}${fence}`)
}
return { segments }
return { segments: [syntax(spans.join(textBreakSpelling))] }
}
function needsPadding(text: string): boolean {
src/markdown/parse/directive-nodes.ts
+3 -3
View File
@@ -13,7 +13,7 @@ import { inlineMarkSpellingFault } from './directive-marks.ts'
import { inlineNodeModel } from '../../adf/inline-nodes.ts'
import { readEmptyKeys, spellsEmpty } from '../empty-keys.ts'
import { readVocabulary } from './directive-attributes.ts'
import { slotLineEndingFault } from '../directive-syntax.ts'
import { slotFault } from '../directive-syntax.ts'
import { textBreakName } from '../adjacent-text.ts'
import { textDirectiveName } from '../text-directive.ts'
@@ -68,8 +68,8 @@ export function readInlineDirectiveNode(
if (text === undefined) {
return failure('unsupported-node-shape', `the ${name} content slot holds one text node carrying neither marks, attributes nor content: this one holds something else`, path)
}
const spans = slotLineEndingFault(name, text)
if (spans !== undefined) return faulted(spans, path)
const fault = slotFault(name, text)
if (fault !== undefined) return faulted(fault, path)
attrs.value[slot] = text
}
return success(namedNode(name, attrs.value, undefined, empty.value.empty))
src/markdown/parse/markdown-to-adf.test.ts
+2 -1
View File
@@ -304,7 +304,7 @@ test('names the directive form a node CommonMark spells refuses', () => {
assert.deepEqual(content(markdownToAdf('!adf:bulletList\n!adf:listItem\n---\n!adf:/listItem\n!adf:/bulletList\n')), [bulletList(item({ type: 'rule' }))])
})
test('refuses the directive form of a blockquote opening with a code span the carry spells', () => {
test('refuses the directive form of a blockquote opening with a code span', () => {
assert.equal(content(markdownToAdf('!adf:blockquote\n` `` `\n!adf:/blockquote\n')), 'unsupported-node-shape: blockquote takes the CommonMark spelling, not the directive form')
})
@@ -1005,6 +1005,7 @@ test('names the content slot no lone plain text node reads back from', () => {
assert.equal(content(markdownToAdf('!adf:status[!adf:text{text="\\n"}]{color=yellow}\n')), spans)
assert.equal(content(markdownToAdf('!adf:status[a&#10;b]{color=yellow}\n')), spans)
assert.equal(content(markdownToAdf('!adf:status[a&#13;b]{color=yellow}\n')), spans)
assert.equal(content(markdownToAdf('!adf:status[!adf:text{text="\\u0000"}]{color=yellow}\n')), 'unsupported-node-shape: the status content slot holds a null character: write the node as !adf:carry')
assert.equal(content(markdownToAdf('Part !adf:mention{id=b1c2 text=A}.\n')), 'unsupported-node-shape: mention spells its text attribute in the content slot, never in {attrs}')
})
src/markdown/plain/adf-to-plain-markdown.test.ts
+4 -4
View File
@@ -313,12 +313,12 @@ test('breaks a line at a newline and trims whitespace at every edge CommonMark s
assert.equal(plain(paragraph(text(' x ', code))), '` x `\n')
assert.equal(plain(node('heading', { level: 1 }, text(' h\ni '))), '# h i\n')
assert.equal(plain(paragraph(text('a\r\u0000b'))), 'ab\n')
assert.equal(plain(paragraph(text('x', link('/u', 't\r\u0000')))), '[x](/u "t")\n')
})
test('drops the code mark of a span opening a line with backticks that read as a fence', () => {
assert.equal(plain(paragraph(text('``` x', code))), '\\`\\`\\` x\n')
assert.equal(plain(paragraph(text('a\n'), text('``` x', code))), 'a\\\n\\`\\`\\` x\n')
assert.equal(plain(paragraph(text('a '), text('``` x', code))), 'a ```` ``` x ````\n')
test('keeps a code span opening a line, whose backticks open no fence', () => {
assert.equal(plain(paragraph(text('``` x', code))), '```` ``` x ````\n')
assert.equal(plain(paragraph(text('a\n'), text('``` x', code))), 'a\\\n```` ``` x ````\n')
})
test('drops an empty paragraph and merges adjacent lists of one type', () => {
src/markdown/plain/inline-reduction.ts
+6 -14
View File
@@ -149,7 +149,7 @@ function plainMarks(marks: readonly AdfMark[], container: LineContainer, text: s
function plainLink(mark: AdfMark, container: LineContainer): AdfMark | undefined {
const attrs = nodeAttrs(mark)
const held = attrs['href']
const title = typeof attrs['title'] === 'string' ? attrs['title'].replace(/\r/g, '').replace(/\n/g, ' ') : undefined
const title = typeof attrs['title'] === 'string' ? attrs['title'].replace(/[\r\0]/g, '').replace(/\n/g, ' ') : undefined
if (typeof held !== 'string') return undefined
const href = writableHref(container === 'table-cell' ? held.replaceAll('|', '%7C') : held)
if (title === undefined || spellLinkTarget(href, title) === undefined || (container === 'table-cell' && title.includes('|'))) return { attrs: { href }, type: 'link' }
@@ -267,21 +267,13 @@ function spellableLine(leaves: AdfNode[], container: LineContainer, path: Conver
function withoutFallback(leaves: readonly AdfNode[], fallback: PlainLineFallback): AdfNode[] | undefined {
if (fallback.kind === 'unspellable-run') return withoutMarks(leaves, fallback.runs)
const first = fallback.kind === 'opening-link' ? 0 : lineStart(leaves, fallback.line)
const mark = nodeMarks(leaves[first] ?? {})[0]
if (mark === undefined || mark.type !== (fallback.kind === 'opening-link' ? 'link' : 'code')) return undefined
let last = first
const mark = nodeMarks(leaves[0] ?? {})[0]
if (mark === undefined || mark.type !== 'link') return undefined
let last = 0
while (sameMarkAt(nodeMarks(leaves[last + 1] ?? {}), [mark], 0)) last += 1
// A code span is what binds the `]` a link definition reads, and dropping it keeps the link target.
const spans = leaves.slice(first, last + 1).some((leaf) => nodeMarks(leaf).length > 1 && nodeMarks(leaf).at(-1)?.type === 'code')
if (mark.type === 'link' && spans) return leaves.map((leaf, index) => (index < first || index > last ? leaf : withMarks(leaf, nodeMarks(leaf).filter((held) => held.type !== 'code'))))
return withoutMarks(leaves, [{ depth: 0, first, last }])
}
function lineStart(leaves: readonly AdfNode[], line: number): number {
let index = 0
for (let breaks = 0; breaks < line && index < leaves.length; index += 1) if (leaves[index]?.type === 'hardBreak') breaks += 1
return index
const spans = leaves.slice(0, last + 1).some((leaf) => nodeMarks(leaf).length > 1 && nodeMarks(leaf).at(-1)?.type === 'code')
return spans ? leaves.map((leaf, index) => (index > last ? leaf : withMarks(leaf, nodeMarks(leaf).filter((held) => held.type !== 'code')))) : undefined
}
// The runs cover disjoint leaves, so one pass drops them all.