Emitter 2a: the corpus runner, the canonical serializer and the CommonMark subset #9
@@ -139,7 +139,8 @@ One-line commit messages and PR titles; short PR summaries. No AI-attribution ma
|
|||||||
## 14. Non-goals
|
## 14. Non-goals
|
||||||
|
|
||||||
No wiki markup (§1), no network or filesystem I/O, no name→id resolution (§3), no ADF schema
|
No wiki markup (§1), no network or filesystem I/O, no name→id resolution (§3), no ADF schema
|
||||||
validation or exported validator, no shipped CSS (§4), no streaming APIs, no performance budget —
|
validation or exported validator — a refusal that keeps the round-trip is not schema validation,
|
||||||
|
so the one a node carrying the same mark type twice earns stays, no shipped CSS (§4), no streaming APIs, no performance budget —
|
||||||
conversions are O(n), real documents are kilobytes. A CLI is a later goal (`todo.md`), not a
|
conversions are O(n), real documents are kilobytes. A CLI is a later goal (`todo.md`), not a
|
||||||
non-goal.
|
non-goal.
|
||||||
|
|
||||||
|
|||||||
@@ -193,9 +193,23 @@ test('refuses a mark spelling that cannot open or close where it sits', () => {
|
|||||||
const em: AdfMark = { type: 'em' }
|
const em: AdfMark = { type: 'em' }
|
||||||
assert.equal(code(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, marked('a.', em), marked('b', strong))))), 'unspellable-mark')
|
assert.equal(code(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, marked('a.', em), marked('b', strong))))), 'unspellable-mark')
|
||||||
assert.equal(emitted({ text: 'x', type: 'text' }, marked('ab', em, strong), { text: 'y', type: 'text' }), 'x***ab***y\n')
|
assert.equal(emitted({ text: 'x', type: 'text' }, marked('ab', em, strong), { text: 'y', type: 'text' }), 'x***ab***y\n')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('refuses a node carrying one mark type twice', () => {
|
||||||
|
const em: AdfMark = { type: 'em' }
|
||||||
assert.equal(code(adfToMarkdown(document(paragraph({ marks: [em, em], text: 'x', type: 'text' })))), 'unsupported-node-shape')
|
assert.equal(code(adfToMarkdown(document(paragraph({ marks: [em, em], text: 'x', type: 'text' })))), 'unsupported-node-shape')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('refuses a nested list the tight spelling would swallow', () => {
|
||||||
|
const item = (...content: AdfNode[]): AdfNode => ({ content, type: 'listItem' })
|
||||||
|
const text = (value: string): AdfNode => ({ content: [{ text: value, type: 'text' }], type: 'paragraph' })
|
||||||
|
const outer = (...content: AdfNode[]): AdfDocument => document({ content: [item(...content)], type: 'bulletList' })
|
||||||
|
const ordered: AdfNode = { attrs: { order: 2 }, content: [item(text('b'))], type: 'orderedList' }
|
||||||
|
assert.equal(code(adfToMarkdown(outer(text('a'), ordered))), 'unspellable-line-start')
|
||||||
|
assert.equal(code(adfToMarkdown(outer(text('a'), { content: [item()], type: 'bulletList' }))), 'unspellable-line-start')
|
||||||
|
assert.equal(markdown(adfToMarkdown(outer(text('a'), { content: [item(text('b'))], type: 'bulletList' }))), '- a\n - b\n')
|
||||||
|
})
|
||||||
|
|
||||||
test('refuses marks and attributes nested deeper than the emitter carries', () => {
|
test('refuses marks and attributes nested deeper than the emitter carries', () => {
|
||||||
const marks: AdfMark[] = Array.from({ length: 600 }, (_, index) => ({ type: index % 2 === 0 ? 'em' : 'strong' }))
|
const marks: AdfMark[] = Array.from({ length: 600 }, (_, index) => ({ type: index % 2 === 0 ? 'em' : 'strong' }))
|
||||||
assert.equal(code(adfToMarkdown(document(paragraph({ marks, text: 'x', type: 'text' })))), 'unsupported-node-shape')
|
assert.equal(code(adfToMarkdown(document(paragraph({ marks, text: 'x', type: 'text' })))), 'unsupported-node-shape')
|
||||||
|
|||||||
+11
-1
@@ -28,7 +28,12 @@ function emitBlocks(nodes: readonly AdfNode[], inListItem: boolean, path: Conver
|
|||||||
if (listTypes.includes(node.type) && previous.type === node.type) {
|
if (listTypes.includes(node.type) && previous.type === node.type) {
|
||||||
return failure('unspellable-adjacent-lists', `two adjacent ${node.type} nodes read back as one list`, nodePath)
|
return failure('unspellable-adjacent-lists', `two adjacent ${node.type} nodes read back as one list`, nodePath)
|
||||||
}
|
}
|
||||||
output += inListItem && listTypes.includes(node.type) ? '\n' : '\n\n'
|
if (inListItem && listTypes.includes(node.type)) {
|
||||||
|
if (!interruptsParagraph(node)) {
|
||||||
|
return failure('unspellable-line-start', `a ${node.type} that cannot interrupt the block above it has no tight spelling`, nodePath)
|
||||||
|
}
|
||||||
|
output += '\n'
|
||||||
|
} else output += '\n\n'
|
||||||
}
|
}
|
||||||
const block = emitBlock(node, nodePath, depth)
|
const block = emitBlock(node, nodePath, depth)
|
||||||
if (!block.ok) return block
|
if (!block.ok) return block
|
||||||
@@ -38,6 +43,11 @@ function emitBlocks(nodes: readonly AdfNode[], inListItem: boolean, path: Conver
|
|||||||
return success(output)
|
return success(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function interruptsParagraph(node: AdfNode): boolean {
|
||||||
|
if (node.type === 'orderedList') return false
|
||||||
|
return ((node.content ?? [])[0]?.content ?? []).length > 0
|
||||||
|
}
|
||||||
|
|
||||||
function emitBlock(node: AdfNode, path: ConvertErrorPath, depth: number): Result<string> {
|
function emitBlock(node: AdfNode, path: ConvertErrorPath, depth: number): Result<string> {
|
||||||
if (node.type === 'blockquote') return emitBlockquote(node, path, depth)
|
if (node.type === 'blockquote') return emitBlockquote(node, path, depth)
|
||||||
if (node.type === 'bulletList' || node.type === 'orderedList') return emitList(node, path, depth)
|
if (node.type === 'bulletList' || node.type === 'orderedList') return emitList(node, path, depth)
|
||||||
|
|||||||
+15
-12
@@ -1,16 +1,14 @@
|
|||||||
import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
|
import { escapesLineClaim, isUnicodeWhitespace, opensBracketedAutolink, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
|
||||||
|
|
||||||
export type InlineSegment = {
|
export type InlineSegment =
|
||||||
kind: 'emphasis-close' | 'emphasis-open' | 'link-text' | 'literal' | 'syntax'
|
| { kind: 'emphasis-close' | 'emphasis-open'; mark: string; text: string }
|
||||||
mark?: string
|
| { kind: 'link-text' | 'literal' | 'syntax'; text: string }
|
||||||
text: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export type AssembledLine = { line: string; unspellableMark: string | undefined }
|
export type AssembledLine = { line: string; unspellableMark: string | undefined }
|
||||||
|
|
||||||
export type LineContainer = 'heading' | 'paragraph'
|
export type LineContainer = 'heading' | 'paragraph'
|
||||||
|
|
||||||
type DelimiterRun = { character: string; closes: boolean; end: number; mark: string; opens: boolean; start: number }
|
type DelimiterRun = { character: string; closeMark: string | undefined; end: number; openMark: string | undefined; start: number }
|
||||||
|
|
||||||
const delimiters = ['*', '_', '`', '~']
|
const delimiters = ['*', '_', '`', '~']
|
||||||
|
|
||||||
@@ -77,8 +75,8 @@ function unspellableMark(segments: readonly InlineSegment[], output: string, pla
|
|||||||
for (const run of delimiterRuns(segments, placements)) {
|
for (const run of delimiterRuns(segments, placements)) {
|
||||||
const before = charAt(output, run.start - 1)
|
const before = charAt(output, run.start - 1)
|
||||||
const after = output.charAt(run.end)
|
const after = output.charAt(run.end)
|
||||||
if (run.opens && !isLeftFlanking(before, after)) return run.mark
|
if (run.openMark !== undefined && !isLeftFlanking(before, after)) return run.openMark
|
||||||
if (run.closes && !isRightFlanking(before, after)) return run.mark
|
if (run.closeMark !== undefined && !isRightFlanking(before, after)) return run.closeMark
|
||||||
}
|
}
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
@@ -92,15 +90,20 @@ function delimiterRuns(segments: readonly InlineSegment[], placements: readonly
|
|||||||
if (segment.kind !== 'emphasis-close' && segment.kind !== 'emphasis-open') continue
|
if (segment.kind !== 'emphasis-close' && segment.kind !== 'emphasis-open') continue
|
||||||
const closes = segment.kind === 'emphasis-close'
|
const closes = segment.kind === 'emphasis-close'
|
||||||
const end = start + segment.text.length
|
const end = start + segment.text.length
|
||||||
const mark = segment.mark ?? segment.text
|
|
||||||
const previous = runs[runs.length - 1]
|
const previous = runs[runs.length - 1]
|
||||||
if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) {
|
if (previous !== undefined && previous.end === start && previous.character === segment.text.charAt(0)) {
|
||||||
previous.closes = previous.closes || closes
|
previous.closeMark = previous.closeMark ?? (closes ? segment.mark : undefined)
|
||||||
previous.end = end
|
previous.end = end
|
||||||
previous.opens = previous.opens || !closes
|
previous.openMark = previous.openMark ?? (closes ? undefined : segment.mark)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
runs.push({ character: segment.text.charAt(0), closes, end, mark, opens: !closes, start })
|
runs.push({
|
||||||
|
character: segment.text.charAt(0),
|
||||||
|
closeMark: closes ? segment.mark : undefined,
|
||||||
|
end,
|
||||||
|
openMark: closes ? undefined : segment.mark,
|
||||||
|
start,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return runs
|
return runs
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,7 +60,9 @@ detail is settled at its own milestone.
|
|||||||
itself under the library's own canonical serializer — one implementation, keys sorted, two
|
itself under the library's own canonical serializer — one implementation, keys sorted, two
|
||||||
spellings: two-space indent for the corpus files and the block carry's body, compact for
|
spellings: two-space indent for the corpus files and the block carry's body, compact for
|
||||||
the inline carry. `commonmark-subset/` green.
|
the inline carry. `commonmark-subset/` green.
|
||||||
- [ ] **2b — Block nodes.** `block-nodes/` green. Block separation becomes
|
- [ ] **2b — Block nodes.** `block-nodes/` green. A nested list that cannot interrupt the block
|
||||||
|
above it is refused meanwhile, not spelled: the maintainer's answer on tight-versus-blank
|
||||||
|
separation turns that refusal into an emission. Block separation becomes
|
||||||
`separationBetween(previous, next, container)` here — a boolean cannot hold the third case
|
`separationBetween(previous, next, container)` here — a boolean cannot hold the third case
|
||||||
`spec/flavour.md` states for two directive blocks in a container body, and the maintainer's
|
`spec/flavour.md` states for two directive blocks in a container body, and the maintainer's
|
||||||
answer on a CommonMark block beside a directive block (1d) drops into the same seam. Give
|
answer on a CommonMark block beside a directive block (1d) drops into the same seam. Give
|
||||||
@@ -81,7 +83,10 @@ detail is settled at its own milestone.
|
|||||||
paragraph line inside a container body shaped like a closing fence (`:::`, `::: x`).
|
paragraph line inside a container body shaped like a closing fence (`:::`, `::: x`).
|
||||||
The gate gains the collision property here: no two corpus documents may emit the same
|
The gate gains the collision property here: no two corpus documents may emit the same
|
||||||
bytes — one spelling for two documents is a round-trip break no parser can undo, and it is
|
bytes — one spelling for two documents is a round-trip break no parser can undo, and it is
|
||||||
provable without one.
|
provable without one. It also settles the emitter's one known approximation: delimiter
|
||||||
|
flanking is exact, but CommonMark's *matching* — the multiple-of-3 rule and the way a run
|
||||||
|
splits across several openers — is not modelled. No reachable violation has been found by
|
||||||
|
hand; the property test is what decides it.
|
||||||
- [ ] **3 — `markdownToAdf`.** The CommonMark parser is the largest single component; split it
|
- [ ] **3 — `markdownToAdf`.** The CommonMark parser is the largest single component; split it
|
||||||
into sub-items before starting (§15). Fixtures land with the code that reads them:
|
into sub-items before starting (§15). Fixtures land with the code that reads them:
|
||||||
`corpus/normalization/` (setext, indented code, loose lists, `*`/`+` bullets, entity
|
`corpus/normalization/` (setext, indented code, loose lists, `*`/`+` bullets, entity
|
||||||
@@ -90,7 +95,9 @@ detail is settled at its own milestone.
|
|||||||
line that does not parse, the content slot, raw HTML with no mapping — each with the error
|
line that does not parse, the content slot, raw HTML with no mapping — each with the error
|
||||||
it must return). The raw-HTML element mapping is empty until milestone 6, so at `0.1.0`
|
it must return). The raw-HTML element mapping is empty until milestone 6, so at `0.1.0`
|
||||||
every raw-HTML construct in input is an error result. The CommonMark spec suite runs
|
every raw-HTML construct in input is an error result. The CommonMark spec suite runs
|
||||||
against it from here (§10). `src/` gets its hierarchy at the same split — `adf/`,
|
against it from here (§10). The parser owes `~` the same `can_open`/`can_close` the emitter
|
||||||
|
assumes — CommonMark flanking, as for `*` — which `spec/flavour.md` does not yet pin.
|
||||||
|
`src/` gets its hierarchy at the same split — `adf/`,
|
||||||
`markdown/`, `html/`, the grammar module shared inside `markdown/` — while the rename is
|
`markdown/`, `html/`, the grammar module shared inside `markdown/` — while the rename is
|
||||||
still mechanical.
|
still mechanical.
|
||||||
- [ ] **4 — Round-trip property tests** over the corpus, both ways — the thing that proves 2 and
|
- [ ] **4 — Round-trip property tests** over the corpus, both ways — the thing that proves 2 and
|
||||||
|
|||||||
Reference in New Issue
Block a user