Emitter 2a: the corpus runner, the canonical serializer and the CommonMark subset #9

Merged
lilleman merged 7 commits from emitter-commonmark-subset into main 2026-08-25 10:03:50 +02:00
14 changed files with 205 additions and 126 deletions
Showing only changes of commit e92484eb85 - Show all commits
+5
View File
@@ -87,6 +87,11 @@ Test for the behaviour wanted first, then implement until green. `node --test`,
Node, tsc and npm never run on the host — only via the pinned images (§9). Tests are independent, Node, tsc and npm never run on the host — only via the pinned images (§9). Tests are independent,
coverage does not decline, containers are torn down after a run. coverage does not decline, containers are torn down after a run.
The floors live in the `test` script, so `npm test` and the gate are one path: 100% of lines and
functions, and a branch floor that only ever moves upward. It sits below 100 because the guards
`noUncheckedIndexedAccess` and ADF's optional keys force — `?? []`, `?? {}`, `?.`, an index
compared against `undefined` — have a half no valid document reaches.
The corpus, all checked in: hand-built fixtures per node and combination; real sanitized ADF from The corpus, all checked in: hand-built fixtures per node and combination; real sanitized ADF from
live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite against live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite against
`markdownToAdf` and `markdownToHtml`. `markdownToAdf` and `markdownToHtml`.
+1 -1
View File
@@ -13,7 +13,7 @@
"node": ">=24" "node": ">=24"
}, },
"scripts": { "scripts": {
"test": "node --test --experimental-test-coverage --test-coverage-exclude=\"src/**/*.test.ts\" --test-coverage-branches=89 --test-coverage-functions=100 --test-coverage-lines=100 \"src/**/*.test.ts\"", "test": "node --test --experimental-test-coverage --test-coverage-exclude=\"src/**/*.test.ts\" --test-coverage-branches=91 --test-coverage-functions=100 --test-coverage-lines=100 \"src/**/*.test.ts\"",
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"devDependencies": { "devDependencies": {
+2 -2
View File
@@ -1,4 +1,4 @@
import { isJsonValue, type JsonValue } from './canonical-json.ts' import { isJsonValue, type JsonValue } from './json-value.ts'
export type AdfAttributes = { [key: string]: JsonValue } export type AdfAttributes = { [key: string]: JsonValue }
@@ -48,7 +48,7 @@ function isAdfNode(value: unknown): value is AdfNode {
} }
function isArrayOf<T>(value: unknown, guard: (item: unknown) => item is T): value is T[] { function isArrayOf<T>(value: unknown, guard: (item: unknown) => item is T): value is T[] {
return Array.isArray(value) && value.every(guard) return Array.isArray(value) && [...value].every(guard)
} }
function isAttributes(value: unknown): value is AdfAttributes { function isAttributes(value: unknown): value is AdfAttributes {
+19 -4
View File
@@ -1,7 +1,7 @@
import assert from 'node:assert/strict' import assert from 'node:assert/strict'
import test from 'node:test' import test from 'node:test'
import type { AdfDocument, AdfNode } from './adf-document.ts' import type { AdfDocument, AdfMark, AdfNode } from './adf-document.ts'
import type { Result } from './result.ts' import type { Result } from './result.ts'
import { adfToMarkdown } from './index.ts' import { adfToMarkdown } from './index.ts'
@@ -109,11 +109,17 @@ test('refuses a node whose content model the canonical form cannot emit', () =>
assert.equal(code(adfToMarkdown(document({ type: 'listItem' }))), 'unsupported-node-shape') assert.equal(code(adfToMarkdown(document({ type: 'listItem' }))), 'unsupported-node-shape')
assert.equal(code(adfToMarkdown(document({ content: [paragraph()], type: 'codeBlock' }))), 'unsupported-node-shape') assert.equal(code(adfToMarkdown(document({ content: [paragraph()], type: 'codeBlock' }))), 'unsupported-node-shape')
assert.equal(code(adfToMarkdown(document({ content: [paragraph()], type: 'bulletList' }))), 'unsupported-node-shape') assert.equal(code(adfToMarkdown(document({ content: [paragraph()], type: 'bulletList' }))), 'unsupported-node-shape')
assert.equal(code(adfToMarkdown(document({ type: 'bulletList' }))), 'unsupported-node-shape')
assert.equal(code(adfToMarkdown(document({ attrs: { order: 2 }, content: [], type: 'orderedList' }))), 'unsupported-node-shape')
}) })
test('refuses an ordered list start no marker spells', () => { test('refuses an ordered list no marker spells', () => {
const items: AdfNode[] = [{ content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }] const item: AdfNode = { content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }
assert.equal(code(adfToMarkdown(document({ attrs: { order: 1.5 }, content: items, type: 'orderedList' }))), 'unsupported-node-shape') const list = (order: number, items: number): AdfDocument =>
document({ attrs: { order }, content: Array.from({ length: items }, () => item), type: 'orderedList' })
assert.equal(code(adfToMarkdown(list(1.5, 1))), 'unsupported-node-shape')
assert.equal(markdown(adfToMarkdown(list(999999999, 1))), '999999999. x\n')
assert.equal(code(adfToMarkdown(list(999999999, 2))), 'unspellable-list-marker')
}) })
test('refuses a code span over anything but one text node', () => { test('refuses a code span over anything but one text node', () => {
@@ -150,6 +156,15 @@ test('escapes a heading closing sequence', () => {
assert.equal(heading('#tag first'), '## #tag first\n') assert.equal(heading('#tag first'), '## #tag first\n')
}) })
test('wraps adjacent nodes carrying one mark once, and a differing mark twice', () => {
const marked = (text: string, ...marks: AdfMark[]): AdfNode => ({ marks, text, type: 'text' })
const emitted = (...content: AdfNode[]): string => markdown(adfToMarkdown(document(paragraph(...content))))
assert.equal(emitted(marked('a', { type: 'strong' }), marked('b', { type: 'strong' }, { type: 'em' })), '**a*b***\n')
assert.equal(emitted(marked('a', { type: 'strong' }), marked('b', { type: 'em' })), '**a**_b_\n')
const link = (href: string): AdfMark => ({ attrs: { href }, type: 'link' })
assert.equal(emitted(marked('a', link('http://x')), marked('b', link('http://y'))), '[a](http://x)[b](http://y)\n')
})
test('emits an empty list item without trailing whitespace', () => { test('emits an empty list item without trailing whitespace', () => {
assert.equal(markdown(adfToMarkdown(document({ content: [{ type: 'listItem' }], type: 'bulletList' }))), '-\n') assert.equal(markdown(adfToMarkdown(document({ content: [{ type: 'listItem' }], type: 'bulletList' }))), '-\n')
}) })
+59 -47
View File
@@ -1,9 +1,11 @@
import type { AdfDocument, AdfNode } from './adf-document.ts' import type { AdfDocument, AdfNode } from './adf-document.ts'
import type { JsonValue } from './json-value.ts'
import { emitInlineLine } from './markdown-inline.ts' import { emitInlineLine } from './markdown-inline.ts'
import { failure, success, type Result } from './result.ts' import { failure, success, type Result } from './result.ts'
import { isAdfDocument } from './adf-document.ts' import { isAdfDocument } from './adf-document.ts'
import { longestBacktickRun } from './backtick-runs.ts' import { longestBacktickRun } from './backtick-runs.ts'
const largestListMarker = 999999999
const listTypes = ['bulletList', 'orderedList'] const listTypes = ['bulletList', 'orderedList']
export function adfToMarkdown(document: AdfDocument): Result<string> { export function adfToMarkdown(document: AdfDocument): Result<string> {
@@ -16,10 +18,8 @@ export function adfToMarkdown(document: AdfDocument): Result<string> {
function emitBlocks(nodes: readonly AdfNode[], inListItem: boolean): Result<string> { function emitBlocks(nodes: readonly AdfNode[], inListItem: boolean): Result<string> {
let output = '' let output = ''
for (let index = 0; index < nodes.length; index += 1) { let previous: AdfNode | undefined
const node = nodes[index] for (const node of nodes) {
if (node === undefined) return failure('unsupported-node-shape', 'the block content holds a hole')
const previous = nodes[index - 1]
if (previous !== undefined) { if (previous !== undefined) {
if (listTypes.includes(node.type) && previous.type === node.type) { if (listTypes.includes(node.type) && previous.type === node.type) {
return failure('unspellable-adjacent-lists', `two adjacent ${node.type} nodes read back as one list`) return failure('unspellable-adjacent-lists', `two adjacent ${node.type} nodes read back as one list`)
@@ -29,6 +29,7 @@ function emitBlocks(nodes: readonly AdfNode[], inListItem: boolean): Result<stri
const block = emitBlock(node) const block = emitBlock(node)
if (!block.ok) return block if (!block.ok) return block
output += block.value output += block.value
previous = node
} }
return success(output) return success(output)
} }
@@ -47,8 +48,8 @@ function emitBlock(node: AdfNode): Result<string> {
} }
function emitBlockquote(node: AdfNode): Result<string> { function emitBlockquote(node: AdfNode): Result<string> {
const invalid = validateBlockNode(node, []) const validation = validateBlockNode(node, [])
if (!invalid.ok) return invalid if (!validation.ok) return validation
const inner = emitBlocks(node.content ?? [], false) const inner = emitBlocks(node.content ?? [], false)
if (!inner.ok) return inner if (!inner.ok) return inner
return success( return success(
@@ -60,17 +61,10 @@ function emitBlockquote(node: AdfNode): Result<string> {
} }
function emitCodeBlock(node: AdfNode): Result<string> { function emitCodeBlock(node: AdfNode): Result<string> {
const invalid = validateBlockNode(node, ['language']) const validation = validateBlockNode(node, ['language'])
if (!invalid.ok) return invalid if (!validation.ok) return validation
const language = node.attrs?.['language'] const info = spellCodeFenceInfo(node.attrs?.['language'])
if (language !== undefined) { if (!info.ok) return info
if (typeof language !== 'string') return failure('unsupported-node-shape', 'a codeBlock language is no string')
if (language === '') return failure('ambiguous-empty-code-block-language', 'an empty codeBlock language and an absent one share one markdown spelling')
if (language === 'adf') return failure('reserved-adf-language', 'the adf info string is reserved for the opaque carry')
if (/[`\n\r]/.test(language) || language !== language.trim()) {
return failure('unspellable-code-block-language', 'a fence info string holds no backtick and no edge whitespace')
}
}
let text = '' let text = ''
for (const child of node.content ?? []) { for (const child of node.content ?? []) {
if (child.type !== 'text' || typeof child.text !== 'string' || (child.marks ?? []).length > 0 || Object.keys(child.attrs ?? {}).length > 0) { if (child.type !== 'text' || typeof child.text !== 'string' || (child.marks ?? []).length > 0 || Object.keys(child.attrs ?? {}).length > 0) {
@@ -79,13 +73,24 @@ function emitCodeBlock(node: AdfNode): Result<string> {
text += child.text text += child.text
} }
const fence = '`'.repeat(Math.max(3, longestBacktickRun(text) + 1)) const fence = '`'.repeat(Math.max(3, longestBacktickRun(text) + 1))
const opening = `${fence}${language ?? ''}` const opening = `${fence}${info.value}`
return success(text === '' ? `${opening}\n${fence}` : `${opening}\n${text}\n${fence}`) return success(text === '' ? `${opening}\n${fence}` : `${opening}\n${text}\n${fence}`)
} }
function spellCodeFenceInfo(language: JsonValue | undefined): Result<string> {
if (language === undefined) return success('')
if (typeof language !== 'string') return failure('unsupported-node-shape', 'a codeBlock language is no string')
if (language === '') return failure('ambiguous-empty-code-block-language', 'an empty codeBlock language and an absent one share one markdown spelling')
if (language === 'adf') return failure('reserved-adf-language', 'the adf info string is reserved for the opaque carry')
if (/[`\n\r]/.test(language) || language !== language.trim()) {
return failure('unspellable-code-block-language', 'a fence info string holds no backtick and no edge whitespace')
}
return success(language)
}
function emitHeading(node: AdfNode): Result<string> { function emitHeading(node: AdfNode): Result<string> {
const invalid = validateBlockNode(node, ['level']) const validation = validateBlockNode(node, ['level'])
if (!invalid.ok) return invalid if (!validation.ok) return validation
const level = node.attrs?.['level'] const level = node.attrs?.['level']
if (typeof level !== 'number' || !Number.isInteger(level) || level < 1 || level > 6) { if (typeof level !== 'number' || !Number.isInteger(level) || level < 1 || level > 6) {
return failure('unsupported-heading-level', `no ATX heading spells level ${JSON.stringify(level ?? null)}`) return failure('unsupported-heading-level', `no ATX heading spells level ${JSON.stringify(level ?? null)}`)
@@ -100,49 +105,56 @@ function emitHeading(node: AdfNode): Result<string> {
function emitList(node: AdfNode): Result<string> { function emitList(node: AdfNode): Result<string> {
const ordered = node.type === 'orderedList' const ordered = node.type === 'orderedList'
const invalid = validateBlockNode(node, ordered ? ['order'] : []) const validation = validateBlockNode(node, ordered ? ['order'] : [])
if (!invalid.ok) return invalid if (!validation.ok) return validation
const items = node.content ?? []
if (items.length === 0) return failure('unsupported-node-shape', `a ${node.type} holds at least one listItem`)
const start = ordered ? node.attrs?.['order'] : 0 const start = ordered ? node.attrs?.['order'] : 0
if (ordered && (start === undefined || start === 1)) { if (ordered && (start === undefined || start === 1)) {
return failure('ambiguous-ordered-list-start', 'an orderedList starting at 1 and one with no order share one markdown spelling') return failure('ambiguous-ordered-list-start', 'an orderedList starting at 1 and one with no order share one markdown spelling')
} }
if (typeof start !== 'number' || !Number.isInteger(start) || start < 0 || start > 999999999) { if (typeof start !== 'number' || !Number.isInteger(start) || start < 0 || start > largestListMarker) {
return failure('unsupported-node-shape', `no list marker spells the order ${JSON.stringify(start ?? null)}`) return failure('unsupported-node-shape', `no list marker spells the order ${JSON.stringify(start ?? null)}`)
} }
const items: string[] = [] if (start + items.length - 1 > largestListMarker) {
for (const [offset, item] of (node.content ?? []).entries()) { return failure('unspellable-list-marker', `no list marker spells the ${items.length} items an orderedList starting at ${start} needs`)
if (item.type !== 'listItem') return failure('unsupported-node-shape', `a ${node.type} holds listItem nodes only`)
const invalidItem = validateBlockNode(item, [])
if (!invalidItem.ok) return invalidItem
const inner = emitBlocks(item.content ?? [], true)
if (!inner.ok) return inner
const marker = ordered ? `${start + offset}. ` : '- '
if (inner.value === '') {
items.push(marker.trimEnd())
continue
}
const indent = ' '.repeat(marker.length)
items.push(
inner.value
.split('\n')
.map((line, lineIndex) => (lineIndex === 0 ? `${marker}${line}` : line === '' ? '' : `${indent}${line}`))
.join('\n'),
)
} }
return success(items.join('\n')) const lines: string[] = []
for (const [offset, item] of items.entries()) {
if (item.type !== 'listItem') return failure('unsupported-node-shape', `a ${node.type} holds listItem nodes only`)
const emitted = emitListItem(item, ordered ? `${start + offset}. ` : '- ')
if (!emitted.ok) return emitted
lines.push(emitted.value)
}
return success(lines.join('\n'))
}
function emitListItem(item: AdfNode, marker: string): Result<string> {
const validation = validateBlockNode(item, [])
if (!validation.ok) return validation
const inner = emitBlocks(item.content ?? [], true)
if (!inner.ok) return inner
if (inner.value === '') return success(marker.trimEnd())
const indent = ' '.repeat(marker.length)
return success(
inner.value
.split('\n')
.map((line, index) => (index === 0 ? `${marker}${line}` : line === '' ? '' : `${indent}${line}`))
.join('\n'),
)
} }
function emitParagraph(node: AdfNode): Result<string> { function emitParagraph(node: AdfNode): Result<string> {
const invalid = validateBlockNode(node, []) const validation = validateBlockNode(node, [])
if (!invalid.ok) return invalid if (!validation.ok) return validation
const content = node.content ?? [] const content = node.content ?? []
if (content.length === 0) return success('::paragraph') if (content.length === 0) return success('::paragraph')
return emitInlineLine(content, 'paragraph') return emitInlineLine(content, 'paragraph')
} }
function emitRule(node: AdfNode): Result<string> { function emitRule(node: AdfNode): Result<string> {
const invalid = validateBlockNode(node, []) const validation = validateBlockNode(node, [])
if (!invalid.ok) return invalid if (!validation.ok) return validation
if ((node.content ?? []).length > 0) return failure('unsupported-node-shape', 'a rule holds no content') if ((node.content ?? []).length > 0) return failure('unsupported-node-shape', 'a rule holds no content')
return success('---') return success('---')
} }
+1 -10
View File
@@ -1,16 +1,7 @@
export type JsonValue = JsonValue[] | boolean | null | number | string | { [key: string]: JsonValue } import type { JsonValue } from './json-value.ts'
export type JsonSpelling = 'compact' | 'two-space' export type JsonSpelling = 'compact' | 'two-space'
export function isJsonValue(value: unknown): value is JsonValue {
if (value === null) return true
if (typeof value === 'boolean' || typeof value === 'string') return true
if (typeof value === 'number') return Number.isFinite(value)
if (Array.isArray(value)) return value.every(isJsonValue)
if (typeof value === 'object') return Object.values(value).every(isJsonValue)
return false
}
export function serializeCanonicalJson(value: JsonValue, spelling: JsonSpelling): string { export function serializeCanonicalJson(value: JsonValue, spelling: JsonSpelling): string {
return serialize(value, spelling === 'compact' ? '' : ' ', 0) return serialize(value, spelling === 'compact' ? '' : ' ', 0)
} }
+31
View File
@@ -0,0 +1,31 @@
export type LinePosition = 'first' | 'later'
const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});'
const anchoredEntityReference = new RegExp(`^(?:${entityReferenceSource})`)
const entityReference = new RegExp(entityReferenceSource)
const firstCharacterOpeners = [/^#{1,6}(?:[ \t]|$)/, /^>/, /^[*+-](?:[ \t]|$)/, /^`{3,}/, /^~{3,}/, /^:{2,}/, /^\|/]
const orderedListOpener = /^(\d{1,9})[.)](?:[ \t]|$)/
const setextUnderline = /^=+$/
const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
export function claimsLine(line: string, position: LinePosition): boolean {
return escapesLineClaim(line, 0, position) || orderedListOpener.test(line)
}
export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean {
if (offset === 0) {
if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true
return position === 'later' && setextUnderline.test(line)
}
const digits = orderedListOpener.exec(line)?.[1]
return digits !== undefined && offset === digits.length
}
export function holdsEntityReference(text: string): boolean {
return entityReference.test(text)
}
export function startsEntityReference(text: string): boolean {
return anchoredEntityReference.test(text)
}
+2 -1
View File
@@ -6,7 +6,8 @@ import { fileURLToPath } from 'node:url'
import { adfToMarkdown } from './adf-to-markdown.ts' import { adfToMarkdown } from './adf-to-markdown.ts'
import { isAdfDocument } from './adf-document.ts' import { isAdfDocument } from './adf-document.ts'
import { isJsonValue, serializeCanonicalJson } from './canonical-json.ts' import { isJsonValue } from './json-value.ts'
import { serializeCanonicalJson } from './canonical-json.ts'
const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus') const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', 'corpus')
const roundTripRoot = join(corpusRoot, 'round-trip') const roundTripRoot = join(corpusRoot, 'round-trip')
+1
View File
@@ -1,4 +1,5 @@
export type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from './adf-document.ts' export type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from './adf-document.ts'
export type { ConvertError, ConvertErrorCode, Result } from './result.ts' export type { ConvertError, ConvertErrorCode, Result } from './result.ts'
export type { JsonValue } from './json-value.ts'
export { adfToMarkdown } from './adf-to-markdown.ts' export { adfToMarkdown } from './adf-to-markdown.ts'
export { isAdfDocument } from './adf-document.ts' export { isAdfDocument } from './adf-document.ts'
+10
View File
@@ -0,0 +1,10 @@
export type JsonValue = JsonValue[] | boolean | null | number | string | { [key: string]: JsonValue }
export function isJsonValue(value: unknown): value is JsonValue {
if (value === null) return true
if (typeof value === 'boolean' || typeof value === 'string') return true
if (typeof value === 'number') return Number.isFinite(value)
if (Array.isArray(value)) return [...value].every(isJsonValue)
if (typeof value === 'object') return Object.values(value).every(isJsonValue)
return false
}
+26 -31
View File
@@ -1,3 +1,5 @@
import { escapesLineClaim, startsEntityReference, type LinePosition } from './commonmark-grammar.ts'
export type InlineSegment = { export type InlineSegment = {
kind: 'emphasis-close' | 'emphasis-open' | 'link-text' | 'literal' | 'syntax' kind: 'emphasis-close' | 'emphasis-open' | 'link-text' | 'literal' | 'syntax'
text: string text: string
@@ -6,33 +8,19 @@ export type InlineSegment = {
export type LineContainer = 'heading' | 'paragraph' export type LineContainer = 'heading' | 'paragraph'
const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
const entityReference = /^&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});/
const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\s<>]*>/, /^<[^\s<>@]+@[^\s<>@]+>/] const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\s<>]*>/, /^<[^\s<>@]+@[^\s<>@]+>/]
const inlineDirective = /^:[a-z][A-Za-z0-9]*[[{]/ const inlineDirective = /^:[a-z][A-Za-z0-9]*[[{]/
const linkOpener = /\](?=[([:])/ const linkOpener = /\](?=[([:])/
const orderedListMarker = /^\d{1,9}$/
const setextUnderline = /^=+$/
const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
const unicodePunctuation = /[\p{P}\p{S}]/u const unicodePunctuation = /[\p{P}\p{S}]/u
const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u
const escapableOpeners = [/^#{1,6}(?:[ \t]|$)/, /^>/, /^[*+-](?:[ \t]|$)/, /^`{3,}/, /^~{3,}/, /^:{2,}/, /^\|/]
const blockOpeners = [...escapableOpeners, /^\d{1,9}[.)](?:[ \t]|$)/]
export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): string { export function assembleInlineLine(segments: readonly InlineSegment[], container: LineContainer): string {
return escape(resolveEmphasis(segments), container) return escape(resolveEmphasis(segments), container)
} }
export function lineOpensBlock(line: string): boolean {
return blockOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)
}
export function isWordCharacter(character: string): boolean {
return character !== '' && !unicodeWhitespace.test(character) && !unicodePunctuation.test(character)
}
function resolveEmphasis(segments: readonly InlineSegment[]): InlineSegment[] { function resolveEmphasis(segments: readonly InlineSegment[]): InlineSegment[] {
const resolved = segments.map((segment) => ({ ...segment })) const resolved = segments.map((segment) => ({ ...segment }))
// Offsets index the pre-swap text: every emphasis spelling this swaps between is one character wide.
const scan = resolved.map((segment) => segment.text).join('') const scan = resolved.map((segment) => segment.text).join('')
const offsets: number[] = [] const offsets: number[] = []
let offset = 0 let offset = 0
@@ -77,19 +65,29 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): s
} }
function opensConstruct(scan: string, index: number, inLinkText: boolean, container: LineContainer, escaped: ReadonlySet<number>): boolean { function opensConstruct(scan: string, index: number, inLinkText: boolean, container: LineContainer, escaped: ReadonlySet<number>): boolean {
const claimsLine = container === 'heading' ? closesHeading(scan, index) : claimsLineStart(scan, index)
return claimsLine || claimsCharacter(scan, index, inLinkText, escaped)
}
function claimsLineStart(scan: string, index: number): boolean {
const start = scan.lastIndexOf('\n', index - 1) + 1
const end = scan.indexOf('\n', index)
const line = scan.slice(start, end === -1 ? undefined : end)
const position: LinePosition = start === 0 ? 'first' : 'later'
return escapesLineClaim(line, index - start, position)
}
function closesHeading(scan: string, index: number): boolean {
if (scan.charAt(index) !== '#' || !/^#+$/.test(scan.slice(index))) return false
return index === 0 || /[ \t]/.test(scan.charAt(index - 1))
}
function claimsCharacter(scan: string, index: number, inLinkText: boolean, escaped: ReadonlySet<number>): boolean {
const character = scan.charAt(index) const character = scan.charAt(index)
const rest = scan.slice(index) const rest = scan.slice(index)
const line = rest.split('\n')[0] ?? ''
if (container === 'paragraph' && (index === 0 || scan.charAt(index - 1) === '\n')) {
if (escapableOpeners.some((opener) => opener.test(rest))) return true
if (thematicBreak.test(line)) return true
if (index > 0 && setextUnderline.test(line)) return true
}
if (container === 'paragraph' && (character === '.' || character === ')') && closesOrderedListMarker(scan, index)) return true
if (container === 'heading' && character === '#' && /^#+$/.test(rest) && (index === 0 || /[ \t]/.test(scan.charAt(index - 1)))) return true
if (inLinkText && (character === '[' || character === ']')) return true if (inLinkText && (character === '[' || character === ']')) return true
if (character === '\\') return asciiPunctuation.test(scan.charAt(index + 1)) if (character === '\\') return asciiPunctuation.test(scan.charAt(index + 1))
if (character === '&') return entityReference.test(rest) if (character === '&') return startsEntityReference(rest)
if (character === '<') return htmlConstructs.some((construct) => construct.test(rest)) if (character === '<') return htmlConstructs.some((construct) => construct.test(rest))
if (character === ':') return inlineDirective.test(rest) if (character === ':') return inlineDirective.test(rest)
if (character === '[') return linkOpener.test(rest) if (character === '[') return linkOpener.test(rest)
@@ -98,13 +96,6 @@ function opensConstruct(scan: string, index: number, inLinkText: boolean, contai
return false return false
} }
function closesOrderedListMarker(scan: string, index: number): boolean {
const lineStart = scan.lastIndexOf('\n', index - 1) + 1
if (!orderedListMarker.test(scan.slice(lineStart, index))) return false
const following = scan.charAt(index + 1)
return following === '' || following === ' ' || following === '\t' || following === '\n'
}
function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>): boolean { function opensCodeSpan(scan: string, index: number, escaped: ReadonlySet<number>): boolean {
if (!startsRun(scan, index, escaped)) return false if (!startsRun(scan, index, escaped)) return false
const length = runLength(scan, index) const length = runLength(scan, index)
@@ -155,6 +146,10 @@ function isWhitespace(character: string): boolean {
return character === '' || unicodeWhitespace.test(character) return character === '' || unicodeWhitespace.test(character)
} }
function isWordCharacter(character: string): boolean {
return character !== '' && !unicodeWhitespace.test(character) && !unicodePunctuation.test(character)
}
function charAt(text: string, index: number): string { function charAt(text: string, index: number): string {
return index < 0 ? '' : text.charAt(index) return index < 0 ? '' : text.charAt(index)
} }
+31 -26
View File
@@ -1,5 +1,6 @@
import type { AdfMark, AdfNode } from './adf-document.ts' import type { AdfMark, AdfNode } from './adf-document.ts'
import { assembleInlineLine, lineOpensBlock, type InlineSegment, type LineContainer } from './markdown-escaping.ts' import { assembleInlineLine, type InlineSegment, type LineContainer } from './markdown-escaping.ts'
import { claimsLine, holdsEntityReference } from './commonmark-grammar.ts'
import { failure, success, type Result } from './result.ts' import { failure, success, type Result } from './result.ts'
import { longestBacktickRun } from './backtick-runs.ts' import { longestBacktickRun } from './backtick-runs.ts'
import { serializeCanonicalJson } from './canonical-json.ts' import { serializeCanonicalJson } from './canonical-json.ts'
@@ -10,20 +11,21 @@ type InlineContext = {
inLinkText: boolean inLinkText: boolean
} }
type InlineRun = { kind: 'marked'; mark: AdfMark; nodes: AdfNode[] } | { kind: 'plain'; node: AdfNode }
const autolink = /^[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\s<>\u0000-\u001f\u007f]*$/ const autolink = /^[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\s<>\u0000-\u001f\u007f]*$/
const controlCharacter = /[\u0000-\u001f\u007f]/ const controlCharacter = /[\u0000-\u001f\u007f]/
const entityReference = /&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});/
const linkAttributes = ['href', 'title'] const linkAttributes = ['href', 'title']
export function emitInlineLine(nodes: readonly AdfNode[], container: LineContainer): Result<string> { export function emitInlineLine(nodes: readonly AdfNode[], container: LineContainer): Result<string> {
const segments = emitRun(nodes, 0, { atBlockEnd: true, container, inLinkText: false }) const segments = emitRun(nodes, 0, { atBlockEnd: true, container, inLinkText: false })
if (!segments.ok) return segments if (!segments.ok) return segments
const line = assembleInlineLine(segments.value, container) const line = assembleInlineLine(segments.value, container)
for (const single of line.split('\n')) { for (const [index, single] of line.split('\n').entries()) {
if (/^[ \t]|[ \t]$/.test(single)) { if (/^[ \t]|[ \t]$/.test(single)) {
return failure('unspellable-whitespace', 'a line begins or ends with whitespace CommonMark strips') return failure('unspellable-whitespace', 'a line begins or ends with whitespace CommonMark strips')
} }
if (container === 'paragraph' && lineOpensBlock(single)) { if (container === 'paragraph' && claimsLine(single, index === 0 ? 'first' : 'later')) {
return failure('unspellable-line-start', `block parsing would claim the emitted line ${JSON.stringify(single)}`) return failure('unspellable-line-start', `block parsing would claim the emitted line ${JSON.stringify(single)}`)
} }
} }
@@ -31,29 +33,32 @@ export function emitInlineLine(nodes: readonly AdfNode[], container: LineContain
} }
function emitRun(nodes: readonly AdfNode[], depth: number, context: InlineContext): Result<InlineSegment[]> { function emitRun(nodes: readonly AdfNode[], depth: number, context: InlineContext): Result<InlineSegment[]> {
const runs = inlineRuns(nodes, depth)
const segments: InlineSegment[] = [] const segments: InlineSegment[] = []
let index = 0 for (const [index, run] of runs.entries()) {
while (index < nodes.length) { const runContext = { ...context, atBlockEnd: context.atBlockEnd && index === runs.length - 1 }
const node = nodes[index] const emitted = run.kind === 'plain' ? emitLeaf(run.node, runContext) : emitMarkedRun(run.nodes, run.mark, depth, runContext)
if (node === undefined) return failure('unsupported-node-shape', 'the inline content holds a hole') if (!emitted.ok) return emitted
const mark = (node.marks ?? [])[depth] segments.push(...emitted.value)
if (mark === undefined) {
const leaf = emitLeaf(node, { ...context, atBlockEnd: context.atBlockEnd && index === nodes.length - 1 })
if (!leaf.ok) return leaf
segments.push(...leaf.value)
index += 1
continue
}
let end = index + 1
while (end < nodes.length && sameMark((nodes[end]?.marks ?? [])[depth], mark)) end += 1
const wrapped = emitMarkedRun(nodes.slice(index, end), mark, depth, { ...context, atBlockEnd: context.atBlockEnd && end === nodes.length })
if (!wrapped.ok) return wrapped
segments.push(...wrapped.value)
index = end
} }
return success(segments) return success(segments)
} }
function inlineRuns(nodes: readonly AdfNode[], depth: number): InlineRun[] {
const runs: InlineRun[] = []
for (const node of nodes) {
const mark = (node.marks ?? [])[depth]
if (mark === undefined) {
runs.push({ kind: 'plain', node })
continue
}
const previous = runs[runs.length - 1]
if (previous?.kind === 'marked' && sameMark(previous.mark, mark)) previous.nodes.push(node)
else runs.push({ kind: 'marked', mark, nodes: [node] })
}
return runs
}
function emitLeaf(node: AdfNode, context: InlineContext): Result<InlineSegment[]> { function emitLeaf(node: AdfNode, context: InlineContext): Result<InlineSegment[]> {
if (node.type !== 'hardBreak' && node.type !== 'text') { if (node.type !== 'hardBreak' && node.type !== 'text') {
return failure('unsupported-node-type', `the canonical form spells no inline node of type ${node.type}`) return failure('unsupported-node-type', `the canonical form spells no inline node of type ${node.type}`)
@@ -127,7 +132,7 @@ function emitLink(nodes: readonly AdfNode[], mark: AdfMark, depth: number, conte
function spellDestination(href: string): Result<string> { function spellDestination(href: string): Result<string> {
if (controlCharacter.test(href)) return failure('unspellable-link-destination', 'a link destination holds a control character') if (controlCharacter.test(href)) return failure('unspellable-link-destination', 'a link destination holds a control character')
if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination') if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination')
if (entityReference.test(href)) { if (holdsEntityReference(href)) {
return failure('unspellable-link-destination', 'a link destination shaped like an entity reference decodes on the way back') return failure('unspellable-link-destination', 'a link destination shaped like an entity reference decodes on the way back')
} }
if (href.includes(' ')) { if (href.includes(' ')) {
@@ -143,7 +148,7 @@ function spellDestination(href: string): Result<string> {
function spellTitle(title: string): Result<string> { function spellTitle(title: string): Result<string> {
if (/["\n\r\\]/.test(title)) return failure('unspellable-link-title', 'no canonical escape spells a quote, backslash or newline in a link title') if (/["\n\r\\]/.test(title)) return failure('unspellable-link-title', 'no canonical escape spells a quote, backslash or newline in a link title')
if (entityReference.test(title)) return failure('unspellable-link-title', 'a link title shaped like an entity reference decodes on the way back') if (holdsEntityReference(title)) return failure('unspellable-link-title', 'a link title shaped like an entity reference decodes on the way back')
return success(` "${title}"`) return success(` "${title}"`)
} }
@@ -157,7 +162,7 @@ function balanced(href: string): boolean {
return depth === 0 return depth === 0
} }
function sameMark(candidate: AdfMark | undefined, mark: AdfMark): boolean { function sameMark(candidate: AdfMark, mark: AdfMark): boolean {
if (candidate === undefined || candidate.type !== mark.type) return false if (candidate.type !== mark.type) return false
return serializeCanonicalJson(candidate.attrs ?? {}, 'compact') === serializeCanonicalJson(mark.attrs ?? {}, 'compact') return serializeCanonicalJson(candidate.attrs ?? {}, 'compact') === serializeCanonicalJson(mark.attrs ?? {}, 'compact')
} }
+1
View File
@@ -8,6 +8,7 @@ export type ConvertErrorCode =
| 'unspellable-line-start' | 'unspellable-line-start'
| 'unspellable-link-destination' | 'unspellable-link-destination'
| 'unspellable-link-title' | 'unspellable-link-title'
| 'unspellable-list-marker'
| 'unspellable-mark' | 'unspellable-mark'
| 'unspellable-whitespace' | 'unspellable-whitespace'
| 'unspelled-block-marks' | 'unspelled-block-marks'
+16 -4
View File
@@ -60,8 +60,15 @@ detail is settled at its own milestone.
itself under the library's own canonical serializer — one implementation, keys sorted, two itself under the library's own canonical serializer — one implementation, keys sorted, two
spellings: two-space indent for the corpus files and the block carry's body, compact for spellings: two-space indent for the corpus files and the block carry's body, compact for
the inline carry. `commonmark-subset/` green. the inline carry. `commonmark-subset/` green.
- [ ] **2b — Block nodes.** `block-nodes/` green. - [ ] **2b — Block nodes.** `block-nodes/` green. Block separation becomes
- [ ] **2c — Inline nodes and marks.** `inline-nodes/` green. `separationBetween(previous, next, container)` here — a boolean cannot hold the third case
`spec/flavour.md` states for two directive blocks in a container body, and the maintainer's
answer on a CommonMark block beside a directive block (1d) drops into the same seam. Give
the emitter's refusals a corpus home while the directories grow: `corpus/unspellable/`,
a `.json` beside the `ConvertErrorCode` it must return, the emitter half of `corpus/errors/`.
- [ ] **2c — Inline nodes and marks.** `inline-nodes/` green. `InlineSegment.kind` splits into
its two axes here — escapability (`attribute` for `:text{text="…"}`, `backslash`, `none`)
and the emphasis role — rather than gaining a third value that means one of each.
- [ ] **2d — The opaque carry** (§3). Fixtures and emitter together, into - [ ] **2d — The opaque carry** (§3). Fixtures and emitter together, into
`corpus/round-trip/opaque-carry/`: an unknown node in both positions, the reserved `adf` `corpus/round-trip/opaque-carry/`: an unknown node in both positions, the reserved `adf`
info string, and the `codeBlock` whose language is `adf`. info string, and the `codeBlock` whose language is `adf`.
@@ -80,9 +87,14 @@ detail is settled at its own milestone.
line that does not parse, the content slot, raw HTML with no mapping — each with the error line that does not parse, the content slot, raw HTML with no mapping — each with the error
it must return). The raw-HTML element mapping is empty until milestone 6, so at `0.1.0` it must return). The raw-HTML element mapping is empty until milestone 6, so at `0.1.0`
every raw-HTML construct in input is an error result. The CommonMark spec suite runs every raw-HTML construct in input is an error result. The CommonMark spec suite runs
against it from here (§10). against it from here (§10). `src/` gets its hierarchy at the same split — `adf/`,
`markdown/`, `html/`, the grammar module shared inside `markdown/` — while the rename is
still mechanical.
- [ ] **4 — Round-trip property tests** over the corpus, both ways — the thing that proves 2 and - [ ] **4 — Round-trip property tests** over the corpus, both ways — the thing that proves 2 and
3. Generators emit editor-normal ADF (§2). Real sanitized ADF from live Atlassian APIs lands 3. Editor-normal (§2) gets its implementation here — `toEditorNormal(doc)` and the equality
the round-trip asserts, which over normalized input is the canonical serializer's compact
spelling — rather than staying spelled inline as `?? []` at every reader.
Generators emit editor-normal ADF (§2). Real sanitized ADF from live Atlassian APIs lands
here too (§10), in `corpus/real-payloads/`: an ADF→markdown→ADF check with no expected here too (§10), in `corpus/real-payloads/`: an ADF→markdown→ADF check with no expected
markdown, the payloads supplied by the maintainer. markdown, the payloads supplied by the maintainer.
- [ ] **5 — Release pipeline, ship `0.1.0`.** Publish-on-version-change (§9), `NPM_TOKEN` secret, - [ ] **5 — Release pipeline, ship `0.1.0`.** Publish-on-version-change (§9), `NPM_TOKEN` secret,