3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-18 22:25:44 +01:00

Give each markdown input its own reader: commonMarkToAdf, losslessMarkdownToAdf and portableMarkdownToAdf, each claiming only its flavour

This commit is contained in:
2026-10-09 02:30:41 +02:00
committed by Lillem4n
parent 80e1124f39
commit 23ec45febd
30 changed files with 1374 additions and 1387 deletions
+14 -24
View File
@@ -4,23 +4,14 @@ import test from 'node:test'
import type { AdfNode } from '../adf/document.ts'
import { adfDocument, propertyRuns, propertyTimeout } from './property-harness.ts'
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { adfToPlainMarkdown, reduceToPlain } from '../markdown/plain/adf-to-plain-markdown.ts'
import { directivePrefix } from '../markdown/directive-syntax.ts'
import { markdownToAdf, plainMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { toEditorNormal } from '../markdown/plain/editor-normal.ts'
import { adfToLosslessMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { adfToPortableMarkdown, reduceToPortable } from '../markdown/portable/adf-to-portable-markdown.ts'
import { losslessMarkdownToAdf, portableMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { toEditorNormal } from '../markdown/portable/editor-normal.ts'
const gateRuns = 1600
const renamedPrefix = '!adg:'
// Renaming the prefix changes what markdown reads only where a directive was read.
function readsNoDirective(markdown: string): boolean {
const read = markdownToAdf(markdown)
const renamed = markdownToAdf(markdown.replaceAll(directivePrefix, renamedPrefix))
return read.ok && renamed.ok && JSON.stringify(read.value).replaceAll(directivePrefix, renamedPrefix) === JSON.stringify(renamed.value)
}
// Each block's text, an expand's title and an image's alt and url, in document order: what the plain pair keeps.
// Each block's text, an expand's title and an image's alt and url, in document order: what the portable pair keeps.
function shownText(nodes: readonly AdfNode[]): string[] {
const shown: string[] = []
for (const node of nodes) {
@@ -37,9 +28,9 @@ function shownText(nodes: readonly AdfNode[]): string[] {
test('a generated document emits markdown that reads back to it', { timeout: propertyTimeout }, () => {
fc.assert(
fc.property(adfDocument, (document) => {
const emitted = adfToMarkdown(document)
const emitted = adfToLosslessMarkdown(document)
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
const read = markdownToAdf(emitted.value)
const read = losslessMarkdownToAdf(emitted.value)
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
assert.deepEqual(read.value, document, `reading ${JSON.stringify(emitted.value)}`)
}),
@@ -47,26 +38,25 @@ test('a generated document emits markdown that reads back to it', { timeout: pro
)
})
test('a generated document writes plain markdown refusing only what the guard refuses, and that markdown reads back to its text and to itself', { timeout: propertyTimeout }, () => {
test('a generated document writes portable markdown refusing only what the guard refuses, and that markdown reads back to its text and to itself', { timeout: propertyTimeout }, () => {
fc.assert(
fc.property(adfDocument, (document) => {
const written = adfToPlainMarkdown(document)
const written = adfToPortableMarkdown(document)
assert.ok(written.ok, written.ok ? '' : `${written.error.code}: ${written.error.message}`)
assert.ok(readsNoDirective(written.value), `a directive in ${JSON.stringify(written.value)}`)
const read = plainMarkdownToAdf(written.value)
const read = portableMarkdownToAdf(written.value)
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(written.value)}`)
const reduced = reduceToPlain(document)
const reduced = reduceToPortable(document)
assert.deepEqual(shownText(read.value.content ?? []), reduced.ok ? shownText(reduced.value.content ?? []) : reduced, `reading ${JSON.stringify(written.value)}`)
assert.deepEqual(adfToPlainMarkdown(read.value), written, `reading ${JSON.stringify(written.value)}`)
assert.deepEqual(adfToPortableMarkdown(read.value), written, `reading ${JSON.stringify(written.value)}`)
}),
propertyRuns(gateRuns),
)
})
test('a generated document writes the plain markdown its editor-normal form writes', { timeout: propertyTimeout }, () => {
test('a generated document writes the portable markdown its editor-normal form writes', { timeout: propertyTimeout }, () => {
fc.assert(
fc.property(adfDocument, (document) => {
assert.deepEqual(adfToPlainMarkdown(document), adfToPlainMarkdown(toEditorNormal(document)))
assert.deepEqual(adfToPortableMarkdown(document), adfToPortableMarkdown(toEditorNormal(document)))
}),
propertyRuns(gateRuns),
)
+5 -5
View File
@@ -6,8 +6,8 @@ import { readFileSync } from 'node:fs'
import test from 'node:test'
import type { AdfDocument, AdfNode } from '../adf/document.ts'
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { adfToLosslessMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { commonMarkToAdf, losslessMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus', 'commonmark-spec')
const checks = ['count', 'fixpoint', 'text'] as const
@@ -289,9 +289,9 @@ function concatenatedText(document: AdfNode): string {
}
function fixpointRefused(example: SpecExample, document: AdfDocument): string | undefined {
const emitted = adfToMarkdown(document)
const emitted = adfToLosslessMarkdown(document)
if (!emitted.ok) return emitted.error.code
const again = markdownToAdf(emitted.value)
const again = losslessMarkdownToAdf(emitted.value)
assert.ok(again.ok, `example ${example.example} emits markdown it cannot read back`)
assert.deepEqual(again.value, document, `example ${example.example} does not hold its own round-trip`)
return undefined
@@ -312,7 +312,7 @@ function countMismatch(example: SpecExample, document: AdfDocument): string | un
for (const example of spec) {
test(`CommonMark example ${example.example} => ${example.section}`, () => {
const parse = markdownToAdf(example.markdown)
const parse = commonMarkToAdf(example.markdown)
const refused = exampleToRefusal.get(example.example)
if (refused !== undefined) {
assert.ok(!parse.ok, `example ${example.example} was expected to refuse with ${refused} but parsed`)
+10 -10
View File
@@ -4,10 +4,10 @@ import { fileURLToPath } from 'node:url'
import { readFileSync, readdirSync } from 'node:fs'
import test from 'node:test'
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { adfToLosslessMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { isAdfDocument } from '../adf/document.ts'
import { isJsonValue } from '../json-value.ts'
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { losslessMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { serializeCanonicalJson } from '../canonical-json.ts'
const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus')
@@ -79,7 +79,7 @@ for (const directory of roundTripDirectories) {
test(`${directory}/${name} emits its markdown byte for byte`, () => {
const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`)
const result = adfToMarkdown(parsed)
const result = adfToLosslessMarkdown(parsed)
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
const expected = readFileSync(join(roundTripRoot, directory, `${name}.md`))
const emitted = Buffer.from(result.value, 'utf8')
@@ -94,7 +94,7 @@ for (const directory of roundTripDirectories) {
test(`${directory}/${name} reads its markdown back to the document beside it`, () => {
const expected: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`)
const result = markdownToAdf(readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8'))
const result = losslessMarkdownToAdf(readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8'))
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
assert.deepEqual(result.value, expected)
})
@@ -122,12 +122,12 @@ for (const name of pairedNames(normalizationRoot, '.md', '.json')) {
test(`normalization/${name} parses to the document beside it, which emits and reads back to itself`, () => {
const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`)
const result = markdownToAdf(readFileSync(join(normalizationRoot, `${name}.md`), 'utf8'))
const result = losslessMarkdownToAdf(readFileSync(join(normalizationRoot, `${name}.md`), 'utf8'))
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
assert.deepEqual(result.value, expected)
const emitted = adfToMarkdown(result.value)
const emitted = adfToLosslessMarkdown(result.value)
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
const again = markdownToAdf(emitted.value)
const again = losslessMarkdownToAdf(emitted.value)
assert.ok(again.ok, again.ok ? '' : `${again.error.code}: ${again.error.message}`)
assert.deepEqual(again.value, expected)
})
@@ -141,9 +141,9 @@ for (const name of names(realPayloadsRoot, '.json')) {
test(`real-payloads/${name} emits markdown that reads back to it`, () => {
const payload: unknown = JSON.parse(readFileSync(join(realPayloadsRoot, `${name}.json`), 'utf8'))
assert.ok(isAdfDocument(payload), `${name}.json is not an ADF document`)
const emitted = adfToMarkdown(payload)
const emitted = adfToLosslessMarkdown(payload)
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
const parsed = markdownToAdf(emitted.value)
const parsed = losslessMarkdownToAdf(emitted.value)
assert.ok(parsed.ok, parsed.ok ? '' : `${parsed.error.code}: ${parsed.error.message}`)
assert.deepEqual(parsed.value, payload)
})
@@ -159,7 +159,7 @@ function lineStarting(markdown: string, offset: number): { line: number; offset:
for (const name of pairedNames(errorsRoot, '.md', '.error')) {
test(`errors/${name} is refused with the error it names, at a line of its own input`, () => {
const markdown = readFileSync(join(errorsRoot, `${name}.md`), 'utf8')
const result = markdownToAdf(markdown)
const result = losslessMarkdownToAdf(markdown)
assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '')
assert.equal(result.error.code, readFileSync(join(errorsRoot, `${name}.error`), 'utf8').trimEnd())
const { position } = result.error
+22 -7
View File
@@ -8,10 +8,11 @@ import type { AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
import type { JsonValue } from '../json-value.ts'
import type { Result } from '../result.ts'
import { adfDocument, attributes, jsonKey, jsonValue, markdownPieces, propertyRuns, propertyTimeout, textOf } from './property-harness.ts'
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { adfToLosslessMarkdown } from '../markdown/emit/adf-to-markdown.ts'
import { blockArgument, listBreakName, marksAttribute } from '../markdown/block-directive.ts'
import { blockNodes } from '../adf/block-nodes.ts'
import { carryName } from '../markdown/opaque-carry.ts'
import { commonMarkToAdf, losslessMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import {
directivePrefix,
spellAttributes,
@@ -27,7 +28,6 @@ import { fencedCodeBlock } from '../markdown/commonmark/backtick-runs.ts'
import { inlineNodes } from '../adf/inline-nodes.ts'
import { markAttributes } from '../adf/mark-attributes.ts'
import { markSpelling } from '../markdown/mark-spellings.ts'
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
import { nodeContent, nodeMarks } from '../adf/document.ts'
import { serializeCanonicalJson } from '../canonical-json.ts'
import { textDirectiveName } from '../markdown/text-directive.ts'
@@ -382,7 +382,7 @@ const cleanMarkdown = markdownOf(false)
const hostileMarkdown = markdownOf(true)
const canonical = adfDocument
.map((document) => adfToMarkdown(document))
.map((document) => adfToLosslessMarkdown(document))
.filter((emitted): emitted is Extract<Result<string>, { ok: true }> => emitted.ok)
.map((emitted) => emitted.value)
@@ -424,16 +424,16 @@ test('generated markdown refuses, or its spelling reads back and spells itself',
let fixpoints = 0
fc.assert(
fc.property(document, (input) => {
const parsed = markdownToAdf(input)
const parsed = losslessMarkdownToAdf(input)
if (!parsed.ok) return
const emitted = adfToMarkdown(parsed.value)
const emitted = adfToLosslessMarkdown(parsed.value)
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message} — spelling ${JSON.stringify(input)}`)
fixpoints += 1
if (holdsDirectiveShape(parsed.value)) directiveShaped += 1
const read = markdownToAdf(emitted.value)
const read = losslessMarkdownToAdf(emitted.value)
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
assert.deepEqual(read.value, parsed.value, `reading ${JSON.stringify(emitted.value)}`)
const respelled = adfToMarkdown(read.value)
const respelled = adfToLosslessMarkdown(read.value)
assert.ok(respelled.ok, respelled.ok ? '' : `${respelled.error.code}: ${respelled.error.message} — spelling ${JSON.stringify(emitted.value)} again`)
assert.equal(respelled.value, emitted.value)
}),
@@ -444,3 +444,18 @@ test('generated markdown refuses, or its spelling reads back and spells itself',
assert.ok(directiveShaped >= directiveShapedFloor, `${directiveShaped} runs reaching the fixpoint held a node or mark outside CommonMark's own types, under the floor of ${directiveShapedFloor}`)
})
test('generated markdown read as CommonMark refuses, or its document reads back through the lossless pair', { timeout: propertyTimeout }, () => {
fc.assert(
fc.property(document, (input) => {
const parsed = commonMarkToAdf(input)
if (!parsed.ok) return
const emitted = adfToLosslessMarkdown(parsed.value)
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message} — spelling ${JSON.stringify(input)}`)
const read = losslessMarkdownToAdf(emitted.value)
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
assert.deepEqual(read.value, parsed.value, `reading ${JSON.stringify(emitted.value)}`)
}),
propertyRuns(gateRuns),
)
})