24 - the conformance gates live in src/conformance/
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fc from 'fast-check'
|
||||
import test from 'node:test'
|
||||
|
||||
import { adfDocument, propertyRuns, propertyTimeout } from './property-harness.ts'
|
||||
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
|
||||
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
|
||||
import { toEditorNormal } from '../adf/editor-normal.ts'
|
||||
|
||||
const gateRuns = 1600
|
||||
|
||||
test('a generated document refuses to emit, or its markdown reads back to it', { timeout: propertyTimeout }, () => {
|
||||
fc.assert(
|
||||
fc.property(adfDocument, (document) => {
|
||||
const emitted = adfToMarkdown(document)
|
||||
if (!emitted.ok) return
|
||||
const read = markdownToAdf(emitted.value)
|
||||
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
|
||||
assert.deepEqual(toEditorNormal(read.value), document, `reading ${JSON.stringify(emitted.value)}`)
|
||||
}),
|
||||
propertyRuns(gateRuns),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
|
||||
import type { AttributeKind, AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
|
||||
import { blockArgument } from '../markdown/block-directive.ts'
|
||||
import { blockNodes } from '../adf/block-nodes.ts'
|
||||
import { inlineNodes } from '../adf/inline-nodes.ts'
|
||||
import { markAttributes } from '../adf/mark-attributes.ts'
|
||||
|
||||
type Held = Map<string, Set<AttributeKind>>
|
||||
type Properties = Map<string, SchemaObject[]>
|
||||
type SchemaObject = Readonly<Record<string, unknown>>
|
||||
type Spelled = [string, Map<string, AttributeKind>]
|
||||
|
||||
const carried = ['alignment', 'annotation', 'backgroundColor', 'blockCard', 'bodiedRule', 'breakout', 'dataConsumer', 'embedCard', 'fontSize', 'fragment', 'indentation', 'inlineExtension', 'placeholder']
|
||||
const definitionReference = '#/definitions/'
|
||||
const gaps: string[] = []
|
||||
const grammarOwn = ['doc', 'text']
|
||||
const readKeywords = ['$ref', 'additionalProperties', 'allOf', 'anyOf', 'enum', 'items', 'maxItems', 'maximum', 'minItems', 'minLength', 'minimum', 'pattern', 'properties', 'required', 'type']
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'spec', 'adf-schema')
|
||||
const schemaFiles = ['full.json', 'stage-0.json']
|
||||
|
||||
test('the ADF JSON Schemas are @atlaskit/adf-schema 57.4.9, vendored byte-exact', () => {
|
||||
const digest = (name: string) => createHash('sha256').update(readFileSync(join(root, name))).digest('hex')
|
||||
assert.equal(digest('full.json'), '75f080928a970250eb8289e9cae5374e3c2a6c0ac3ca22478acaa9d3f39484a3')
|
||||
assert.equal(digest('stage-0.json'), '56747e5a71c0d5f8c58f94180d69e4482f62c08020d66aaf327333681fcc1c8f')
|
||||
})
|
||||
|
||||
test('the tables spell the attribute names and kinds the ADF JSON Schemas give each type they spell, the pinned gaps apart', () => {
|
||||
const held = schemaTypes()
|
||||
const spelledTypes = spelled()
|
||||
const found: string[] = []
|
||||
const gapsHeld = new Set<string>()
|
||||
for (const [type, spelledKinds] of spelledTypes) {
|
||||
const kinds = held.get(type)
|
||||
if (kinds === undefined) {
|
||||
found.push(`${type}: the tables spell the type, the schema holds no definition of it`)
|
||||
continue
|
||||
}
|
||||
for (const [attribute, schemaKinds] of kinds) {
|
||||
const kind = spelledKinds.get(attribute)
|
||||
const holds = [...schemaKinds].sort().join(' or ')
|
||||
if (kind === undefined && gaps.includes(`${type}.${attribute}`)) gapsHeld.add(`${type}.${attribute}`)
|
||||
else if (kind === undefined) found.push(`${type}.${attribute}: the schema holds ${holds}, the tables spell nothing and the gaps list does not name it`)
|
||||
else if (schemaKinds.size !== 1 || !schemaKinds.has(kind)) found.push(`${type}.${attribute}: the tables spell ${kind}, the schema holds ${holds}`)
|
||||
}
|
||||
for (const [attribute, kind] of spelledKinds) if (!kinds.has(attribute)) found.push(`${type}.${attribute}: the tables spell ${kind}, the schema holds nothing`)
|
||||
}
|
||||
for (const gap of gaps) {
|
||||
if (gapsHeld.has(gap)) continue
|
||||
const [type = '', attribute = ''] = gap.split('.')
|
||||
const spelledKinds = spelledTypes.find(([name]) => name === type)?.[1]
|
||||
const kind = spelledKinds?.get(attribute)
|
||||
if (spelledKinds === undefined) found.push(`${gap}: the gaps list names it, the tables spell no ${type} type`)
|
||||
else if (kind === undefined) found.push(`${gap}: the gaps list names it, the schema holds nothing`)
|
||||
else found.push(`${gap}: the gaps list names it, the tables spell ${kind}`)
|
||||
}
|
||||
assert.deepEqual(found, [])
|
||||
})
|
||||
|
||||
test("the ADF JSON Schemas hold no type the tables leave unspelled, the pinned carried ones and the grammar's own apart", () => {
|
||||
const held = schemaTypes()
|
||||
const spelledNames = new Set(spelled().map(([type]) => type))
|
||||
const found: string[] = []
|
||||
for (const type of held.keys()) {
|
||||
if (!spelledNames.has(type) && !carried.includes(type) && !grammarOwn.includes(type)) found.push(`${type}: the schema holds the type, the tables spell none of it and the carried list does not name it`)
|
||||
}
|
||||
for (const type of carried) {
|
||||
if (!held.has(type)) found.push(`${type}: the carried list names the type, the schema holds no definition of it`)
|
||||
if (spelledNames.has(type)) found.push(`${type}: the carried list names the type, and the tables spell it`)
|
||||
}
|
||||
assert.deepEqual(found, [])
|
||||
})
|
||||
|
||||
function spelled(): Spelled[] {
|
||||
return [
|
||||
...Object.entries(blockNodes).map(([type, model]) => spelledType(type, model.attributes, blockArgument(type))),
|
||||
...Object.entries(inlineNodes).map(([type, model]) => spelledType(type, model.attributes)),
|
||||
...Object.entries(markAttributes).map(([type, attributes]) => spelledType(type, attributes)),
|
||||
]
|
||||
}
|
||||
|
||||
function spelledType(type: string, attributes: AttributeVocabulary, argument?: string): Spelled {
|
||||
const kinds = new Map(Object.entries(attributes))
|
||||
if (argument !== undefined) kinds.set(argument, 'string')
|
||||
return [type, kinds]
|
||||
}
|
||||
|
||||
function schemaTypes(): Map<string, Held> {
|
||||
const held = new Map<string, Held>()
|
||||
for (const file of schemaFiles) {
|
||||
const definitions = schemaObject(schemaObject(JSON.parse(readFileSync(join(root, file), 'utf8')), file)['definitions'], `${file} definitions`)
|
||||
for (const [name, definition] of Object.entries(definitions)) {
|
||||
const where = `${file} ${name}`
|
||||
for (const properties of alternatives(schemaObject(definition, where), definitions, where)) {
|
||||
const attributes = (properties.get('attrs') ?? []).flatMap((attrs) => alternatives(attrs, definitions, `${where} attrs`)).flatMap((alternative) => [...alternative])
|
||||
for (const type of (properties.get('type') ?? []).flatMap((schema) => enumStrings(schema, `${where} type`))) {
|
||||
const kinds = held.get(type) ?? new Map<string, Set<AttributeKind>>()
|
||||
held.set(type, kinds)
|
||||
for (const [attribute, schemas] of attributes) {
|
||||
const attributeKinds = kinds.get(attribute) ?? new Set<AttributeKind>()
|
||||
kinds.set(attribute, attributeKinds)
|
||||
for (const schema of schemas) for (const kind of propertyKinds(schema, `${where} ${type}.${attribute}`)) attributeKinds.add(kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
function alternatives(schema: SchemaObject, definitions: SchemaObject, where: string): Properties[] {
|
||||
const own = Object.entries(schemaObject(readSchema(schema, where)['properties'] ?? {}, `${where} properties`))
|
||||
let found: Properties[] = [new Map(own.map(([name, property]): [string, SchemaObject[]] => [name, [readSchema(schemaObject(property, `${where} ${name}`), `${where} ${name}`)]]))]
|
||||
if (schema['$ref'] !== undefined) found = intersect(found, alternatives(referenced(schema['$ref'], definitions, where), definitions, `${where} ${String(schema['$ref'])}`))
|
||||
for (const branch of branches(schema['allOf'], `${where} allOf`)) found = intersect(found, alternatives(branch, definitions, `${where} allOf`))
|
||||
if (schema['anyOf'] !== undefined) found = intersect(found, branches(schema['anyOf'], `${where} anyOf`).flatMap((branch) => alternatives(branch, definitions, `${where} anyOf`)))
|
||||
return found
|
||||
}
|
||||
|
||||
function readSchema(schema: SchemaObject, where: string): SchemaObject {
|
||||
const unread = Object.keys(schema).find((keyword) => !readKeywords.includes(keyword))
|
||||
if (unread !== undefined) return assert.fail(`${where}: the schema holds the keyword ${unread}, which the gate does not read`)
|
||||
const extra = schema['additionalProperties']
|
||||
return extra === undefined || typeof extra === 'boolean' ? schema : assert.fail(`${where}: additionalProperties holds a schema, which the gate does not read`)
|
||||
}
|
||||
|
||||
function intersect(left: readonly Properties[], right: readonly Properties[]): Properties[] {
|
||||
return left.flatMap((own) =>
|
||||
right.map((other) => {
|
||||
const merged = new Map(own)
|
||||
for (const [name, schemas] of other) merged.set(name, [...(merged.get(name) ?? []), ...schemas])
|
||||
return merged
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
function referenced(reference: unknown, definitions: SchemaObject, where: string): SchemaObject {
|
||||
const name = typeof reference === 'string' && reference.startsWith(definitionReference) ? reference.slice(definitionReference.length) : undefined
|
||||
return schemaObject(name === undefined ? undefined : definitions[name], `${where}: the reference ${String(reference)}`)
|
||||
}
|
||||
|
||||
function branches(value: unknown, where: string): SchemaObject[] {
|
||||
if (value === undefined) return []
|
||||
return Array.isArray(value) ? value.map((branch, index) => schemaObject(branch, `${where} ${index}`)) : assert.fail(`${where} is no array of schemas`)
|
||||
}
|
||||
|
||||
function enumStrings(schema: SchemaObject, where: string): string[] {
|
||||
const values = schema['enum']
|
||||
return Array.isArray(values) ? values.filter((value: unknown): value is string => typeof value === 'string') : assert.fail(`${where}: the type property holds no enum naming the type`)
|
||||
}
|
||||
|
||||
function propertyKinds(property: SchemaObject, where: string): AttributeKind[] {
|
||||
const type = property['type']
|
||||
if (type === 'boolean') return ['boolean']
|
||||
if (type === 'integer' || type === 'number') return ['number']
|
||||
if (type === 'string') return ['string']
|
||||
if (type === 'array' || type === 'object') return ['json']
|
||||
if (type !== undefined) return assert.fail(`${where}: the schema types it ${JSON.stringify(type)}, which reads as no attribute kind`)
|
||||
const combinator = ['$ref', 'allOf', 'anyOf'].find((keyword) => property[keyword] !== undefined)
|
||||
if (combinator !== undefined) return assert.fail(`${where}: the schema holds the keyword ${combinator}, which the gate reads as no attribute kind`)
|
||||
const values = property['enum']
|
||||
return Array.isArray(values) ? values.map(valueKind) : ['json']
|
||||
}
|
||||
|
||||
function valueKind(value: unknown): AttributeKind {
|
||||
if (typeof value === 'boolean') return 'boolean'
|
||||
if (typeof value === 'number') return 'number'
|
||||
if (typeof value === 'string') return 'string'
|
||||
return 'json'
|
||||
}
|
||||
|
||||
function schemaObject(value: unknown, where: string): SchemaObject {
|
||||
return isSchemaObject(value) ? value : assert.fail(`${where} is no JSON Schema object`)
|
||||
}
|
||||
|
||||
function isSchemaObject(value: unknown): value is SchemaObject {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
|
||||
import type { AdfDocument, AdfNode } from '../adf/document.ts'
|
||||
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
|
||||
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus', 'commonmark-spec')
|
||||
const checks = ['count', 'fixpoint', 'text'] as const
|
||||
|
||||
type Check = (typeof checks)[number]
|
||||
type ExceptionKind = 'mark-model' | 'pending' | 'unspellable'
|
||||
|
||||
type SpecExample = { example: number; html: string; markdown: string; section: string }
|
||||
|
||||
type Exception = { check: Check; divergence: string; example: number; kind: ExceptionKind; reason: string }
|
||||
|
||||
type Refusal = { code: string; example: number }
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function isCheck(value: unknown): value is Check {
|
||||
return checks.some((check) => check === value)
|
||||
}
|
||||
|
||||
function isKind(value: unknown): value is ExceptionKind {
|
||||
return value === 'mark-model' || value === 'pending' || value === 'unspellable'
|
||||
}
|
||||
|
||||
function isSpecExample(value: unknown): value is SpecExample {
|
||||
if (!isRecord(value)) return false
|
||||
return typeof value['example'] === 'number' && typeof value['html'] === 'string' && typeof value['markdown'] === 'string' && typeof value['section'] === 'string'
|
||||
}
|
||||
|
||||
function isException(value: unknown): value is Exception {
|
||||
if (!isRecord(value)) return false
|
||||
return (
|
||||
isCheck(value['check']) &&
|
||||
typeof value['divergence'] === 'string' &&
|
||||
value['divergence'].length > 0 &&
|
||||
typeof value['example'] === 'number' &&
|
||||
isKind(value['kind']) &&
|
||||
typeof value['reason'] === 'string' &&
|
||||
value['reason'].length > 0
|
||||
)
|
||||
}
|
||||
|
||||
function isRefusal(value: unknown): value is Refusal {
|
||||
if (!isRecord(value)) return false
|
||||
return typeof value['code'] === 'string' && value['code'].length > 0 && typeof value['example'] === 'number'
|
||||
}
|
||||
|
||||
function readJson<T>(name: string, guard: (value: unknown) => value is T, shape: string): T[] {
|
||||
const parsed: unknown = JSON.parse(readFileSync(join(root, name), 'utf8'))
|
||||
assert.ok(Array.isArray(parsed), `${name} is not an array`)
|
||||
return parsed.map((value, index) => {
|
||||
assert.ok(guard(value), `${name} holds a ${shape} with the wrong shape at ${index}`)
|
||||
return value
|
||||
})
|
||||
}
|
||||
|
||||
const spec = readJson('spec.json', isSpecExample, 'spec example')
|
||||
const exceptions = readJson('exceptions.json', isException, 'exception')
|
||||
const refusals = readJson('refusals.json', isRefusal, 'refusal')
|
||||
|
||||
const exampleToRefusal = new Map(refusals.map((refusal) => [refusal.example, refusal.code]))
|
||||
const exceptionIndex = new Map(exceptions.map((entry) => [`${entry.example}:${entry.check}`, entry]))
|
||||
|
||||
test('the CommonMark spec suite is 0.31.2, vendored byte-exact', () => {
|
||||
const digest = createHash('sha256').update(readFileSync(join(root, 'spec.json'))).digest('hex')
|
||||
assert.equal(digest, 'd431b29d97b6f73e69d547109cf5081578fac931e72afe95639ebe766c1b2a20')
|
||||
})
|
||||
|
||||
test('every exception is unique, names a parsing example, and files a fixpoint only as unspellable', () => {
|
||||
assert.equal(exceptionIndex.size, exceptions.length, 'one exception repeats an example and check another holds')
|
||||
for (const entry of exceptions) {
|
||||
assert.ok(spec.some((candidate) => candidate.example === entry.example), `exception ${entry.example} names no example in the suite`)
|
||||
assert.equal(exampleToRefusal.get(entry.example), undefined, `exception ${entry.example} is on the refusal list, not an exception`)
|
||||
if (entry.check === 'fixpoint') assert.equal(entry.kind, 'unspellable', `exception ${entry.example} files a fixpoint divergence as ${entry.kind}; a fixable hole is given the spelling instead`)
|
||||
}
|
||||
})
|
||||
|
||||
test('the refusal list is unique per example and names real examples', () => {
|
||||
assert.equal(exampleToRefusal.size, refusals.length, 'one refusal repeats an example another holds')
|
||||
for (const example of exampleToRefusal.keys()) assert.ok(spec.some((entry) => entry.example === example), `refusal ${example} names no example in the suite`)
|
||||
})
|
||||
|
||||
// A mark is counted once per text node it touches (AGENTS.md §14).
|
||||
const countKeys = ['a', 'blockquote', 'br', 'code', 'em', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'img', 'li', 'ol', 'pre', 'strong', 'ul']
|
||||
const nodeElement: Record<string, string> = {
|
||||
blockquote: 'blockquote',
|
||||
bulletList: 'ul',
|
||||
codeBlock: 'pre',
|
||||
hardBreak: 'br',
|
||||
listItem: 'li',
|
||||
media: 'img',
|
||||
mediaInline: 'img',
|
||||
orderedList: 'ol',
|
||||
rule: 'hr',
|
||||
}
|
||||
const markElement: Record<string, string> = { code: 'code', em: 'em', link: 'a', strong: 'strong' }
|
||||
const blockTags = new Set(['blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'li', 'ol', 'p', 'pre', 'ul'])
|
||||
|
||||
function tagName(tag: string): string {
|
||||
return tag.slice(1).replace(/^\//, '').split(/[\s/>]/)[0] ?? ''
|
||||
}
|
||||
|
||||
function emptyCounts(): Record<string, number> {
|
||||
return Object.fromEntries(countKeys.map((key) => [key, 0]))
|
||||
}
|
||||
|
||||
function referenceCounts(html: string): Record<string, number> {
|
||||
const counts = emptyCounts()
|
||||
let inPre = false
|
||||
for (let index = 0; index < html.length; index += 1) {
|
||||
if (html[index] !== '<') continue
|
||||
const close = html.indexOf('>', index)
|
||||
if (close === -1) break
|
||||
const tag = html.slice(index, close + 1)
|
||||
if (tag.startsWith('</')) {
|
||||
if (tagName(tag) === 'pre') inPre = false
|
||||
index = close
|
||||
continue
|
||||
}
|
||||
const name = tagName(tag)
|
||||
if (name === 'pre') {
|
||||
inPre = true
|
||||
counts['pre'] = (counts['pre'] ?? 0) + 1
|
||||
} else if (name === 'code' && inPre) {
|
||||
// A code block's `<code>` is the `<pre>`'s body, already counted.
|
||||
} else if (countKeys.includes(name)) {
|
||||
counts[name] = (counts[name] ?? 0) + 1
|
||||
}
|
||||
index = close
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
function nodeCounts(document: AdfNode): Record<string, number> {
|
||||
const counts = emptyCounts()
|
||||
const pending: AdfNode[] = [document]
|
||||
while (pending.length > 0) {
|
||||
const node = pending.pop()
|
||||
if (node === undefined) continue
|
||||
if (node.text !== undefined) {
|
||||
const seen = new Set<string>()
|
||||
for (const mark of node.marks ?? []) {
|
||||
const element = markElement[mark.type]
|
||||
if (element !== undefined) seen.add(element)
|
||||
}
|
||||
for (const element of seen) counts[element] = (counts[element] ?? 0) + 1
|
||||
continue
|
||||
}
|
||||
if (node.type === 'heading') {
|
||||
const level = node.attrs?.['level']
|
||||
if (typeof level === 'number') counts[`h${level}`] = (counts[`h${level}`] ?? 0) + 1
|
||||
pending.push(...(node.content ?? []))
|
||||
continue
|
||||
}
|
||||
const element = nodeElement[node.type]
|
||||
if (element !== undefined) counts[element] = (counts[element] ?? 0) + 1
|
||||
pending.push(...(node.content ?? []))
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
const namedEntity: Record<string, string> = { amp: '&', gt: '>', lt: '<', ouml: 'ö', quot: '"' }
|
||||
|
||||
function decodeHtmlEntity(text: string, index: number): { length: number; text: string } | undefined {
|
||||
if (text[index] !== '&') return undefined
|
||||
const end = text.indexOf(';', index)
|
||||
if (end === -1 || end - index > 8) return undefined
|
||||
const reference = text.slice(index, end + 1)
|
||||
const named = namedEntity[reference.slice(1, -1)]
|
||||
return named === undefined ? undefined : { length: reference.length, text: named }
|
||||
}
|
||||
|
||||
test('the oracle decodes every entity the reference HTML holds', () => {
|
||||
for (const example of spec) {
|
||||
for (const [reference] of example.html.matchAll(/&#?[0-9A-Za-z]+;/g)) {
|
||||
assert.ok(decodeHtmlEntity(reference, 0) !== undefined, `example ${example.example} holds ${reference}, which the oracle would leave literal`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
function referenceText(html: string): string {
|
||||
const parts: string[] = []
|
||||
let preDepth = 0
|
||||
let atBoundary = true
|
||||
let skipNewline = false
|
||||
for (let index = 0; index < html.length; index += 1) {
|
||||
const character = html.charAt(index)
|
||||
if (character === '<') {
|
||||
const close = html.indexOf('>', index)
|
||||
if (close === -1) break
|
||||
const tag = html.slice(index, close + 1)
|
||||
const name = tagName(tag)
|
||||
if (name === 'br') {
|
||||
parts.push(' ')
|
||||
atBoundary = false
|
||||
skipNewline = true
|
||||
index = close
|
||||
continue
|
||||
}
|
||||
if (name === 'pre') {
|
||||
if (tag.startsWith('</')) {
|
||||
preDepth -= 1
|
||||
trimTrailingNewline(parts)
|
||||
} else {
|
||||
preDepth += 1
|
||||
}
|
||||
atBoundary = true
|
||||
} else {
|
||||
atBoundary = blockTags.has(name)
|
||||
}
|
||||
index = close
|
||||
continue
|
||||
}
|
||||
if (character === '\n') {
|
||||
if (skipNewline) {
|
||||
skipNewline = false
|
||||
continue
|
||||
}
|
||||
if (preDepth > 0) {
|
||||
parts.push('\n')
|
||||
continue
|
||||
}
|
||||
if (!atBoundary && !followedByBlock(html, index + 1)) parts.push(' ')
|
||||
continue
|
||||
}
|
||||
const reference = decodeHtmlEntity(html, index)
|
||||
if (reference !== undefined) {
|
||||
parts.push(reference.text)
|
||||
atBoundary = false
|
||||
index += reference.length - 1
|
||||
continue
|
||||
}
|
||||
parts.push(character)
|
||||
atBoundary = false
|
||||
}
|
||||
return parts.join('')
|
||||
}
|
||||
|
||||
function trimTrailingNewline(parts: string[]): void {
|
||||
const last = parts[parts.length - 1]
|
||||
if (last === undefined) return
|
||||
parts[parts.length - 1] = last.endsWith('\n') ? last.slice(0, -1) : last
|
||||
}
|
||||
|
||||
// A newline beside a block open or close is a boundary rather than a soft break, so it spells no space.
|
||||
function followedByBlock(html: string, index: number): boolean {
|
||||
let next = index
|
||||
while (next < html.length && (html[next] === '\n' || html[next] === ' ' || html[next] === '\t')) next += 1
|
||||
if (next >= html.length) return true
|
||||
if (html[next] !== '<') return false
|
||||
const close = html.indexOf('>', next)
|
||||
return close !== -1 && blockTags.has(tagName(html.slice(next, close + 1)))
|
||||
}
|
||||
|
||||
function concatenatedText(document: AdfNode): string {
|
||||
const parts: string[] = []
|
||||
const pending: { inCode: boolean; node: AdfNode }[] = [{ inCode: false, node: document }]
|
||||
while (pending.length > 0) {
|
||||
const frame = pending.pop()
|
||||
if (frame === undefined) continue
|
||||
const { inCode, node } = frame
|
||||
if (node.text !== undefined) {
|
||||
parts.push(inCode ? node.text : node.text.replace(/\n/g, ' '))
|
||||
continue
|
||||
}
|
||||
if (node.type === 'hardBreak') {
|
||||
parts.push(' ')
|
||||
continue
|
||||
}
|
||||
const childInCode = inCode || node.type === 'codeBlock'
|
||||
const content = node.content ?? []
|
||||
for (let index = content.length - 1; index >= 0; index -= 1) {
|
||||
const child = content[index]
|
||||
if (child !== undefined) pending.push({ inCode: childInCode, node: child })
|
||||
}
|
||||
}
|
||||
return parts.join('')
|
||||
}
|
||||
|
||||
function fixpointRefused(example: SpecExample, document: AdfDocument): string | undefined {
|
||||
const emitted = adfToMarkdown(document)
|
||||
if (!emitted.ok) return emitted.error.code
|
||||
const again = markdownToAdf(emitted.value)
|
||||
assert.ok(again.ok, `example ${example.example} emits markdown it cannot read back`)
|
||||
assert.deepEqual(again.value, document, `example ${example.example} does not hold its own round-trip`)
|
||||
return undefined
|
||||
}
|
||||
|
||||
function textMismatch(example: SpecExample, document: AdfDocument): string | undefined {
|
||||
const expected = referenceText(example.html)
|
||||
const actual = concatenatedText(document)
|
||||
return expected === actual ? undefined : `${JSON.stringify(expected)} against ${JSON.stringify(actual)}`
|
||||
}
|
||||
|
||||
function countMismatch(example: SpecExample, document: AdfDocument): string | undefined {
|
||||
const expected = referenceCounts(example.html)
|
||||
const actual = nodeCounts(document)
|
||||
const names = countKeys.filter((key) => expected[key] !== actual[key])
|
||||
return names.length === 0 ? undefined : names.map((name) => `${name} ${expected[name]}/${actual[name]}`).join(' ')
|
||||
}
|
||||
|
||||
for (const example of spec) {
|
||||
test(`CommonMark example ${example.example} => ${example.section}`, () => {
|
||||
const parse = markdownToAdf(example.markdown)
|
||||
const refused = exampleToRefusal.get(example.example)
|
||||
if (refused !== undefined) {
|
||||
assert.ok(!parse.ok, `example ${example.example} was expected to refuse with ${refused} but parsed`)
|
||||
assert.equal(parse.error.code, refused, `example ${example.example} refused with a different code`)
|
||||
return
|
||||
}
|
||||
if (!parse.ok) assert.fail(`example ${example.example} was expected to parse but refused with ${parse.error.code}`)
|
||||
|
||||
const divergences: Record<Check, string | undefined> = {
|
||||
count: countMismatch(example, parse.value),
|
||||
fixpoint: fixpointRefused(example, parse.value),
|
||||
text: textMismatch(example, parse.value),
|
||||
}
|
||||
for (const check of checks) {
|
||||
const entry = exceptionIndex.get(`${example.example}:${check}`)
|
||||
const divergence = divergences[check]
|
||||
if (divergence === undefined) {
|
||||
assert.equal(entry, undefined, `example ${example.example} passes its ${check} check but files an exception`)
|
||||
} else {
|
||||
assert.ok(entry !== undefined, `example ${example.example} ${check} check fails: ${divergence}`)
|
||||
assert.equal(entry.divergence, divergence, `example ${example.example} ${check} diverged differently than filed`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { basename, dirname, join, sep } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
|
||||
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
|
||||
import { isAdfDocument } from '../adf/document.ts'
|
||||
import { isJsonValue } from '../json-value.ts'
|
||||
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
|
||||
import { serializeCanonicalJson } from '../canonical-json.ts'
|
||||
import { toEditorNormal } from '../adf/editor-normal.ts'
|
||||
|
||||
const corpusRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus')
|
||||
const errorsRoot = join(corpusRoot, 'errors')
|
||||
const normalizationRoot = join(corpusRoot, 'normalization')
|
||||
const realPayloadsRoot = join(corpusRoot, 'real-payloads')
|
||||
const roundTripRoot = join(corpusRoot, 'round-trip')
|
||||
|
||||
const roundTripDirectories = ['block-nodes', 'combinations', 'commonmark-subset', 'inline-nodes', 'opaque-carry']
|
||||
|
||||
function directoryNames(root: string): string[] {
|
||||
return readdirSync(root, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort()
|
||||
}
|
||||
|
||||
function fixtureNames(directory: string, extension: string): string[] {
|
||||
return names(join(roundTripRoot, directory), extension)
|
||||
}
|
||||
|
||||
function names(root: string, extension: string): string[] {
|
||||
return readdirSync(root)
|
||||
.filter((name) => name.endsWith(extension))
|
||||
.map((name) => name.slice(0, -extension.length))
|
||||
.sort()
|
||||
}
|
||||
|
||||
// One kind's fixture pairs, its two tests declared with them.
|
||||
function pairedNames(root: string, first: string, second: string): string[] {
|
||||
const kind = basename(root)
|
||||
|
||||
test(`${kind} pairs every ${first} with a ${second}`, () => {
|
||||
assert.deepEqual(names(root, first), names(root, second))
|
||||
})
|
||||
|
||||
test(`${kind} holds fixtures`, () => {
|
||||
assert.ok(names(root, first).length > 0)
|
||||
})
|
||||
|
||||
return names(root, first)
|
||||
}
|
||||
|
||||
function corpusJsonPaths(): string[] {
|
||||
return readdirSync(corpusRoot, { encoding: 'utf8', recursive: true })
|
||||
.filter((name) => name.endsWith('.json'))
|
||||
.filter((name) => name !== `commonmark-spec${sep}spec.json`)
|
||||
.map((name) => join(corpusRoot, name))
|
||||
.sort()
|
||||
}
|
||||
|
||||
test('every corpus directory is a kind the runner reads', () => {
|
||||
assert.deepEqual(directoryNames(corpusRoot), ['commonmark-spec', 'errors', 'normalization', 'real-payloads', 'round-trip'])
|
||||
})
|
||||
|
||||
test('every round-trip directory is a kind the runner reads', () => {
|
||||
assert.deepEqual(directoryNames(roundTripRoot), [...roundTripDirectories].sort())
|
||||
})
|
||||
|
||||
for (const directory of roundTripDirectories) {
|
||||
const names = [...new Set([...fixtureNames(directory, '.json'), ...fixtureNames(directory, '.md')])].sort()
|
||||
|
||||
test(`${directory} pairs every .json with a .md`, () => {
|
||||
assert.ok(names.length > 0, `${directory} is expected to emit but holds no fixture pairs`)
|
||||
assert.deepEqual(fixtureNames(directory, '.json'), fixtureNames(directory, '.md'))
|
||||
})
|
||||
|
||||
for (const name of names) {
|
||||
test(`${directory}/${name} emits its markdown byte for byte`, () => {
|
||||
const parsed: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
|
||||
assert.ok(isAdfDocument(parsed), `${name}.json is not an ADF document`)
|
||||
const result = adfToMarkdown(parsed)
|
||||
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
|
||||
const expected = readFileSync(join(roundTripRoot, directory, `${name}.md`))
|
||||
const emitted = Buffer.from(result.value, 'utf8')
|
||||
if (!emitted.equals(expected)) assert.equal(result.value, expected.toString('utf8'))
|
||||
assert.ok(emitted.equals(expected))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const directory of roundTripDirectories) {
|
||||
for (const name of fixtureNames(directory, '.md')) {
|
||||
test(`${directory}/${name} reads its markdown back to the document beside it`, () => {
|
||||
const expected: unknown = JSON.parse(readFileSync(join(roundTripRoot, directory, `${name}.json`), 'utf8'))
|
||||
assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`)
|
||||
const result = markdownToAdf(readFileSync(join(roundTripRoot, directory, `${name}.md`), 'utf8'))
|
||||
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
|
||||
assert.deepEqual(toEditorNormal(result.value), expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function roundTripFixtures(): { name: string; path: string }[] {
|
||||
return roundTripDirectories.flatMap((directory) =>
|
||||
fixtureNames(directory, '.json').map((name) => ({ name: `${directory}/${name}`, path: join(roundTripRoot, directory, `${name}.json`) })),
|
||||
)
|
||||
}
|
||||
|
||||
test('no round-trip fixture repeats the document another holds', () => {
|
||||
const documents = new Map<string, string>()
|
||||
for (const fixture of roundTripFixtures()) {
|
||||
const parsed: unknown = JSON.parse(readFileSync(fixture.path, 'utf8'))
|
||||
assert.ok(isJsonValue(parsed), `${fixture.name} does not hold a JSON value`)
|
||||
const document = serializeCanonicalJson(parsed, 'compact')
|
||||
assert.equal(documents.get(document), undefined, `${fixture.name} repeats the document ${documents.get(document)} holds`)
|
||||
documents.set(document, fixture.name)
|
||||
}
|
||||
})
|
||||
|
||||
for (const name of pairedNames(normalizationRoot, '.md', '.json')) {
|
||||
test(`normalization/${name} parses to the document beside it, which emits and reads back to itself`, () => {
|
||||
const expected: unknown = JSON.parse(readFileSync(join(normalizationRoot, `${name}.json`), 'utf8'))
|
||||
assert.ok(isAdfDocument(expected), `${name}.json is not an ADF document`)
|
||||
const result = markdownToAdf(readFileSync(join(normalizationRoot, `${name}.md`), 'utf8'))
|
||||
assert.ok(result.ok, result.ok ? '' : `${result.error.code}: ${result.error.message}`)
|
||||
assert.deepEqual(toEditorNormal(result.value), expected)
|
||||
const emitted = adfToMarkdown(result.value)
|
||||
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
|
||||
const again = markdownToAdf(emitted.value)
|
||||
assert.ok(again.ok, again.ok ? '' : `${again.error.code}: ${again.error.message}`)
|
||||
assert.deepEqual(toEditorNormal(again.value), expected)
|
||||
})
|
||||
}
|
||||
|
||||
test('real-payloads holds payloads', () => {
|
||||
assert.ok(names(realPayloadsRoot, '.json').length > 0)
|
||||
})
|
||||
|
||||
for (const name of names(realPayloadsRoot, '.json')) {
|
||||
test(`real-payloads/${name} emits markdown that reads back to it`, () => {
|
||||
const payload: unknown = JSON.parse(readFileSync(join(realPayloadsRoot, `${name}.json`), 'utf8'))
|
||||
assert.ok(isAdfDocument(payload), `${name}.json is not an ADF document`)
|
||||
const emitted = adfToMarkdown(payload)
|
||||
assert.ok(emitted.ok, emitted.ok ? '' : `${emitted.error.code}: ${emitted.error.message}`)
|
||||
const parsed = markdownToAdf(emitted.value)
|
||||
assert.ok(parsed.ok, parsed.ok ? '' : `${parsed.error.code}: ${parsed.error.message}`)
|
||||
assert.deepEqual(toEditorNormal(parsed.value), payload)
|
||||
})
|
||||
}
|
||||
|
||||
// The position the input itself gives an offset, recomputed rather than trusted from the parser.
|
||||
function lineStarting(markdown: string, offset: number): { line: number; offset: number } | undefined {
|
||||
const before = markdown.slice(0, offset)
|
||||
if (offset !== 0 && !/(?:\r\n|[\n\r])$/.test(before)) return undefined
|
||||
return { line: before.split(/\r\n|[\n\r]/).length, offset }
|
||||
}
|
||||
|
||||
for (const name of pairedNames(errorsRoot, '.md', '.error')) {
|
||||
test(`errors/${name} is refused with the error it names, at a line of its own input`, () => {
|
||||
const markdown = readFileSync(join(errorsRoot, `${name}.md`), 'utf8')
|
||||
const result = markdownToAdf(markdown)
|
||||
assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '')
|
||||
assert.equal(result.error.code, readFileSync(join(errorsRoot, `${name}.error`), 'utf8').trimEnd())
|
||||
const { position } = result.error
|
||||
assert.deepEqual(position, lineStarting(markdown, position.offset))
|
||||
})
|
||||
}
|
||||
|
||||
test('the corpus holds JSON to gate', () => {
|
||||
assert.ok(corpusJsonPaths().length > 0)
|
||||
})
|
||||
|
||||
for (const path of corpusJsonPaths()) {
|
||||
test(`${path.slice(corpusRoot.length + 1)} re-serializes to itself`, () => {
|
||||
const raw = readFileSync(path, 'utf8')
|
||||
const parsed: unknown = JSON.parse(raw)
|
||||
assert.ok(isJsonValue(parsed), `${path} does not hold a JSON value`)
|
||||
assert.equal(`${serializeCanonicalJson(parsed, 'two-space')}\n`, raw)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
|
||||
import type { AttributeKind, AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
|
||||
import { blockNodes } from '../adf/block-nodes.ts'
|
||||
import { inlineNodes } from '../adf/inline-nodes.ts'
|
||||
import { markAttributes } from '../adf/mark-attributes.ts'
|
||||
import { textDirectiveName } from '../markdown/text-directive.ts'
|
||||
|
||||
type Declared = { attributes: AttributeVocabulary }
|
||||
|
||||
const specPath = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'spec', 'flavour.md')
|
||||
const introducer = 'Attributes: '
|
||||
const codeFence = /^`{3,}/
|
||||
const directiveName = /`([a-z][A-Za-z0-9]*)`/g
|
||||
const namedType = /^`([a-z][A-Za-z0-9]*)` \(([^)]*)\)/
|
||||
const owned = ' — '
|
||||
|
||||
function bullets(heading: string): string[] {
|
||||
const items: string[] = []
|
||||
let fence: string | undefined
|
||||
let item: string | undefined
|
||||
let inside = false
|
||||
for (const line of readFileSync(specPath, 'utf8').split('\n')) {
|
||||
if (line.startsWith('## ')) inside = line === `## ${heading}`
|
||||
if (!inside) continue
|
||||
const marker = codeFence.exec(line)?.[0]
|
||||
if (fence !== undefined) {
|
||||
if (marker !== undefined && marker.length >= fence.length) fence = undefined
|
||||
continue
|
||||
}
|
||||
if (marker !== undefined) {
|
||||
fence = marker
|
||||
continue
|
||||
}
|
||||
if (line.startsWith('- ')) {
|
||||
if (item !== undefined) items.push(item)
|
||||
item = line.slice(2)
|
||||
} else if (item !== undefined && line.startsWith(' ')) item += ` ${line.trim()}`
|
||||
else if (item !== undefined) {
|
||||
items.push(item)
|
||||
item = undefined
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
function declarations(heading: string): Record<string, Declared> {
|
||||
const declared: Record<string, Declared> = {}
|
||||
for (const item of bullets(heading)) {
|
||||
const cut = item.indexOf(owned)
|
||||
assert.notEqual(cut, -1, `${heading}: the bullet ${item} names no node ahead of a ${owned.trim()}`)
|
||||
const attributes = attributeList(heading, item.slice(cut))
|
||||
for (const [, name] of item.slice(0, cut).matchAll(directiveName)) {
|
||||
assert.equal(declared[name ?? ''], undefined, `${heading}: ${name ?? ''} is declared twice`)
|
||||
declared[name ?? ''] = { attributes }
|
||||
}
|
||||
}
|
||||
return declared
|
||||
}
|
||||
|
||||
function attributeList(heading: string, prose: string): AttributeVocabulary {
|
||||
const at = prose.indexOf(introducer)
|
||||
assert.notEqual(at, -1, `${heading}: ${prose} lists no attributes`)
|
||||
let rest = prose.slice(at + introducer.length)
|
||||
if (rest.startsWith('none')) return {}
|
||||
const attributes: Record<string, AttributeKind> = {}
|
||||
for (;;) {
|
||||
const pair = namedType.exec(rest)
|
||||
assert.notEqual(pair, null, `${heading}: ${rest} reads no \`name\` (type) pair`)
|
||||
attributes[pair?.[1] ?? ''] = attributeKind(heading, pair?.[2] ?? '')
|
||||
rest = rest.slice(pair?.[0].length ?? 0)
|
||||
if (!rest.startsWith(', ')) return attributes
|
||||
rest = rest.slice(2)
|
||||
}
|
||||
}
|
||||
|
||||
function attributeKind(heading: string, parenthesized: string): AttributeKind {
|
||||
const first = parenthesized.split(/[\s,]/)[0] ?? ''
|
||||
if (first === 'boolean' || first === 'json' || first === 'number' || first === 'string') return first
|
||||
assert.ok(first.startsWith('`'), `${heading}: ${first} is neither an attribute kind nor a value set`)
|
||||
return 'string'
|
||||
}
|
||||
|
||||
function vocabularies(table: Readonly<Record<string, Declared>>): Record<string, Declared> {
|
||||
return Object.fromEntries(Object.entries(table).map(([type, entry]) => [type, { attributes: entry.attributes }]))
|
||||
}
|
||||
|
||||
test('the block node table holds the attributes spec/flavour.md gives each node', () => {
|
||||
assert.deepEqual(declarations('Block nodes'), vocabularies(blockNodes))
|
||||
})
|
||||
|
||||
test('the inline node table holds the attributes spec/flavour.md gives each node', () => {
|
||||
assert.deepEqual(declarations('Inline nodes'), vocabularies(inlineNodes))
|
||||
})
|
||||
|
||||
// A name in two tables would make the position a directive is read in ambiguous.
|
||||
test('no name is spelled in more than one position', () => {
|
||||
const names = [...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...Object.keys(markAttributes), textDirectiveName]
|
||||
assert.equal(new Set(names).size, names.length)
|
||||
})
|
||||
|
||||
test('the mark table holds the attributes spec/flavour.md gives each mark', () => {
|
||||
assert.deepEqual(declarations('Marks'), vocabularies(Object.fromEntries(Object.entries(markAttributes).map(([type, attributes]) => [type, { attributes }]))))
|
||||
})
|
||||
@@ -0,0 +1,447 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fc from 'fast-check'
|
||||
import test from 'node:test'
|
||||
|
||||
import type { AdfDocument } from '../adf/document.ts'
|
||||
import type { Arbitrary, DepthIdentifier } from 'fast-check'
|
||||
import type { AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
|
||||
import type { JsonValue } from '../json-value.ts'
|
||||
import type { Result } from '../result.ts'
|
||||
import { adfDocument, attributes, jsonKey, jsonValue, markdownPieces, propertyRuns, propertyTimeout, textOf } from './property-harness.ts'
|
||||
import { adfToMarkdown } from '../markdown/emit/adf-to-markdown.ts'
|
||||
import { blockArgument, listBreakName, marksAttribute } from '../markdown/block-directive.ts'
|
||||
import { blockNodes } from '../adf/block-nodes.ts'
|
||||
import { carryName } from '../markdown/opaque-carry.ts'
|
||||
import {
|
||||
directivePrefix,
|
||||
spellAttributes,
|
||||
spellDirectiveCloser,
|
||||
spellDirectiveOpener,
|
||||
spellInlineDirectiveOpener,
|
||||
spellInlineLeafDirective,
|
||||
spellJsonAttribute,
|
||||
spellStringAttribute,
|
||||
spellVocabulary,
|
||||
} from '../markdown/directive-syntax.ts'
|
||||
import { fencedCodeBlock } from '../markdown/commonmark/backtick-runs.ts'
|
||||
import { inlineNodes } from '../adf/inline-nodes.ts'
|
||||
import { markAttributes } from '../adf/mark-attributes.ts'
|
||||
import { markSpelling } from '../markdown/mark-spellings.ts'
|
||||
import { markdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
|
||||
import { nodeContent, nodeMarks } from '../adf/document.ts'
|
||||
import { serializeCanonicalJson } from '../canonical-json.ts'
|
||||
import { textDirectiveName } from '../markdown/text-directive.ts'
|
||||
import { toEditorNormal } from '../adf/editor-normal.ts'
|
||||
import { vocabularyPairs } from '../adf/attribute-vocabulary.ts'
|
||||
|
||||
type Choice = { arbitrary: Arbitrary<string>; hostile?: true; weight: number }
|
||||
|
||||
type Edit = [at: number, removed: number, inserted: string]
|
||||
|
||||
type InlineMarkdown = { destination: Arbitrary<string>; inlines: Arbitrary<string>; label: Arbitrary<string>; oneLine: Arbitrary<string>; text: Arbitrary<string>; word: Arbitrary<string> }
|
||||
|
||||
type LeafMarkdown = { fencedCode: Arbitrary<string>; leafBlock: Arbitrary<string> }
|
||||
|
||||
const commonMarkTypes = new Set(['blockquote', 'bulletList', 'codeBlock', 'hardBreak', 'heading', 'listItem', 'orderedList', 'paragraph', 'rule', 'text'])
|
||||
const directiveShapedFloor = 330
|
||||
const fixpointFloor = 600
|
||||
const gateRuns = 1000
|
||||
const markdownMarkTypes = new Set(Object.keys(markAttributes).filter((type) => markSpelling(type)?.kind !== 'directive'))
|
||||
|
||||
const vocabularies = [...Object.values(blockNodes).map((model) => model.attributes), ...Object.values(inlineNodes).map((model) => model.attributes), ...Object.values(markAttributes)]
|
||||
const attributeKeys = [
|
||||
...new Set([...vocabularies.flatMap((vocabulary) => Object.keys(vocabulary)), ...Object.keys(blockNodes).flatMap((type) => blockArgument(type) ?? []), marksAttribute, 'json', textDirectiveName]),
|
||||
]
|
||||
const directiveNames = [...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...Object.keys(markAttributes), carryName, listBreakName, textDirectiveName]
|
||||
|
||||
// Hostile generation reaches refusals; clean generation holds none a single piece would trip, so a whole document reaches the emitter.
|
||||
function choose(hostile: boolean, choices: readonly Choice[], depth?: { depthIdentifier: DepthIdentifier; maxDepth: number }): Arbitrary<string> {
|
||||
const held = choices.filter((choice) => hostile || choice.hostile !== true).map(({ arbitrary, weight }) => ({ arbitrary, weight }))
|
||||
return depth === undefined ? fc.oneof(...held) : fc.oneof({ ...depth, depthSize: 'small' }, ...held)
|
||||
}
|
||||
|
||||
const bareToken = fc.stringMatching(/^[A-Za-z0-9_-]{1,8}$/)
|
||||
const prose = fc.stringMatching(/^[A-Za-z][a-z]{0,6}(?: [a-z]{1,6}){0,3}$/)
|
||||
const cleanText = fc.string({ maxLength: 12, unit: fc.constantFrom(...'aZ09 \t!"#$%&\'()*+,-./;=?@[\\]^_`{}~é\xa0🎉') })
|
||||
const syntaxTokens = fc.constantFrom(
|
||||
...[...String.fromCodePoint(0x0, 0xb, 0xc, 0x85, 0xa0, 0x200b, 0x2028, 0x3000, 0xfeff)],
|
||||
'\n',
|
||||
'\r\n',
|
||||
'\r',
|
||||
'\t',
|
||||
' ',
|
||||
' \n',
|
||||
'\\\n',
|
||||
'**',
|
||||
'__',
|
||||
'~~',
|
||||
'***',
|
||||
'```',
|
||||
'~~~',
|
||||
' ',
|
||||
'# ',
|
||||
'---',
|
||||
'===',
|
||||
'| ',
|
||||
' |',
|
||||
'<!--',
|
||||
'-->',
|
||||
'&',
|
||||
'&#',
|
||||
)
|
||||
const piece = fc.oneof(markdownPieces, syntaxTokens)
|
||||
|
||||
const namedEntity = fc.constantFrom('&', '<', '"', '©', ' ', 'ö', '&bogus;', '&', '&#;', '&', '≧̸')
|
||||
const numericEntity = fc
|
||||
.tuple(fc.oneof(fc.integer({ max: 0x7f, min: 0 }), fc.integer({ max: 0xffff, min: 0 }), fc.integer({ max: 0x110000, min: 0 })), fc.boolean())
|
||||
.map(([code, hex]) => (hex ? `&#x${code.toString(16)};` : `&#${code};`))
|
||||
const escape = fc.constantFrom(...'!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~a \n').map((escaped) => `\\${escaped}`)
|
||||
const backticks = fc.integer({ max: 3, min: 1 }).map((count) => '`'.repeat(count))
|
||||
const codeSpan = fc
|
||||
.tuple(backticks, fc.oneof(prose, textOf(0)), fc.oneof({ arbitrary: fc.constant(undefined), weight: 4 }, { arbitrary: backticks, weight: 1 }))
|
||||
.map(([opener, body, closer]) => `${opener}${body}${closer ?? opener}`)
|
||||
const autolink = fc.oneof(
|
||||
fc.tuple(fc.constantFrom('http://', 'https://', 'mailto:', 'ab:', 'x+y.z-:'), prose).map(([scheme, rest]) => `<${scheme}${rest.replaceAll(' ', '/')}>`),
|
||||
fc.stringMatching(/^<[a-z.+]{1,6}@[a-z-]{1,6}(?:\.[a-z]{1,4})?>$/),
|
||||
)
|
||||
const hostileAutolink = fc.tuple(fc.constantFrom('http://', 'ab:', 'a:'), textOf(0)).map(([scheme, rest]) => `<${scheme}${rest}>`)
|
||||
const inlineHtml = fc.constantFrom('<span>', '</span>', '<a href="x">', "<b class='y'/>", '<!-- c -->', '<!---->', '<?x?>', '<![CDATA[x]]>', '<!X y>', '<br/>', '<b', '<3', '< a>')
|
||||
const hardBreak = fc.constantFrom('\\\n', ' \n', '\n', spellInlineLeafDirective('hardBreak', ''))
|
||||
const spellTextDirective = (held: string) => spellInlineLeafDirective(textDirectiveName, `{${textDirectiveName}=${spellStringAttribute(held)}}`)
|
||||
const textDirective = fc.constantFrom(' ', ' ', '\t', '\n', '\n\n').map(spellTextDirective)
|
||||
const hostileTextDirective = fc.constantFrom(' \n', 'a', '').map(spellTextDirective)
|
||||
|
||||
const url = fc.stringMatching(/^https?:\/\/[a-z]{1,6}\.[a-z]{2,3}(?:\/[a-z0-9()]{0,5})?$/)
|
||||
const title = (text: Arbitrary<string>) => fc.oneof(fc.constant(''), text.map((held) => ` "${held}"`), text.map((held) => ` '${held}'`), text.map((held) => ` (${held})`))
|
||||
|
||||
const attributeValue = fc.oneof(
|
||||
{ arbitrary: bareToken, weight: 3 },
|
||||
{ arbitrary: fc.constantFrom('true', 'false', '0', '1', '3', '-1', '1.5', '1e2', '"1"', '01', 'null', '"[]"', '"{}"', '"#deebff"'), weight: 2 },
|
||||
{ arbitrary: textOf(0).map(spellStringAttribute), weight: 3 },
|
||||
{ arbitrary: jsonValue.map(spellJsonAttribute), weight: 2 },
|
||||
{ arbitrary: textOf(0).map((held) => JSON.stringify(held)), weight: 1 },
|
||||
{ arbitrary: textOf(0).map((held) => `"${held}`), weight: 1 },
|
||||
)
|
||||
const attributePairs = fc.uniqueArray(fc.tuple(fc.oneof({ arbitrary: fc.constantFrom(...attributeKeys), weight: 4 }, { arbitrary: bareToken, weight: 1 }), attributeValue), {
|
||||
maxLength: 3,
|
||||
minLength: 1,
|
||||
selector: ([key]) => key,
|
||||
})
|
||||
const hostileAttributes = fc.oneof(
|
||||
{ arbitrary: fc.constant(''), weight: 3 },
|
||||
{
|
||||
arbitrary: fc.tuple(attributePairs, fc.boolean()).map(([pairs, sorted]) => {
|
||||
const ordered = sorted ? pairs.toSorted(([left], [right]) => (left < right ? -1 : 1)) : pairs
|
||||
return `{${ordered.map(([key, value]) => `${key}=${value}`).join(' ')}}`
|
||||
}),
|
||||
weight: 6,
|
||||
},
|
||||
{ arbitrary: fc.constantFrom('{}', '{ }', '{a}', '{a=}', '{=b}', '{a=b', '{a=b c=d}', '{a="}"}'), weight: 1 },
|
||||
)
|
||||
|
||||
function tableAttributes(vocabulary: AttributeVocabulary, slot?: string): Arbitrary<string> {
|
||||
return attributes(vocabulary).map((attrs) => spellAttributes(spellVocabulary(vocabularyPairs(attrs, vocabulary, slot === undefined ? [] : [slot]) ?? [])))
|
||||
}
|
||||
|
||||
const directiveName = fc.oneof({ arbitrary: fc.constantFrom(...directiveNames), weight: 8 }, { arbitrary: fc.stringMatching(/^[A-Za-z0-9-]{1,6}$/), weight: 1 })
|
||||
const hostileArgument = fc.oneof(
|
||||
{ arbitrary: fc.constant(''), weight: 3 },
|
||||
{ arbitrary: fc.constantFrom(' info', ' warning', ' custom', ' DONE', ' TODO'), weight: 2 },
|
||||
{ arbitrary: fc.oneof(bareToken.map((held) => ` ${held}`), fc.constantFrom(' info', ' a b', ' "a"')), weight: 1 },
|
||||
)
|
||||
|
||||
function hostileOpener(name: string, argument: string, attrs: string): string {
|
||||
return `${directivePrefix}${name}${argument}${attrs === '' ? '' : ` ${attrs}`}`
|
||||
}
|
||||
|
||||
function container(opener: string, body: string, closer: string): string {
|
||||
return [opener, body, closer].filter((line) => line !== '').join('\n')
|
||||
}
|
||||
|
||||
const carriedNode = fc.oneof(
|
||||
fc.tuple(fc.constantFrom('mention', 'paragraph', 'status', 'widget'), fc.dictionary(jsonKey, jsonValue, { maxKeys: 2, noNullPrototype: true })).map(([type, attrs]): JsonValue => ({ attrs, type })),
|
||||
textOf(1).map((held): JsonValue => ({ text: held, type: 'text' })),
|
||||
)
|
||||
const spellCarry = (json: string) => spellInlineLeafDirective(carryName, `{json=${spellStringAttribute(json)}}`)
|
||||
const inlineCarry = carriedNode.map((node) => spellCarry(serializeCanonicalJson(node, 'compact')))
|
||||
const hostileInlineCarry = fc.oneof(carriedNode, jsonValue).map((node) => spellCarry(JSON.stringify(node, null, 1)))
|
||||
const blockCarry = carriedNode.map((node) => fencedCodeBlock(carryName, serializeCanonicalJson(node, 'two-space')))
|
||||
const hostileBlockCarry = fc.oneof(carriedNode, jsonValue).map((node) => fencedCodeBlock(carryName, JSON.stringify(node)))
|
||||
|
||||
function prefixLines(body: string, first: string, rest: (index: number) => string): string {
|
||||
return body
|
||||
.split('\n')
|
||||
.map((line, index) => (index === 0 ? `${first}${line}` : line === '' ? rest(index).trimEnd() : `${rest(index)}${line}`))
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
const separator = fc.oneof({ arbitrary: fc.constant('\n\n'), weight: 4 }, { arbitrary: fc.constant('\n'), weight: 3 }, { arbitrary: fc.constantFrom('\n\n\n', '\n \n', '\n\t\n'), weight: 1 })
|
||||
const quotePrefix = fc.oneof({ arbitrary: fc.constant('> '), weight: 6 }, { arbitrary: fc.constantFrom('>', ' > ', '> ', '>\t', ''), weight: 1 })
|
||||
const listMarker = fc.oneof(
|
||||
{ arbitrary: fc.constantFrom('-', '*', '+'), weight: 3 },
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(fc.oneof({ arbitrary: fc.integer({ max: 3, min: 0 }), weight: 4 }, { arbitrary: fc.integer({ max: 1000000000, min: 0 }), weight: 1 }), fc.constantFrom('.', ')'))
|
||||
.map(([start, delimiter]) => `${start}${delimiter}`),
|
||||
weight: 2,
|
||||
},
|
||||
)
|
||||
const indentDrift = fc.oneof({ arbitrary: fc.constant(0), weight: 6 }, { arbitrary: fc.integer({ max: 2, min: -2 }), weight: 1 })
|
||||
const closerDrift = fc.option(fc.oneof(directiveName.map(spellDirectiveCloser), fc.constantFrom('', `${spellDirectiveCloser('panel')} x`)), { freq: 6 })
|
||||
|
||||
function markdownOf(hostile: boolean): Arbitrary<string> {
|
||||
const inline = inlineMarkdown(hostile)
|
||||
return blockMarkdown(hostile, inline, leafBlocks(hostile, inline))
|
||||
}
|
||||
|
||||
function inlineMarkdown(hostile: boolean): InlineMarkdown {
|
||||
const inlineDepth = fc.createDepthIdentifier()
|
||||
const text = hostile ? fc.oneof(cleanText, textOf(0)) : cleanText
|
||||
const word = fc.oneof({ arbitrary: prose, weight: 3 }, { arbitrary: text.filter((held) => held !== ''), weight: 2 })
|
||||
const destination = fc.oneof(url, text, fc.stringMatching(/^<[0-9.#][a-z0-9 ]{0,6}>$/), fc.constant(''), ...(hostile ? [text.map((held) => `<${held}>`)] : []))
|
||||
const label = fc.oneof(prose, word)
|
||||
|
||||
const { inlines } = fc.letrec<{ inline: string; inlines: string }>((tie) => ({
|
||||
inline: choose(
|
||||
hostile,
|
||||
[
|
||||
{ arbitrary: word, weight: 12 },
|
||||
{ arbitrary: fc.oneof(codeSpan, autolink, namedEntity, numericEntity, escape, hardBreak, textDirective, inlineCarry), weight: 8 },
|
||||
{ arbitrary: fc.oneof(hostileAutolink, hostileInlineCarry, hostileTextDirective, inlineHtml, piece), hostile: true, weight: 4 },
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(fc.constantFrom('*', '_', '**', '__', '***', '~~', '~'), fc.constantFrom('', '', ' '), tie('inlines'), fc.constantFrom('', '', ' '), fc.option(fc.constantFrom('*', '_', '**', '~~'), { freq: 4 }))
|
||||
.map(([opener, inside, body, closing, closer]) => `${opener}${inside}${body}${closing}${closer ?? opener}`),
|
||||
weight: 4,
|
||||
},
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(fc.constantFrom('', '', hostile ? '!' : ''), tie('inlines'), fc.oneof(fc.tuple(destination, title(text)).map(([target, titled]) => `(${target}${titled})`), label.map((held) => `[${held}]`), fc.constantFrom('', '[]')))
|
||||
.map(([image, content, target]) => `${image}[${content}]${target}`),
|
||||
weight: 3,
|
||||
},
|
||||
{
|
||||
arbitrary: fc.oneof(
|
||||
...Object.entries(inlineNodes).map(([name, model]) =>
|
||||
fc
|
||||
.tuple(model.textAttribute === undefined ? fc.constant(null) : fc.option(hostile ? word : prose), tableAttributes(model.attributes, model.textAttribute))
|
||||
.map(([slot, attrs]) => (slot === null ? spellInlineLeafDirective(name, attrs) : `${spellInlineDirectiveOpener(name)}${slot}]${attrs}`)),
|
||||
),
|
||||
...Object.entries(markAttributes)
|
||||
.filter(([name]) => hostile || markSpelling(name)?.kind === 'directive')
|
||||
.map(([name, vocabulary]) => fc.tuple(tie('inlines'), tableAttributes(vocabulary)).map(([content, attrs]) => `${spellInlineDirectiveOpener(name)}${content}]${attrs}`)),
|
||||
),
|
||||
weight: 2,
|
||||
},
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(directiveName, fc.option(tie('inlines'), { freq: 3 }), hostileAttributes)
|
||||
.map(([name, content, attrs]) => `${directivePrefix}${name}${content === null ? '' : `[${content}]`}${attrs}`),
|
||||
hostile: true,
|
||||
weight: 2,
|
||||
},
|
||||
],
|
||||
{ depthIdentifier: inlineDepth, maxDepth: 3 },
|
||||
),
|
||||
inlines: fc.array(tie('inline'), { depthIdentifier: inlineDepth, maxLength: 4, minLength: 1 }).map((parts) => parts.join('')),
|
||||
}))
|
||||
return { destination, inlines, label, oneLine: inlines.map((held) => held.replace(/[\n\r]/g, ' ')), text, word }
|
||||
}
|
||||
|
||||
function leafBlocks(hostile: boolean, { destination, inlines, label, oneLine, text, word }: InlineMarkdown): LeafMarkdown {
|
||||
const fencedCode = fc
|
||||
.tuple(
|
||||
fc.constantFrom('```', '```', '~~~', '````', '``'),
|
||||
fc.oneof(fc.constant(''), bareToken, text),
|
||||
fc.array(fc.oneof(prose, text, fc.constantFrom('```', '~~~', spellDirectiveCloser('panel'), ' x')), { maxLength: 3 }),
|
||||
fc.constantFrom('', '', '`', '~', 'none'),
|
||||
)
|
||||
.map(([fence, info, lines, closer]) => [`${fence}${info}`, ...lines, ...(closer === 'none' ? [] : [`${fence}${closer}`])].join('\n'))
|
||||
const pipeTable = fc
|
||||
.record({
|
||||
body: fc.array(fc.array(oneLine, { maxLength: 3 }), { maxLength: 2 }),
|
||||
delimiter: fc.array(fc.constantFrom('---', '-', ':--', '--:', ':-:', '', '==='), { maxLength: 3, minLength: 1 }),
|
||||
header: fc.array(oneLine, { maxLength: 3, minLength: 1 }),
|
||||
leading: fc.boolean(),
|
||||
regular: hostile ? fc.boolean() : fc.constant(true),
|
||||
trailing: fc.boolean(),
|
||||
})
|
||||
.map(({ body, delimiter, header, leading, regular, trailing }) => {
|
||||
const row = (cells: readonly string[]) => (regular || leading ? `| ${cells.join(' | ')}` : cells.join(' | ')) + (regular || trailing ? ' |' : '')
|
||||
const width = (cells: readonly string[]) => (regular ? header.map((_, index) => cells[index] ?? '') : cells)
|
||||
return [row(header), row(regular ? header.map(() => '---') : delimiter), ...body.map((cells) => row(width(cells)))].join('\n')
|
||||
})
|
||||
|
||||
const leafBlock = choose(hostile, [
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(fc.integer({ max: 7, min: 1 }), fc.constantFrom(' ', ' ', '', '\t'), oneLine, fc.constantFrom('', '', ' #', '#', ' ## '))
|
||||
.map(([level, gap, content, closer]) => `${'#'.repeat(level)}${gap}${content}${closer}`),
|
||||
weight: 3,
|
||||
},
|
||||
{
|
||||
arbitrary: fc.tuple(inlines, fc.constantFrom('=', '-'), fc.integer({ max: 4, min: 1 }), fc.constantFrom('', ' ')).map(([content, underline, length, trailing]) => `${content}\n${underline.repeat(length)}${trailing}`),
|
||||
weight: 2,
|
||||
},
|
||||
{ arbitrary: fc.constantFrom('---', '***', '___', '- - -', ' * * *', '_____', '--', '*-*'), weight: 1 },
|
||||
{ arbitrary: fencedCode, weight: 2 },
|
||||
{ arbitrary: blockCarry, weight: 1 },
|
||||
{
|
||||
arbitrary: fc.tuple(fc.constantFrom(' ', '\t', ' '), fc.array(fc.oneof(prose, text), { maxLength: 3, minLength: 1 })).map(([indent, lines]) => lines.map((line) => `${indent}${line}`).join('\n')),
|
||||
weight: 1,
|
||||
},
|
||||
{ arbitrary: pipeTable, weight: 2 },
|
||||
{ arbitrary: fc.tuple(label, destination, title(text)).map(([name, target, titled]) => `[${name}]: ${target}${titled}`), weight: 1 },
|
||||
{ arbitrary: fc.tuple(word, fc.oneof(url, fc.constant(''))).map(([alt, target]) => ``), weight: 1 },
|
||||
{ arbitrary: hostileBlockCarry, hostile: true, weight: 1 },
|
||||
{
|
||||
arbitrary: fc.constantFrom('<div>\ntext\n</div>', '<!-- comment -->', '<pre>\nx\n</pre>', '<?php echo 1; ?>', '<!DOCTYPE html>', '<table>', '<custom-tag attr="1">', '</div>', '<![CDATA[\nx\n]]>'),
|
||||
hostile: true,
|
||||
weight: 1,
|
||||
},
|
||||
{
|
||||
arbitrary: fc.tuple(directiveName, hostileArgument, hostileAttributes).map(([name, argument, attrs]) => hostileOpener(name, argument, attrs)),
|
||||
hostile: true,
|
||||
weight: 1,
|
||||
},
|
||||
{
|
||||
arbitrary: fc.constantFrom(`${directivePrefix}/`, spellDirectiveCloser('panel'), `${directivePrefix}panel`, `${directivePrefix} panel`, `${directivePrefix}panel info extra`, `${directivePrefix}Panel`),
|
||||
hostile: true,
|
||||
weight: 1,
|
||||
},
|
||||
])
|
||||
return { fencedCode, leafBlock }
|
||||
}
|
||||
|
||||
function blockMarkdown(hostile: boolean, { inlines, oneLine }: InlineMarkdown, { fencedCode, leafBlock }: LeafMarkdown): Arbitrary<string> {
|
||||
const blockDepth = fc.createDepthIdentifier()
|
||||
const { blocks } = fc.letrec<{ block: string; blocks: string }>((tie) => {
|
||||
const bodyByModel = { block: fc.oneof(tie('blocks'), fc.constant('')), code: fencedCode, inline: fc.oneof(oneLine, fc.constant('')) }
|
||||
const tableDirectives = Object.entries(blockNodes).map(([name, model]) => {
|
||||
const argument =
|
||||
blockArgument(name) === undefined
|
||||
? fc.constant(undefined)
|
||||
: fc.oneof({ arbitrary: fc.constantFrom('DONE', 'TODO', 'custom', 'info', 'warning'), weight: 3 }, { arbitrary: bareToken, weight: 1 })
|
||||
const attrs = hostile ? fc.oneof({ arbitrary: tableAttributes(model.attributes), weight: 4 }, { arbitrary: hostileAttributes, weight: 1 }) : tableAttributes(model.attributes)
|
||||
if (model.contentModel === 'none') return fc.tuple(argument, attrs).map(([held, spelled]) => spellDirectiveOpener(name, held, spelled))
|
||||
return fc
|
||||
.tuple(argument, attrs, bodyByModel[model.contentModel], hostile ? closerDrift : fc.constant(null))
|
||||
.map(([held, spelled, body, closer]) => container(spellDirectiveOpener(name, held, spelled), body, closer ?? spellDirectiveCloser(name)))
|
||||
})
|
||||
return {
|
||||
block: choose(
|
||||
hostile,
|
||||
[
|
||||
{ arbitrary: fc.array(inlines, { maxLength: 3, minLength: 1 }).map((lines) => lines.join('\n')), weight: 10 },
|
||||
{ arbitrary: leafBlock, weight: 10 },
|
||||
{
|
||||
arbitrary: fc.tuple(tie('blocks'), fc.array(quotePrefix, { maxLength: 3, minLength: 1 })).map(([body, prefixes]) => prefixLines(body, '> ', (index) => prefixes[index % prefixes.length] ?? '')),
|
||||
weight: 3,
|
||||
},
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(listMarker, fc.array(fc.tuple(fc.option(listMarker, { freq: 4 }), fc.constantFrom(' ', ' ', ' ', ' ', '\t', '', ' '), tie('blocks'), indentDrift), { maxLength: 3, minLength: 1 }), separator)
|
||||
.map(([listed, items, gap]) =>
|
||||
items
|
||||
.map(([own, space, body, drift]) => {
|
||||
const marker = own ?? listed
|
||||
return prefixLines(body, `${marker}${space}`, () => ' '.repeat(Math.max(0, marker.length + space.length + drift)))
|
||||
})
|
||||
.join(gap === '\n\n' ? '\n\n' : '\n'),
|
||||
),
|
||||
weight: 4,
|
||||
},
|
||||
{ arbitrary: fc.oneof(...tableDirectives), weight: 3 },
|
||||
{
|
||||
arbitrary: fc
|
||||
.tuple(directiveName, hostileArgument, hostileAttributes, tie('blocks'), closerDrift)
|
||||
.map(([name, argument, attrs, body, closer]) => container(hostileOpener(name, argument, attrs), body, closer ?? spellDirectiveCloser(name))),
|
||||
hostile: true,
|
||||
weight: 1,
|
||||
},
|
||||
],
|
||||
{ depthIdentifier: blockDepth, maxDepth: 3 },
|
||||
),
|
||||
blocks: fc
|
||||
.array(fc.tuple(separator, tie('block')), { depthIdentifier: blockDepth, maxLength: 3, minLength: 1 })
|
||||
.map((entries) => entries.map(([gap, block], index) => (index === 0 ? block : `${gap}${block}`)).join('')),
|
||||
}
|
||||
})
|
||||
return blocks
|
||||
}
|
||||
|
||||
const cleanMarkdown = markdownOf(false)
|
||||
const hostileMarkdown = markdownOf(true)
|
||||
|
||||
const canonical = adfDocument
|
||||
.map((document) => adfToMarkdown(document))
|
||||
.filter((emitted): emitted is Extract<Result<string>, { ok: true }> => emitted.ok)
|
||||
.map((emitted) => emitted.value)
|
||||
|
||||
function edited(markdown: string, edits: readonly Edit[]): string {
|
||||
let text = markdown
|
||||
for (const [at, removed, inserted] of edits) {
|
||||
const index = at % (text.length + 1)
|
||||
text = `${text.slice(0, index)}${inserted}${text.slice(index + removed)}`
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
const edit: Arbitrary<Edit> = fc.tuple(fc.nat(), fc.nat({ max: 3 }), fc.oneof(piece, fc.constant('')))
|
||||
|
||||
const markdown = fc.oneof(
|
||||
{ arbitrary: cleanMarkdown, weight: 4 },
|
||||
{ arbitrary: hostileMarkdown, weight: 2 },
|
||||
{ arbitrary: canonical, weight: 2 },
|
||||
{ arbitrary: fc.tuple(fc.oneof(cleanMarkdown, canonical), fc.array(edit, { maxLength: 3, minLength: 1 })).map(([held, edits]) => edited(held, edits)), weight: 3 },
|
||||
{ arbitrary: fc.string({ maxLength: 40, unit: piece }), weight: 1 },
|
||||
)
|
||||
|
||||
const document = fc
|
||||
.tuple(markdown, fc.oneof({ arbitrary: fc.constant('\n'), weight: 8 }, { arbitrary: fc.constantFrom('\r\n', '\r'), weight: 1 }), fc.constantFrom('', '', '\n', ' \n'))
|
||||
.map(([held, ending, trailing]) => `${held}${trailing}`.replaceAll('\n', ending))
|
||||
|
||||
function holdsDirectiveShape(document: AdfDocument): boolean {
|
||||
const pending = [...nodeContent(document)]
|
||||
for (let node = pending.pop(); node !== undefined; node = pending.pop()) {
|
||||
if (!commonMarkTypes.has(node.type) || nodeMarks(node).some((mark) => !markdownMarkTypes.has(mark.type))) return true
|
||||
pending.push(...nodeContent(node))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
test('generated markdown refuses, or what it parses to refuses to emit, or its spelling reads back and spells itself', { timeout: propertyTimeout }, () => {
|
||||
const parameters = propertyRuns(gateRuns)
|
||||
let directiveShaped = 0
|
||||
let fixpoints = 0
|
||||
fc.assert(
|
||||
fc.property(document, (input) => {
|
||||
const parsed = markdownToAdf(input)
|
||||
if (!parsed.ok) return
|
||||
const emitted = adfToMarkdown(parsed.value)
|
||||
if (!emitted.ok) return
|
||||
fixpoints += 1
|
||||
if (holdsDirectiveShape(parsed.value)) directiveShaped += 1
|
||||
const read = markdownToAdf(emitted.value)
|
||||
assert.ok(read.ok, read.ok ? '' : `${read.error.code}: ${read.error.message} — reading ${JSON.stringify(emitted.value)}`)
|
||||
assert.deepEqual(toEditorNormal(read.value), toEditorNormal(parsed.value), `reading ${JSON.stringify(emitted.value)}`)
|
||||
const respelled = adfToMarkdown(read.value)
|
||||
assert.ok(respelled.ok, respelled.ok ? '' : `${respelled.error.code}: ${respelled.error.message} — spelling ${JSON.stringify(emitted.value)} again`)
|
||||
assert.equal(respelled.value, emitted.value)
|
||||
}),
|
||||
parameters,
|
||||
)
|
||||
if (!parameters.gate) return
|
||||
assert.ok(fixpoints >= fixpointFloor, `${fixpoints} of ${gateRuns} runs reached the fixpoint, under the floor of ${fixpointFloor}`)
|
||||
assert.ok(directiveShaped >= directiveShapedFloor, `${directiveShaped} runs reaching the fixpoint held a node or mark outside CommonMark's own types, under the floor of ${directiveShapedFloor}`)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { env } from 'node:process'
|
||||
import fc from 'fast-check'
|
||||
|
||||
import type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from '../adf/document.ts'
|
||||
import type { Arbitrary } from 'fast-check'
|
||||
import type { AttributeKind, AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
|
||||
import type { JsonValue } from '../json-value.ts'
|
||||
import { blockArgument } from '../markdown/block-directive.ts'
|
||||
import { blockNodes } from '../adf/block-nodes.ts'
|
||||
import { directivePrefix } from '../markdown/directive-syntax.ts'
|
||||
import { inlineNodes } from '../adf/inline-nodes.ts'
|
||||
import { markAttributes } from '../adf/mark-attributes.ts'
|
||||
import { toEditorNormal } from '../adf/editor-normal.ts'
|
||||
|
||||
type Positions = { block: AdfNode; inline: AdfNode }
|
||||
|
||||
const deepRunsVariable = 'PROPERTY_RUNS'
|
||||
const gateSeed = 20260914
|
||||
// Bun's test runner stops a test after five seconds unless the test sets its own timeout.
|
||||
export const propertyTimeout = 600000
|
||||
|
||||
const depthIdentifier = fc.createDepthIdentifier()
|
||||
const emptyCell: AdfNode = { content: [{ type: 'paragraph' }], type: 'tableCell' }
|
||||
const flatCommonMarkShapeWeight = 4
|
||||
export const markdownPieces = fc.constantFrom(...'aZ09 \t\n!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~é\xa0🎉', 'ab:', 'http://', directivePrefix, `${directivePrefix}a[`, `${directivePrefix}a{`)
|
||||
const nestingCommonMarkShapeWeight = 21
|
||||
const spelledTypes = new Set(['text', ...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...Object.keys(markAttributes)])
|
||||
|
||||
export function textOf(minLength: number): Arbitrary<string> {
|
||||
return fc.oneof(
|
||||
{ arbitrary: fc.string({ maxLength: 12, minLength, unit: markdownPieces }), weight: 4 },
|
||||
{ arbitrary: fc.string({ maxLength: 6, minLength, unit: 'grapheme' }), weight: 1 },
|
||||
)
|
||||
}
|
||||
|
||||
const text = textOf(1)
|
||||
const unknownType = fc.oneof(fc.stringMatching(/^[a-z][A-Za-z0-9]{0,7}$/), text).filter((type) => !spelledTypes.has(type))
|
||||
const numberValue = fc.oneof({ arbitrary: fc.integer({ max: 10, min: -1 }), weight: 3 }, { arbitrary: fc.double({ noDefaultInfinity: true, noNaN: true }), weight: 1 })
|
||||
|
||||
// V8's JSON.parse returns a wrong key after parsing a key holding an escaped backslash (https://issues.chromium.org/issues/521080746); Bun is unaffected.
|
||||
const keyPiece = fc
|
||||
.oneof({ arbitrary: markdownPieces, weight: 4 }, { arbitrary: fc.string({ maxLength: 1, minLength: 1, unit: 'grapheme' }), weight: 1 })
|
||||
.filter((piece) => !/[\\"\x00-\x1f]/.test(piece))
|
||||
export const jsonKey = fc.string({ maxLength: 8, unit: keyPiece })
|
||||
|
||||
export const { jsonValue } = fc.letrec<{ jsonValue: JsonValue }>((tie) => ({
|
||||
jsonValue: fc.oneof(
|
||||
{ depthSize: 'small', maxDepth: 2 },
|
||||
fc.oneof(fc.constant(null), fc.boolean(), numberValue, textOf(0)),
|
||||
fc.array(tie('jsonValue'), { maxLength: 3 }),
|
||||
fc.dictionary(jsonKey, tie('jsonValue'), { maxKeys: 3, noNullPrototype: true }),
|
||||
),
|
||||
}))
|
||||
|
||||
const valueByKind: Readonly<Record<AttributeKind, Arbitrary<JsonValue>>> = {
|
||||
boolean: fc.boolean(),
|
||||
json: jsonValue,
|
||||
number: numberValue,
|
||||
string: textOf(0),
|
||||
}
|
||||
|
||||
export function attributes(vocabulary: AttributeVocabulary): Arbitrary<AdfAttributes> {
|
||||
const model = Object.fromEntries(
|
||||
Object.entries(vocabulary).map(([key, kind]) => [key, fc.oneof({ arbitrary: fc.constant(undefined), weight: 2 }, { arbitrary: valueByKind[kind], weight: 1 })]),
|
||||
)
|
||||
return fc.record(model).map(heldAttributes)
|
||||
}
|
||||
|
||||
function heldAttributes(held: Readonly<Record<string, JsonValue | undefined>>): AdfAttributes {
|
||||
const attrs: AdfAttributes = {}
|
||||
for (const [key, value] of Object.entries(held)) if (value !== undefined) attrs[key] = value
|
||||
return attrs
|
||||
}
|
||||
|
||||
function pipeTable({ body, header }: { body: AdfNode[][]; header: AdfNode[] }): AdfNode {
|
||||
const rows = [header, ...body.map((cells) => header.map((_, index) => cells[index] ?? emptyCell))]
|
||||
return { content: rows.map((content): AdfNode => ({ content, type: 'tableRow' })), type: 'table' }
|
||||
}
|
||||
|
||||
const mark: Arbitrary<AdfMark> = fc.oneof(
|
||||
{ arbitrary: fc.oneof(...Object.entries(markAttributes).map(([type, vocabulary]) => attributes(vocabulary).map((attrs) => ({ attrs, type })))), weight: 9 },
|
||||
{ arbitrary: fc.record({ attrs: fc.dictionary(jsonKey, jsonValue, { maxKeys: 2, noNullPrototype: true }), type: unknownType }), weight: 1 },
|
||||
)
|
||||
const marks = fc.uniqueArray(mark, { maxLength: 3, selector: (held) => held.type })
|
||||
|
||||
const textNode = fc.record({ marks, text }).map((held): AdfNode => ({ ...held, type: 'text' }))
|
||||
|
||||
const backtickRunNode = fc
|
||||
.record({ marks: fc.oneof(fc.constant<AdfMark[]>([]), fc.constant<AdfMark[]>([{ type: 'code' }]), marks), text: fc.string({ maxLength: 6, minLength: 1, unit: fc.constantFrom('`', '``', ' ', 'a') }) })
|
||||
.map((held): AdfNode => ({ ...held, type: 'text' }))
|
||||
|
||||
const autolinkTextNode = fc
|
||||
.record({ href: fc.tuple(fc.constantFrom('ab:', 'http://'), textOf(0)).map(([scheme, rest]) => `${scheme}${rest}`), marks })
|
||||
.map(({ href, marks: held }): AdfNode => ({ marks: [...held.filter((outer) => outer.type !== 'link'), { attrs: { href }, type: 'link' }], text: href, type: 'text' }))
|
||||
|
||||
const inlineArbitraries = Object.entries(inlineNodes).map(([type, model]) =>
|
||||
fc.record({ attrs: attributes(model.attributes), marks }).map((held): AdfNode => ({ ...held, type })),
|
||||
)
|
||||
|
||||
function weighted(arbitraries: readonly Arbitrary<AdfNode>[], weight: number): { arbitrary: Arbitrary<AdfNode>; weight: number }[] {
|
||||
return arbitraries.map((arbitrary) => ({ arbitrary, weight }))
|
||||
}
|
||||
|
||||
const positions = fc.letrec<Positions>((tie) => {
|
||||
const blockContent = fc.array(tie('block'), { depthIdentifier, maxLength: 3 })
|
||||
const inlineContent = fc.array(tie('inline'), { depthIdentifier, maxLength: 4 })
|
||||
const contentByModel = {
|
||||
block: blockContent,
|
||||
code: fc.array(text.map((held): AdfNode => ({ text: held, type: 'text' })), { maxLength: 2 }),
|
||||
inline: inlineContent,
|
||||
none: fc.constant<AdfNode[]>([]),
|
||||
}
|
||||
const blockMarks = fc.oneof({ arbitrary: fc.constant<AdfMark[]>([]), weight: 4 }, { arbitrary: marks, weight: 1 })
|
||||
const blockArbitraries = Object.entries(blockNodes).map(([type, model]) => {
|
||||
const argument = blockArgument(type)
|
||||
const vocabulary: AttributeVocabulary = argument === undefined ? model.attributes : { ...model.attributes, [argument]: 'string' }
|
||||
const node = fc.record({ attrs: attributes(vocabulary), content: contentByModel[model.contentModel], marks: blockMarks }).map((held): AdfNode => ({ ...held, type }))
|
||||
return { leaf: model.contentModel === 'code' || model.contentModel === 'none', node }
|
||||
})
|
||||
const unknownNode = fc
|
||||
.record({ attrs: fc.dictionary(jsonKey, jsonValue, { maxKeys: 2, noNullPrototype: true }), content: fc.array(tie('inline'), { depthIdentifier, maxLength: 2 }), marks, type: unknownType })
|
||||
.map((held): AdfNode => held)
|
||||
const leafBlocks = blockArbitraries.filter((entry) => entry.leaf).map((entry) => entry.node)
|
||||
const containerBlocks = blockArbitraries.filter((entry) => !entry.leaf).map((entry) => entry.node)
|
||||
const misplacedWeight = 7
|
||||
const paragraph = fc.oneof({ arbitrary: inlineContent, weight: 3 }, { arbitrary: fc.array(backtickRunNode, { maxLength: 4, minLength: 2 }), weight: 1 }).map((content): AdfNode => ({ content, type: 'paragraph' }))
|
||||
const cell = (type: string) => paragraph.map((held): AdfNode => ({ content: [held], type }))
|
||||
const listItems = fc.array(
|
||||
blockContent.map((content): AdfNode => ({ content, type: 'listItem' })),
|
||||
{ depthIdentifier, maxLength: 3, minLength: 1 },
|
||||
)
|
||||
const flatCommonMarkShapes = [
|
||||
fc.record({ content: inlineContent, level: fc.integer({ max: 6, min: 1 }) }).map(({ content, level }): AdfNode => ({ attrs: { level }, content, type: 'heading' })),
|
||||
paragraph,
|
||||
fc.record({ body: fc.array(fc.array(cell('tableCell'), { maxLength: 3 }), { maxLength: 2 }), header: fc.array(cell('tableHeader'), { maxLength: 3, minLength: 1 }) }).map(pipeTable),
|
||||
]
|
||||
const nestingCommonMarkShapes = [
|
||||
blockContent.map((content): AdfNode => ({ content, type: 'blockquote' })),
|
||||
listItems.map((content): AdfNode => ({ content, type: 'bulletList' })),
|
||||
fc
|
||||
.record({ content: listItems, order: fc.oneof({ arbitrary: fc.integer({ max: 3, min: 0 }), weight: 4 }, { arbitrary: fc.integer({ max: 999999999, min: 0 }), weight: 1 }) })
|
||||
.map(({ content, order }): AdfNode => ({ attrs: { order }, content, type: 'orderedList' })),
|
||||
]
|
||||
const flatBlocks = [...weighted(leafBlocks, 2), ...weighted(flatCommonMarkShapes, flatCommonMarkShapeWeight)]
|
||||
return {
|
||||
block: fc.oneof(
|
||||
{ depthIdentifier, depthSize: 'small', maxDepth: 4 },
|
||||
{ arbitrary: fc.oneof(...flatBlocks), weight: flatBlocks.reduce((sum, entry) => sum + entry.weight, 0) },
|
||||
{ arbitrary: fc.oneof(...containerBlocks), weight: containerBlocks.length * 2 },
|
||||
{ arbitrary: fc.oneof(textNode, ...inlineArbitraries, unknownNode), weight: misplacedWeight },
|
||||
{ arbitrary: fc.oneof(...nestingCommonMarkShapes), weight: nestingCommonMarkShapes.length * nestingCommonMarkShapeWeight },
|
||||
),
|
||||
inline: fc.oneof(
|
||||
{ depthIdentifier, depthSize: 'small', maxDepth: 4 },
|
||||
{ arbitrary: textNode, weight: 12 },
|
||||
{ arbitrary: autolinkTextNode, weight: 2 },
|
||||
{ arbitrary: backtickRunNode, weight: 3 },
|
||||
{ arbitrary: fc.oneof(...inlineArbitraries), weight: 7 },
|
||||
{ arbitrary: fc.oneof(...blockArbitraries.map((entry) => entry.node), unknownNode), weight: 2 },
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
export const adfDocument = fc.array(positions.block, { depthIdentifier, maxLength: 4, minLength: 1 }).map((content): AdfDocument => toEditorNormal({ content, type: 'doc', version: 1 }))
|
||||
|
||||
export function propertyRuns(gateRuns: number): { gate: boolean; numRuns: number; seed?: number } {
|
||||
const deepRuns = env[deepRunsVariable]
|
||||
if (deepRuns === undefined) return { gate: true, numRuns: gateRuns, seed: gateSeed }
|
||||
assert.ok(/^[1-9]\d*$/.test(deepRuns), `${deepRunsVariable} is a run count in digits, such as ${deepRunsVariable}=10000: found ${JSON.stringify(deepRuns)}`)
|
||||
return { gate: false, numRuns: Number(deepRuns) }
|
||||
}
|
||||
Reference in New Issue
Block a user