5b1: name the line and the offset in the input a parse refusal sits at

This commit is contained in:
2026-09-03 14:37:39 +02:00
parent becc023b51
commit d3888129b0
10 changed files with 146 additions and 66 deletions
+13 -2
View File
@@ -192,11 +192,22 @@ for (const name of pairedNames(normalizationRoot, '.md', '.json')) {
})
}
// The position the input itself gives an offset: undefined where the offset starts no line.
function lineStarting(markdown: string, offset: number): { line: number; offset: number } | undefined {
const before = markdown.slice(0, offset)
if (offset !== 0 && !/(?:\r\n|[\n\r])$/.test(before)) return undefined
return { line: before.split(/\r\n|[\n\r]/).length, offset }
}
for (const name of pairedNames(errorsRoot, '.md', '.error')) {
test(`errors/${name} is refused with the error it names`, () => {
const result = markdownToAdf(readFileSync(join(errorsRoot, `${name}.md`), 'utf8'))
test(`errors/${name} is refused with the error it names, at a line of its own input`, () => {
const markdown = readFileSync(join(errorsRoot, `${name}.md`), 'utf8')
const result = markdownToAdf(markdown)
assert.ok(!result.ok, result.ok ? `built ${JSON.stringify(result.value)}` : '')
assert.equal(result.error.code, readFileSync(join(errorsRoot, `${name}.error`), 'utf8').trimEnd())
const { position } = result.error
assert.ok(position !== undefined, 'the refusal names no position in the input')
assert.deepEqual(position, lineStarting(markdown, position.offset))
})
}
+6 -1
View File
@@ -25,12 +25,17 @@ function path(result: Result<string>): readonly (number | string)[] {
return result.ok ? ['emitted'] : result.error.path
}
test('names the node a refusal came from', () => {
function position(result: Result<string>): unknown {
return result.ok ? 'emitted' : result.error.position
}
test('names the node a refusal came from, and no source the emitter never read', () => {
const unspellable: AdfNode = { text: 'x', type: 'paragraph' }
const list: AdfNode = { content: [{ content: [paragraph({ text: 'x', type: 'text' })], type: 'listItem' }, { content: [unspellable], type: 'listItem' }], type: 'bulletList' }
assert.deepEqual(path(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }), list))), ['content', 1, 'content', 1, 'content', 0])
assert.deepEqual(path(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }, { type: 'text' })))), ['content', 0, 'content', 1])
assert.deepEqual(path(adfToMarkdown({ type: 'doc', version: 2 })), [])
assert.equal(position(adfToMarkdown(document(paragraph({ text: 'x', type: 'text' }), list))), undefined)
})
test('refuses a value that is not an ADF document', () => {
+5 -2
View File
@@ -91,12 +91,15 @@ test('holds a directive container open until the fence that closes it', () => {
{
argument: 'info',
attributes: new Map([['panelColor', { decoded: '#ff0000', spelling: '"#ff0000"' }]]),
blocks: [{ kind: 'paragraph', text: 'Part.' }],
blocks: [{ kind: 'paragraph', position: { line: 2, offset: 37 }, text: 'Part.' }],
kind: 'directive',
name: 'panel',
position: { line: 1, offset: 0 },
},
])
assert.deepEqual(parseBlocks('::rule\n').blocks, [{ argument: undefined, attributes: new Map(), blocks: undefined, kind: 'directive', name: 'rule' }])
assert.deepEqual(parseBlocks('::rule\n').blocks, [
{ argument: undefined, attributes: new Map(), blocks: undefined, kind: 'directive', name: 'rule', position: { line: 1, offset: 0 } },
])
})
test('names the directive fence a container does not sit longer than', () => {
+70 -47
View File
@@ -1,4 +1,4 @@
import type { ConvertFault } from '../../result.ts'
import type { ConvertFault, SourcePosition } from '../../result.ts'
import type { DirectiveAttributes, DirectiveLine } from '../directive-syntax.ts'
import type { LinkDefinition } from '../link-syntax.ts'
import {
@@ -18,7 +18,7 @@ import { isPipeAlignment, isPipeDelimiter, malformedPipeTable, pipeCells } from
import { malformedDirective, readDirectiveLine } from '../directive-syntax.ts'
import { readLinkDefinitions } from './link-reference-definitions.ts'
export type Block =
export type Block = { position: SourcePosition } & (
| { argument: string | undefined; attributes: DirectiveAttributes; blocks: Block[] | undefined; kind: 'directive'; name: string }
| { blocks: Block[]; kind: 'blockquote' }
| { construct: string; kind: 'html' }
@@ -30,6 +30,7 @@ export type Block =
| { kind: 'paragraph'; text: string }
| { kind: 'rule' }
| { kind: 'table'; rows: string[][] }
)
export type ParsedBlocks = { blocks: Block[]; definitions: Map<string, LinkDefinition> }
@@ -37,26 +38,27 @@ export type DirectiveBlock = Extract<Block, { kind: 'directive' }>
type ListBlock = Extract<Block, { items: Block[][] }>
type OpenDirective = { blocks: Block[]; colons: number; index: number; kind: 'directive'; parent: Block[] }
type OpenDirective = { blocks: Block[]; colons: number; index: number; kind: 'directive'; parent: Block[]; position: SourcePosition }
type OpenContainer =
| Extract<Block, { kind: 'blockquote' }>
| OpenDirective
| { blocks: Block[]; indentation: number; kind: 'item'; list: ListBlock }
type OpenLeaf =
type OpenLeaf = { position: SourcePosition } & (
| { closer: RegExp | undefined; construct: string; kind: 'html' }
| { held: string[]; kind: 'indented-code'; lines: string[] }
| { indentation: number; info: string; kind: 'fenced-code'; lines: string[]; marker: string }
| { kind: 'paragraph'; lines: string[] }
| { kind: 'pipe-table'; rows: [string[], ...string[][]] }
)
type ContainerStart = { kind: 'blockquote'; rest: Line } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; rest: Line }
// The line from an absolute column on: a tab a cut splits keeps the stop it is measured against.
type Line = { column: number; text: string }
type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] }
type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; position: SourcePosition; stack: OpenContainer[] }
const blankLine = /^[ \t]*$/
const indentedCodeColumns = 4
@@ -65,8 +67,11 @@ const leafColons = 2
const tabStop = 4
export function parseBlocks(markdown: string): ParsedBlocks {
const walk: Walk = { blocks: [], definitions: new Map(), leaf: undefined, stack: [] }
for (const text of normalizeInput(markdown).split('\n')) readLine(walk, { column: 0, text })
const walk: Walk = { blocks: [], definitions: new Map(), leaf: undefined, position: { line: 1, offset: 0 }, stack: [] }
for (const line of sourceLines(markdown)) {
walk.position = line.position
readLine(walk, { column: 0, text: line.text })
}
closeContainers(walk, 0)
return { blocks: walk.blocks, definitions: walk.definitions }
}
@@ -127,7 +132,7 @@ function openContainers(walk: Walk, line: Line, paragraphOpen: boolean, depth: n
let opened = false
let rest = line
while (leadingColumns(rest) < indentedCodeColumns) {
const start = containerStart(rest, opened ? false : paragraphOpen, opened ? undefined : unmatched)
const start = containerStart(rest, opened ? false : paragraphOpen, opened ? undefined : unmatched, walk.position)
if (start === undefined) break
if (!opened) closeContainers(walk, depth)
opened = true
@@ -137,15 +142,15 @@ function openContainers(walk: Walk, line: Line, paragraphOpen: boolean, depth: n
return { opened, rest }
}
function containerStart(line: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined {
function containerStart(line: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined, position: SourcePosition): ContainerStart | undefined {
const opener = removeColumns(line, largestOpenerIndentation)
const blockquote = blockquoteRest(opener)
if (blockquote !== undefined) return { kind: 'blockquote', rest: blockquote }
if (isThematicBreak(opener.text)) return undefined
return itemStart(line, opener, paragraphOpen, enclosing)
return itemStart(line, opener, paragraphOpen, enclosing, position)
}
function itemStart(line: Line, opener: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined {
function itemStart(line: Line, opener: Line, paragraphOpen: boolean, enclosing: OpenContainer | undefined, position: SourcePosition): ContainerStart | undefined {
const marker = listMarker(opener.text)
if (marker === undefined) return undefined
const after: Line = { column: opener.column + marker.width, text: opener.text.slice(marker.width) }
@@ -159,19 +164,19 @@ function itemStart(line: Line, opener: Line, paragraphOpen: boolean, enclosing:
fresh: !continued,
indentation: leadingColumns(line) + marker.width + padding,
kind: 'item',
list: continued ? enclosing.list : openList(marker.start),
list: continued ? enclosing.list : openList(marker.start, position),
rest: blank ? after : removeColumns(after, padding),
}
}
function openList(start: number | undefined): ListBlock {
return start === undefined ? { items: [], kind: 'bulletList' } : { items: [], kind: 'orderedList', start }
function openList(start: number | undefined, position: SourcePosition): ListBlock {
return start === undefined ? { items: [], kind: 'bulletList', position } : { items: [], kind: 'orderedList', position, start }
}
function openContainer(walk: Walk, start: ContainerStart): void {
const blocks: Block[] = []
if (start.kind === 'blockquote') {
const blockquote: OpenContainer = { blocks, kind: 'blockquote' }
const blockquote: OpenContainer = { blocks, kind: 'blockquote', position: walk.position }
currentBlocks(walk).push(blockquote)
walk.stack.push(blockquote)
return
@@ -195,7 +200,11 @@ function closeContainers(walk: Walk, depth: number): void {
closeLeaf(walk)
for (const container of walk.stack.slice(depth)) {
if (container.kind !== 'directive') continue
container.parent[container.index] = { fault: malformedDirective(`a container fenced with ${container.colons} colons is unclosed`), kind: 'fault' }
container.parent[container.index] = {
fault: malformedDirective(`a container fenced with ${container.colons} colons is unclosed`),
kind: 'fault',
position: container.position,
}
}
dropContainers(walk, depth)
}
@@ -211,10 +220,12 @@ function openDirective(walk: Walk, directive: Extract<DirectiveLine, { kind: 'he
blocks: directive.colons > leafColons ? [] : undefined,
kind: 'directive',
name: directive.name,
position: walk.position,
}
const parent = currentBlocks(walk)
parent.push(block)
if (block.blocks !== undefined) walk.stack.push({ blocks: block.blocks, colons: directive.colons, index: parent.length - 1, kind: 'directive', parent })
const { position } = block
if (block.blocks !== undefined) walk.stack.push({ blocks: block.blocks, colons: directive.colons, index: parent.length - 1, kind: 'directive', parent, position })
}
function applyDirectiveLine(walk: Walk, directive: DirectiveLine): void {
@@ -251,7 +262,7 @@ function innermostDirective(walk: Walk): { container: OpenDirective; depth: numb
}
function pushFault(walk: Walk, fault: ConvertFault): void {
currentBlocks(walk).push({ fault, kind: 'fault' })
currentBlocks(walk).push({ fault, kind: 'fault', position: walk.position })
}
// A claimed line ends the lazy continuation CommonMark would fold it into (spec/flavour.md).
@@ -291,7 +302,7 @@ function readBlockLine(walk: Walk, line: Line): void {
return
}
if (walk.leaf === undefined && leadingColumns(line) >= indentedCodeColumns) {
walk.leaf = { held: [], kind: 'indented-code', lines: [removeColumns(line, indentedCodeColumns).text] }
walk.leaf = { held: [], kind: 'indented-code', lines: [removeColumns(line, indentedCodeColumns).text], position: walk.position }
return
}
openLeaf(walk, line)
@@ -320,14 +331,14 @@ function openLeaf(walk: Walk, line: Line): void {
const cells = pipeCells(opener)
if (cells !== undefined) {
closeLeaf(walk)
walk.leaf = { kind: 'pipe-table', rows: [cells] }
walk.leaf = { kind: 'pipe-table', position: walk.position, rows: [cells] }
return
}
if (readLineBlock(walk, opener)) return
const fence = openingCodeFence(opener)
if (fence !== undefined) {
closeLeaf(walk)
walk.leaf = { indentation: leadingColumns(line), info: fence.info, kind: 'fenced-code', lines: [], marker: fence.marker }
walk.leaf = { indentation: leadingColumns(line), info: fence.info, kind: 'fenced-code', lines: [], marker: fence.marker, position: walk.position }
return
}
const html = openingHtmlBlock(opener, walk.leaf?.kind === 'paragraph')
@@ -336,7 +347,7 @@ function openLeaf(walk: Walk, line: Line): void {
return
}
closeLeaf(walk)
walk.leaf = { closer: html.closer, construct: html.construct, kind: 'html' }
walk.leaf = { closer: html.closer, construct: html.construct, kind: 'html', position: walk.position }
if (html.closer?.test(line.text) === true) closeLeaf(walk)
}
@@ -344,21 +355,21 @@ function openLeaf(walk: Walk, line: Line): void {
function readLineBlock(walk: Walk, opener: string): boolean {
const level = walk.leaf?.kind === 'paragraph' ? setextHeadingLevel(opener) : undefined
if (level !== undefined) {
const text = takeParagraph(walk)
if (text !== undefined) {
currentBlocks(walk).push({ kind: 'heading', level, text })
const paragraph = takeParagraph(walk)
if (paragraph !== undefined) {
currentBlocks(walk).push({ kind: 'heading', level, position: paragraph.position, text: paragraph.text })
return true
}
}
if (isThematicBreak(opener)) {
closeLeaf(walk)
currentBlocks(walk).push({ kind: 'rule' })
currentBlocks(walk).push({ kind: 'rule', position: walk.position })
return true
}
const heading = atxHeading(opener)
if (heading === undefined) return false
closeLeaf(walk)
currentBlocks(walk).push({ kind: 'heading', level: heading.level, text: heading.text })
currentBlocks(walk).push({ kind: 'heading', level: heading.level, position: walk.position, text: heading.text })
return true
}
@@ -366,57 +377,69 @@ function appendParagraph(walk: Walk, line: string): void {
const leaf = walk.leaf
const text = line.replace(/^[ \t]+/, '')
if (leaf?.kind === 'paragraph') leaf.lines.push(text)
else walk.leaf = { kind: 'paragraph', lines: [text] }
else walk.leaf = { kind: 'paragraph', lines: [text], position: walk.position }
}
function closeLeaf(walk: Walk): void {
const leaf = walk.leaf
if (leaf === undefined) return
if (leaf.kind === 'paragraph') {
const text = takeParagraph(walk)
if (text !== undefined) currentBlocks(walk).push({ kind: 'paragraph', text })
const paragraph = takeParagraph(walk)
if (paragraph !== undefined) currentBlocks(walk).push(paragraph)
return
}
walk.leaf = undefined
if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html' })
else if (leaf.kind === 'pipe-table') currentBlocks(walk).push(pipeTableBlock(leaf.rows))
else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? decodeTextEscapes(leaf.info) : '', text: leaf.lines.join('\n') })
const { position } = leaf
if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html', position })
else if (leaf.kind === 'pipe-table') currentBlocks(walk).push(pipeTableBlock(leaf.rows, position))
else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? decodeTextEscapes(leaf.info) : '', position, text: leaf.lines.join('\n') })
}
function pipeTableBlock(rows: readonly [string[], ...string[][]]): Block {
function pipeTableBlock(rows: readonly [string[], ...string[][]], position: SourcePosition): Block {
const [header, delimiter, ...body] = rows
if (delimiter !== undefined && delimiter.some(isPipeAlignment)) {
return faultedBlock('a pipe table carries no column alignment ADF could hold')
return faultedBlock('a pipe table carries no column alignment ADF could hold', position)
}
if (delimiter === undefined || !delimiter.every(isPipeDelimiter)) {
return faultedBlock('a pipe table underlines its header with a row of `-` runs')
return faultedBlock('a pipe table underlines its header with a row of `-` runs', position)
}
const ragged = [delimiter, ...body].find((row) => row.length !== header.length)
if (ragged !== undefined) return faultedBlock(`a pipe table row holds ${ragged.length} cells where its header holds ${header.length}`)
return { kind: 'table', rows: [header, ...body] }
if (ragged !== undefined) return faultedBlock(`a pipe table row holds ${ragged.length} cells where its header holds ${header.length}`, position)
return { kind: 'table', position, rows: [header, ...body] }
}
function faultedBlock(message: string): Block {
return { fault: malformedPipeTable(message), kind: 'fault' }
function faultedBlock(message: string, position: SourcePosition): Block {
return { fault: malformedPipeTable(message), kind: 'fault', position }
}
function takeParagraph(walk: Walk): string | undefined {
function takeParagraph(walk: Walk): Extract<Block, { kind: 'paragraph' }> | undefined {
const leaf = walk.leaf
if (leaf?.kind !== 'paragraph') return undefined
walk.leaf = undefined
const text = readLinkDefinitions(walk.definitions, leaf.lines.join('\n'))
return text === '' ? undefined : text
return text === '' ? undefined : { kind: 'paragraph', position: leaf.position, text }
}
function currentBlocks(walk: Walk): Block[] {
return walk.stack.at(-1)?.blocks ?? walk.blocks
}
function normalizeInput(markdown: string): string {
return markdown
.replace(/\r\n?/g, '\n')
.replaceAll('\u0000', '\ufffd')
.replace(/\n$/, '')
// The line ending stays as the input spells it: an offset indexes the string the caller passed.
function sourceLines(markdown: string): { position: SourcePosition; text: string }[] {
const source = markdown.replaceAll('\u0000', '\ufffd')
const lines: { position: SourcePosition; text: string }[] = []
let line = 1
let start = 0
for (let index = 0; index < source.length; index += 1) {
const character = source.charAt(index)
if (character !== '\n' && character !== '\r') continue
lines.push({ position: { line, offset: start }, text: source.slice(start, index) })
if (character === '\r' && source.charAt(index + 1) === '\n') index += 1
line += 1
start = index + 1
}
if (start < source.length) lines.push({ position: { line, offset: start }, text: source.slice(start) })
return lines
}
function leadingColumns(line: Line): number {
+19 -1
View File
@@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
import test from 'node:test'
import type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from '../../adf/document.ts'
import type { Result } from '../../result.ts'
import type { Result, SourcePosition } from '../../result.ts'
import { largestNesting } from '../../nesting.ts'
import { markdownToAdf } from './markdown-to-adf.ts'
@@ -23,6 +23,11 @@ function path(result: Result<AdfDocument>): readonly (number | string)[] {
return result.ok ? ['built'] : result.error.path
}
function position(result: Result<AdfDocument>): SourcePosition | string {
if (result.ok) return 'built'
return result.error.position ?? 'no position'
}
function text(value: string): AdfNode {
return { text: value, type: 'text' }
}
@@ -443,6 +448,19 @@ test('swallows an HTML block ahead of the claim a line inside it would make', ()
assert.equal(code(markdownToAdf('<div>\n| x |\n</div>\n')), 'unmappable-html')
})
test('names the line and the offset in the input a refusal sits at, the innermost block winning', () => {
assert.deepEqual(position(markdownToAdf('<div>\n')), { line: 1, offset: 0 })
assert.deepEqual(position(markdownToAdf('Part.\n\n<div>\n')), { line: 3, offset: 7 })
assert.deepEqual(position(markdownToAdf('> Part.\n>\n> a <span>b</span>\n')), { line: 3, offset: 10 })
assert.deepEqual(position(markdownToAdf('- Part.\n- a <span>b</span>\n')), { line: 2, offset: 8 })
assert.deepEqual(position(markdownToAdf('Part.\n\n:::panel info\nMore.\n')), { line: 3, offset: 7 })
assert.deepEqual(position(markdownToAdf('x\n\na <span>b</span>\n===\n')), { line: 3, offset: 3 })
assert.deepEqual(position(markdownToAdf('x\n\n```adf\n{\n```\n')), { line: 3, offset: 3 })
assert.deepEqual(position(markdownToAdf('x\n\n| a |\n')), { line: 3, offset: 3 })
assert.deepEqual(position(markdownToAdf('a\nb <span>c</span>\n')), { line: 1, offset: 0 })
assert.deepEqual(position(markdownToAdf('Part.\r\n\r\n<div>\r\n')), { line: 3, offset: 9 })
})
test('gives up the link reference definitions a paragraph opens with', () => {
assert.deepEqual(content(markdownToAdf('[a]: /url\n')), [])
assert.deepEqual(content(markdownToAdf('[a]: /url\n[b]: /other\nPart.\n')), [paragraph('Part.')])
+2 -2
View File
@@ -4,7 +4,7 @@ import type { BlockDirectiveNode } from './directive-nodes.ts'
import type { LinkDefinitions } from './inline-content.ts'
import { carryName, readCarriedBlock } from '../opaque-carry.ts'
import { commonMarkSpelling } from '../emit/adf-to-markdown.ts'
import { failure, faulted, success, type ConvertErrorPath, type Result } from '../../result.ts'
import { failure, faulted, positioned, success, type ConvertErrorPath, type Result } from '../../result.ts'
import { languageSlot } from '../code-language.ts'
import { largestNesting } from '../../nesting.ts'
import { parseBlocks } from './blocks.ts'
@@ -22,7 +22,7 @@ function blockNodes(blocks: readonly Block[], definitions: LinkDefinitions, path
if (depth > largestNesting) return failure('unsupported-nesting-depth', `the input nests deeper than the ${largestNesting} levels the parser carries`, path)
const content: AdfNode[] = []
for (const [index, block] of blocks.entries()) {
const node = blockNode(block, definitions, [...path, 'content', index], depth)
const node = positioned(blockNode(block, definitions, [...path, 'content', index], depth), block.position)
if (!node.ok) return node
content.push(node.value)
}
+9 -1
View File
@@ -17,13 +17,16 @@ export type ConvertErrorCode =
export type ConvertErrorPath = readonly (number | string)[]
export type SourcePosition = { line: number; offset: number }
export type ConvertError = {
code: ConvertErrorCode
message: string
path: ConvertErrorPath
position?: SourcePosition
}
export type ConvertFault = Omit<ConvertError, 'path'>
export type ConvertFault = Omit<ConvertError, 'path' | 'position'>
export type Result<T> = { error: ConvertError; ok: false } | { ok: true; value: T }
@@ -35,6 +38,11 @@ export function faulted<T>(fault: ConvertFault, path: ConvertErrorPath): Result<
return failure(fault.code, fault.message, path)
}
export function positioned<T>(result: Result<T>, position: SourcePosition): Result<T> {
if (result.ok || result.error.position !== undefined) return result
return { error: { ...result.error, position }, ok: false }
}
export function success<T>(value: T): Result<T> {
return { ok: true, value }
}