5b1: guarantee the parse direction's position in the type, and name it one level deeper

src/markdown/commonmark-grammar.ts

@@ -1,4 +1,4 @@
-import { readEntityReference } from './entity-references.ts'
+import { readEntityReference, replacementCharacter } from './entity-references.ts'
 
 export type LinePosition = 'first' | 'later'
 
@@ -27,6 +27,7 @@ const bracketedAutolink = new RegExp(`<(?:${autolinkSource})>`, 'y')
 const controlCharacter = new RegExp(`[${controlCharacterRange}]`)
 const htmlTag = new RegExp(htmlTagSource, 'y')
 const nullCharacter = new RegExp(nullCharacterSource)
+const nullCharacters = new RegExp(nullCharacterSource, 'g')
 const tagName = new RegExp(`^</?(${tagNameSource})[\\s\\S]*$`)
 // The opener's own match ends with the terminator where the construct is complete on its own (`<!-->`).
 const inlineHtmlConstructs = [
@@ -137,6 +138,10 @@ export function holdsNullCharacter(text: string): boolean {
   return nullCharacter.test(text)
 }
 
+export function replaceNullCharacters(text: string): string {
+  return text.replaceAll(nullCharacters, replacementCharacter)
+}
+
 function htmlTagName(text: string): string {
   return text.replace(tagName, '<$1>')
 }

src/markdown/entity-references.ts

@@ -5,7 +5,7 @@ const anchoredEntityReference = new RegExp(`(?:${entityReferenceSource})`, 'y')
 const decimalReference = /^&#(\d+);/
 const hexadecimalReference = /^&#[Xx]([A-Fa-f0-9]+);/
 const largestCodePoint = 0x10ffff
-const replacementCharacter = '\ufffd'
+export const replacementCharacter = '\ufffd'
 const surrogates = { first: 0xd800, last: 0xdfff }
 
 // HTML5's named character references (https://html.spec.whatwg.org/entities.json), the semicolon-terminated half

src/markdown/parse/blocks.ts

@@ -12,6 +12,7 @@ import {
   markerInterruptsParagraph,
   openingCodeFence,
   openingHtmlBlock,
+  replaceNullCharacters,
   setextHeadingLevel,
 } from '../commonmark-grammar.ts'
 import { isPipeAlignment, isPipeDelimiter, malformedPipeTable, pipeCells } from '../pipe-table-syntax.ts'
@@ -424,24 +425,26 @@ function currentBlocks(walk: Walk): Block[] {
   return walk.stack.at(-1)?.blocks ?? walk.blocks
 }
 
-// The line ending stays as the input spells it: an offset indexes the string the caller passed.
 function sourceLines(markdown: string): { position: SourcePosition; text: string }[] {
-  const source = markdown.replaceAll('\u0000', '\ufffd')
   const lines: { position: SourcePosition; text: string }[] = []
   let line = 1
   let start = 0
-  for (let index = 0; index < source.length; index += 1) {
-    const character = source.charAt(index)
+  for (let index = 0; index < markdown.length; index += 1) {
+    const character = markdown.charAt(index)
     if (character !== '\n' && character !== '\r') continue
-    lines.push({ position: { line, offset: start }, text: source.slice(start, index) })
-    if (character === '\r' && source.charAt(index + 1) === '\n') index += 1
+    lines.push(sourceLine(markdown, line, start, index))
+    if (character === '\r' && markdown.charAt(index + 1) === '\n') index += 1
     line += 1
     start = index + 1
   }
-  if (start < source.length) lines.push({ position: { line, offset: start }, text: source.slice(start) })
+  if (start < markdown.length) lines.push(sourceLine(markdown, line, start, markdown.length))
   return lines
 }
 
+function sourceLine(markdown: string, line: number, start: number, end: number): { position: SourcePosition; text: string } {
+  return { position: { line, offset: start }, text: replaceNullCharacters(markdown.slice(start, end)) }
+}
+
 function leadingColumns(line: Line): number {
   let columns = 0
   for (const character of line.text) {

src/markdown/parse/markdown-to-adf.test.ts

@@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
 import test from 'node:test'
 
 import type { AdfAttributes, AdfDocument, AdfMark, AdfNode } from '../../adf/document.ts'
-import type { Result, SourcePosition } from '../../result.ts'
+import type { ParseError, Result, SourcePosition } from '../../result.ts'
 import { largestNesting } from '../../nesting.ts'
 import { markdownToAdf } from './markdown-to-adf.ts'
 
@@ -23,9 +23,8 @@ function path(result: Result<AdfDocument>): readonly (number | string)[] {
   return result.ok ? ['built'] : result.error.path
 }
 
-function position(result: Result<AdfDocument>): SourcePosition | string {
-  if (result.ok) return 'built'
-  return result.error.position ?? 'no position'
+function position(result: Result<AdfDocument, ParseError>): SourcePosition | string {
+  return result.ok ? 'built' : result.error.position
 }
 
 function text(value: string): AdfNode {
@@ -459,6 +458,9 @@ test('names the line and the offset in the input a refusal sits at, the innermos
   assert.deepEqual(position(markdownToAdf('x\n\n| a |\n')), { line: 3, offset: 3 })
   assert.deepEqual(position(markdownToAdf('a\nb <span>c</span>\n')), { line: 1, offset: 0 })
   assert.deepEqual(position(markdownToAdf('Part.\r\n\r\n<div>\r\n')), { line: 3, offset: 9 })
+  assert.deepEqual(position(markdownToAdf('a\u0000b\n\n<div>\n')), { line: 3, offset: 5 })
+  assert.deepEqual(position(markdownToAdf(':::caption\na <span>b</span>\n:::\n')), { line: 2, offset: 11 })
+  assert.deepEqual(position(markdownToAdf('x\n\n:::caption\n- a\n:::\n')), { line: 3, offset: 3 })
 })
 
 test('gives up the link reference definitions a paragraph opens with', () => {

src/markdown/parse/markdown-to-adf.ts

@@ -4,16 +4,18 @@ import type { BlockDirectiveNode } from './directive-nodes.ts'
 import type { LinkDefinitions } from './inline-content.ts'
 import { carryName, readCarriedBlock } from '../opaque-carry.ts'
 import { commonMarkSpelling } from '../emit/adf-to-markdown.ts'
-import { failure, faulted, positioned, success, type ConvertErrorPath, type Result } from '../../result.ts'
+import { failure, faulted, positioned, success, type ConvertErrorPath, type ParseError, type Result, type SourcePosition } from '../../result.ts'
 import { languageSlot } from '../code-language.ts'
 import { largestNesting } from '../../nesting.ts'
 import { parseBlocks } from './blocks.ts'
 import { parseInlineContent } from './inline-content.ts'
 import { readBlockDirectiveNode } from './directive-nodes.ts'
 
-export function markdownToAdf(markdown: string): Result<AdfDocument> {
+const documentStart: SourcePosition = { line: 1, offset: 0 }
+
+export function markdownToAdf(markdown: string): Result<AdfDocument, ParseError> {
   const parsed = parseBlocks(markdown)
-  const content = blockNodes(parsed.blocks, parsed.definitions, [], 0)
+  const content = positioned(blockNodes(parsed.blocks, parsed.definitions, [], 0), documentStart)
   if (!content.ok) return content
   return success(content.value.length === 0 ? { type: 'doc', version: 1 } : { content: content.value, type: 'doc', version: 1 })
 }
@@ -111,7 +113,7 @@ function inlineBodyNode(node: AdfNode, blocks: readonly Block[], definitions: Li
   if (blocks.length === 0) return failure('unsupported-node-shape', `an empty ${node.type} takes the leaf form, ::`, path)
   const only = blocks.length === 1 ? blocks[0] : undefined
   if (only?.kind !== 'paragraph') return failure('unsupported-node-shape', `${node.type} takes one paragraph as its body`, path)
-  return contentNode(node, only.text, definitions, path)
+  return positioned(contentNode(node, only.text, definitions, path), only.position)
 }
 
 function containerNode(node: AdfNode, blocks: readonly Block[], definitions: LinkDefinitions, path: ConvertErrorPath, depth: number): Result<AdfNode> {