AGENTS.md

@@ -138,6 +138,9 @@ live Atlassian APIs; property-generated ADF trees; the CommonMark spec suite aga
 - Nothing recurses unbounded: the guards walk iteratively, and blocks, marks and JSON values — an
   attribute's and a carried node's alike — are all held to 500 levels, so a deep document is a
   `Result` rather than the stack overflow that waits near 2000.
+- A reader takes the text and an index — a sticky regex, `indexOf` — never a fresh slice per
+  character, and a per-character walk hoists what does not vary with the character. The pipeline
+  persona feeds documents nobody typed, so an ordinary megabyte stays milliseconds.
 - No casts: `as`, `as unknown as`, non-null `!`. A boundary owes a type guard validating the
   fields it claims (`isAdfDocument`); past it everything is typed. Make invalid states
   unrepresentable.
@@ -177,8 +180,8 @@ One-line commit messages and PR titles; short PR summaries. No AI-attribution ma
 No wiki markup (§1), no network or filesystem I/O, no name→id resolution (§3), no ADF schema
 validation or exported validator — a refusal that keeps the round-trip is not schema validation,
 so the one a spelled node carrying the same mark type twice earns stays, no shipped CSS (§4), no
-streaming APIs, no performance budget — real documents are kilobytes. A CLI is a later goal
+streaming APIs, no performance budget past §11's linear scan — nothing here is tuned, and no
-(`todo.md`), not a non-goal.
+figure is promised. A CLI is a later goal (`todo.md`), not a non-goal.
 
 ## 15. The working loop
 

src/markdown/commonmark-grammar.ts

@@ -1,14 +1,12 @@
 import { readEntityReference } from './entity-references.ts'
 
-export type HtmlConstruct = { length: number; name: string }
-
 export type LinePosition = 'first' | 'later'
 
-export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
+type OpenHtmlBlock = { closer: RegExp | undefined; construct: string }
 
 type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp }
 
-export const htmlConstructNames = {
+const htmlConstructNames = {
   cdata: 'a CDATA section',
   comment: 'an HTML comment',
   declaration: 'an HTML declaration',
@@ -22,19 +20,20 @@ const tagNameSource = '[A-Za-z][A-Za-z0-9-]*'
 const htmlSpaceSource = '[ \\t\\n]'
 const attributeSource = `(?:${htmlSpaceSource}+[A-Za-z_:][A-Za-z0-9_.:-]*(?:${htmlSpaceSource}*=${htmlSpaceSource}*(?:[^ \\t\\n"'=<>\`]+|'[^']*'|"[^"]*"))?)`
 
-export const htmlTagSource = `(?:<${tagNameSource}${attributeSource}*${htmlSpaceSource}*/?>|</${tagNameSource}${htmlSpaceSource}*>)`
+const htmlTagSource = `(?:<${tagNameSource}${attributeSource}*${htmlSpaceSource}*/?>|</${tagNameSource}${htmlSpaceSource}*>)`
 
 const autolink = new RegExp(`^(?:${autolinkSource})$`)
-const bracketedAutolink = new RegExp(`^<(?:${autolinkSource})>`)
+const bracketedAutolink = new RegExp(`<(?:${autolinkSource})>`, 'y')
 const controlCharacter = new RegExp(`[${controlCharacterRange}]`)
-const htmlTag = new RegExp(`^${htmlTagSource}`)
+const htmlTag = new RegExp(htmlTagSource, 'y')
 const nullCharacter = new RegExp(nullCharacterSource)
 const tagName = new RegExp(`^</?(${tagNameSource})[\\s\\S]*$`)
+// The opener's own match ends with the terminator where the construct is complete on its own (`<!-->`).
 const inlineHtmlConstructs = [
-  { name: htmlConstructNames.cdata, pattern: /^<!\[CDATA\[[\s\S]*?\]\]>/ },
+  { name: htmlConstructNames.cdata, opener: /<!\[CDATA\[/y, terminator: ']]>' },
-  { name: htmlConstructNames.comment, pattern: /^(?:<!-->|<!--->|<!--[\s\S]*?-->)/ },
+  { name: htmlConstructNames.comment, opener: /<!(?:--->|-->|--)/y, terminator: '-->' },
-  { name: htmlConstructNames.declaration, pattern: /^<![A-Za-z][^>]*>/ },
+  { name: htmlConstructNames.declaration, opener: /<![A-Za-z]/y, terminator: '>' },
-  { name: htmlConstructNames.processingInstruction, pattern: /^<\?[\s\S]*?\?>/ },
+  { name: htmlConstructNames.processingInstruction, opener: /<\?/y, terminator: '?>' },
 ]
 // CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists.
 const blockTagNames =
@@ -57,7 +56,7 @@ const pipeClaim = /^\|/
 const bulletListOpener = /^[*+-](?:[ \t]|$)/
 // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip.
 const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim]
-const emailAutolink = /^<[^\s<>@]+@[^\s<>@]+>/
+const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y
 const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/
 const setextUnderline = /^(=+|-+)[ \t]*$/
 const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
@@ -105,7 +104,7 @@ export function decodeTextEscapes(text: string): string {
       index += 2
       continue
     }
-    const reference = text.charAt(index) === '&' ? readEntityReference(text.slice(index)) : undefined
+    const reference = readEntityReference(text, index)
     if (reference !== undefined) {
       decoded += reference.text
       index += reference.length
@@ -135,17 +134,21 @@ export function holdsNullCharacter(text: string): boolean {
   return nullCharacter.test(text)
 }
 
-export function htmlTagName(text: string): string {
+function htmlTagName(text: string): string {
   return text.replace(tagName, '<$1>')
 }
 
-export function inlineHtmlConstruct(text: string): HtmlConstruct | undefined {
+export function inlineHtmlConstruct(text: string, index: number): string | undefined {
   for (const construct of inlineHtmlConstructs) {
-    const matched = construct.pattern.exec(text)?.[0]
+    construct.opener.lastIndex = index
-    if (matched !== undefined) return { length: matched.length, name: construct.name }
+    const opened = construct.opener.exec(text)?.[0]
+    if (opened === undefined) continue
+    if (opened.endsWith(construct.terminator)) return construct.name
+    return text.includes(construct.terminator, index + opened.length) ? construct.name : undefined
   }
+  htmlTag.lastIndex = index
   const tag = htmlTag.exec(text)?.[0]
-  return tag === undefined ? undefined : { length: tag.length, name: htmlTagName(tag) }
+  return tag === undefined ? undefined : htmlTagName(tag)
 }
 
 export function isAsciiPunctuation(character: string): boolean {
@@ -193,11 +196,13 @@ export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlB
   return undefined
 }
 
-export function opensBracketedAutolink(text: string): boolean {
+export function opensBracketedAutolink(text: string, index: number): boolean {
+  bracketedAutolink.lastIndex = index
   return bracketedAutolink.test(text)
 }
 
-export function opensEmailAutolink(text: string): boolean {
+export function opensEmailAutolink(text: string, index: number): boolean {
+  emailAutolink.lastIndex = index
   return emailAutolink.test(text)
 }
 

src/markdown/directive-attributes.ts

@@ -3,7 +3,7 @@ import type { VocabularyPair } from '../adf/attribute-vocabulary.ts'
 import { serializeCanonicalJson } from '../canonical-json.ts'
 
 const bareToken = /^[A-Za-z0-9_-]+$/
-const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/
+const inlineDirectiveOpener = /:[a-z][A-Za-z0-9]*[[{]/y
 
 // spec/flavour.md, Attributes.
 const quotedEscapes = /[&<`|]/g
@@ -12,7 +12,8 @@ export function isBareToken(text: string): boolean {
   return bareToken.test(text)
 }
 
-export function opensInlineDirective(text: string): boolean {
+export function opensInlineDirective(text: string, index: number): boolean {
+  inlineDirectiveOpener.lastIndex = index
   return inlineDirectiveOpener.test(text)
 }
 

src/markdown/emit/adf-to-markdown.test.ts

@@ -216,6 +216,8 @@ test('escapes only text that would otherwise open a construct', () => {
   assert.equal(emitted('a <b@c.d> e'), 'a \\<b@c.d> e\n')
   assert.equal(emitted('a <b 2'), 'a <b 2\n')
   assert.equal(emitted('a <div b'), 'a <div b\n')
+  assert.equal(emitted('a <!-- b'), 'a <!-- b\n')
+  assert.equal(emitted('a <!-- b --> c'), 'a \\<!-- b --> c\n')
   const later = paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, { text: '<div', type: 'text' })
   assert.equal(markdown(adfToMarkdown(document(later))), 'a\\\n\\<div\n')
   assert.equal(markdown(adfToMarkdown(document({ content: [paragraph({ text: '<!-- x', type: 'text' })], type: 'blockquote' }))), '> \\<!-- x\n')

src/markdown/emit/line-escaping.ts

@@ -1,6 +1,6 @@
 import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { delimiterFlags, isWordCharacter, matchEmphasis } from '../emphasis-matching.ts'
-import { escapesLineClaim, inlineHtmlConstruct, isAsciiPunctuation, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts'
+import { backslashEscape, escapesLineClaim, inlineHtmlConstruct, opensBracketedAutolink, opensEmailAutolink, type LinePosition } from '../commonmark-grammar.ts'
 import { opensInlineDirective } from '../directive-attributes.ts'
 import { readEntityReference } from '../entity-references.ts'
 
@@ -16,6 +16,8 @@ export type InlineSegment =
 
 export type AssembledLine = { line: string; unspellableRun: NodeRange | undefined }
 
+type ScanLine = { position: LinePosition; start: number; text: string }
+
 export type LineContainer = 'heading' | 'paragraph' | 'table-cell'
 
 type EmittedDelimiter = { closes: boolean; offset: number; pair: number; width: number }
@@ -67,10 +69,17 @@ function escape(segments: readonly InlineSegment[], container: LineContainer): A
   const escaped = new Set<number>()
   const placements: number[] = []
   let output = ''
+  let line = scanLine(scan, 0)
   for (let index = 0; index < scan.length; index += 1) {
+    if (index > line.start + line.text.length) line = scanLine(scan, line.start + line.text.length + 1)
     const escaping = escapings[index]
     const escapable = escaping === 'backslash' || escaping === 'bracketed'
-    if (escapable && (mergesWithSyntax(scan, escapings, index) || opensConstruct(scan, escapings, index, escaping === 'bracketed', container, escaped))) {
+    if (
+      escapable &&
+      (claimsLineStart(line, index, container) ||
+        mergesWithSyntax(scan, escapings, index) ||
+        opensConstruct(scan, escapings, index, escaping === 'bracketed', container, escaped))
+    ) {
       output += '\\'
       escaped.add(index)
     }
@@ -183,16 +192,16 @@ function opensConstruct(
   escaped: ReadonlySet<number>,
 ): boolean {
   if (container === 'heading' && closesHeading(scan, index)) return true
-  if (container === 'paragraph' && claimsLineStart(scan, index)) return true
   return claimsCharacter(scan, escapings, index, inBrackets, container, escaped)
 }
 
-function claimsLineStart(scan: string, index: number): boolean {
+function claimsLineStart(line: ScanLine, index: number, container: LineContainer): boolean {
-  const start = scan.lastIndexOf('\n', index - 1) + 1
+  return container === 'paragraph' && escapesLineClaim(line.text, index - line.start, line.position)
-  const end = scan.indexOf('\n', index)
+}
-  const line = scan.slice(start, end === -1 ? undefined : end)
+
-  const position: LinePosition = start === 0 ? 'first' : 'later'
+function scanLine(scan: string, start: number): ScanLine {
-  return escapesLineClaim(line, index - start, position)
+  const end = scan.indexOf('\n', start)
+  return { position: start === 0 ? 'first' : 'later', start, text: scan.slice(start, end === -1 ? undefined : end) }
 }
 
 function closesHeading(scan: string, index: number): boolean {
@@ -209,13 +218,12 @@ function claimsCharacter(
   escaped: ReadonlySet<number>,
 ): boolean {
   const character = scan.charAt(index)
-  const rest = scan.slice(index)
   if (inBrackets && (character === '[' || character === ']')) return true
   if (character === '|') return container === 'table-cell'
-  if (character === '\\') return isAsciiPunctuation(scan.charAt(index + 1))
+  if (character === '\\') return backslashEscape(scan, index) !== undefined
-  if (character === '&') return readEntityReference(rest) !== undefined
+  if (character === '&') return readEntityReference(scan, index) !== undefined
-  if (character === '<') return opensBracketedAutolink(rest) || opensEmailAutolink(rest) || inlineHtmlConstruct(rest) !== undefined
+  if (character === '<') return opensBracketedAutolink(scan, index) || opensEmailAutolink(scan, index) || inlineHtmlConstruct(scan, index) !== undefined
-  if (character === ':') return opensInlineDirective(rest)
+  if (character === ':') return opensInlineDirective(scan, index)
   if (character === '[') return opensLink(scan, escapings, index)
   if (character === '`') return opensCodeSpan(scan, index, escaped)
   if (character === '*' || character === '_' || character === '~') return claimsEmphasis(scan, index, escaped)

src/markdown/entity-references.test.ts

@@ -0,0 +1,17 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { readFileSync } from 'node:fs'
+
+const source = readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'entity-references.ts'), 'utf8')
+
+// The packing the module rests on, read off the rows rather than trusted: a value holding a space or a
+// tilde mis-splits into a wrong name and a lost one, at load, with nothing failing.
+test('every packed reference parts into one name and one value', () => {
+  const packed = /const packedReferences =\n {2}'([\s\S]*?)'\n/.exec(source)?.[1]
+  assert.ok(packed !== undefined)
+  const references = packed.split(/(?:\\\n)? +/)
+  assert.equal(references.length, 2125)
+  for (const reference of references) assert.match(reference, /^[A-Za-z][A-Za-z0-9]*~[^ ~]+$/)
+})

src/markdown/entity-references.ts

@@ -1,7 +1,7 @@
-export type EntityReference = { length: number; text: string }
+type EntityReference = { length: number; text: string }
 
 const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});'
-const anchoredEntityReference = new RegExp(`^(?:${entityReferenceSource})`)
+const anchoredEntityReference = new RegExp(`(?:${entityReferenceSource})`, 'y')
 const decimalReference = /^&#(\d+);/
 const hexadecimalReference = /^&#[Xx]([A-Fa-f0-9]+);/
 const largestCodePoint = 0x10ffff
@@ -204,13 +204,14 @@ const namedReferences = new Map(packedReferences.split(/ +/).map(namedReference)
 
 export function holdsEntityReference(text: string): boolean {
   for (let index = text.indexOf('&'); index !== -1; index = text.indexOf('&', index + 1)) {
-    if (readEntityReference(text.slice(index)) !== undefined) return true
+    if (readEntityReference(text, index) !== undefined) return true
   }
   return false
 }
 
-// The reference text opens with, `undefined` where it opens with none.
+// The reference standing at `index`, `undefined` where none does.
-export function readEntityReference(text: string): EntityReference | undefined {
+export function readEntityReference(text: string, index: number): EntityReference | undefined {
+  anchoredEntityReference.lastIndex = index
   const reference = anchoredEntityReference.exec(text)?.[0]
   if (reference === undefined) return undefined
   const decoded = decodeReference(reference)

src/markdown/parse/blocks.ts

@@ -178,7 +178,7 @@ function continuesLazily(walk: Walk, line: Line): boolean {
   if (leadingColumns(line) >= indentedCodeColumns) return true
   const opener = removeColumns(line, largestOpenerIndentation).text
   if (claimedConstruct(opener) !== undefined || isThematicBreak(opener)) return false
-  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, false) === undefined
+  return atxHeading(opener) === undefined && openingCodeFence(opener) === undefined && openingHtmlBlock(opener, true) === undefined
 }
 
 function readBlockLine(walk: Walk, line: Line): void {

src/markdown/parse/inline-content.ts

@@ -2,16 +2,14 @@ import type { AdfNode } from '../../adf/document.ts'
 import { backslashEscape, decodeTextEscapes, inlineHtmlConstruct } from '../commonmark-grammar.ts'
 import { backtickRun, closingBacktickRun } from '../backtick-runs.ts'
 import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts'
-import { readEntityReference } from '../entity-references.ts'
 
-// `text` holds the decoded content of the text node being built; `start` where its undecoded tail begins.
+type Run = { nodes: AdfNode[]; text: string; undecodedFrom: number }
-type Run = { nodes: AdfNode[]; start: number; text: string }
 
 const hardBreakSpaces = / {2,}$/
 const trailingSpace = /[ \t]+$/
 
 export function parseInlineContent(source: string, path: ConvertErrorPath): Result<AdfNode[]> {
-  const run: Run = { nodes: [], start: 0, text: '' }
+  const run: Run = { nodes: [], text: '', undecodedFrom: 0 }
   let index = 0
   while (index < source.length) {
     const character = source.charAt(index)
@@ -23,7 +21,7 @@ export function parseInlineContent(source: string, path: ConvertErrorPath): Resu
       continue
     }
     if (character === '\n') {
-      const hard = hardBreakSpaces.test(source.slice(run.start, index))
+      const hard = hardBreakSpaces.test(source.slice(run.undecodedFrom, index))
       takeRun(run, source, index, index + 1, true)
       if (hard) pushNode(run, { type: 'hardBreak' })
       else run.text += ' '
@@ -42,12 +40,8 @@ export function parseInlineContent(source: string, path: ConvertErrorPath): Resu
       continue
     }
     if (character === '<') {
-      const construct = inlineHtmlConstruct(source.slice(index))
+      const construct = inlineHtmlConstruct(source, index)
-      if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct.name}`, path)
+      if (construct !== undefined) return failure('unmappable-html', `no ADF node carries ${construct}`, path)
-    }
-    if (character === '&') {
-      index += readEntityReference(source.slice(index))?.length ?? 1
-      continue
     }
     index += backslashEscape(source, index) === undefined ? 1 : 2
   }
@@ -57,9 +51,9 @@ export function parseInlineContent(source: string, path: ConvertErrorPath): Resu
 }
 
 function takeRun(run: Run, source: string, end: number, resume: number, strip: boolean): void {
-  const raw = source.slice(run.start, end)
+  const raw = source.slice(run.undecodedFrom, end)
   run.text += decodeTextEscapes(strip ? raw.replace(trailingSpace, '') : raw)
-  run.start = resume
+  run.undecodedFrom = resume
 }
 
 function pushText(run: Run): void {

src/markdown/parse/markdown-to-adf.test.ts

@@ -227,6 +227,8 @@ test('folds a lazy continuation into the paragraph the container holds', () => {
   assert.deepEqual(content(markdownToAdf('> One\n---\n')), [quote(paragraph('One')), { type: 'rule' }])
   assert.deepEqual(content(markdownToAdf('> One\n```\n')), [quote(paragraph('One')), { type: 'codeBlock' }])
   assert.equal(code(markdownToAdf('> One\n<div>\n')), 'unmappable-html')
+  assert.deepEqual(path(markdownToAdf('> One\n<div>\n')), ['content', 1])
+  assert.deepEqual(path(markdownToAdf('> One\n<span>\n')), ['content', 0, 'content', 0])
 })
 
 test('ends a lazy continuation at a claimed line', () => {
@@ -307,6 +309,7 @@ test('refuses the raw inline HTML no element mapping carries, naming it', () =>
   assert.equal(content(markdownToAdf('Part <!DOCTYPE html> here.\n')), 'unmappable-html: no ADF node carries an HTML declaration')
   assert.equal(content(markdownToAdf('Part <![CDATA[x]]> here.\n')), 'unmappable-html: no ADF node carries a CDATA section')
   assert.equal(content(markdownToAdf('Part <!--> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
+  assert.equal(content(markdownToAdf('Part <!---> here.\n')), 'unmappable-html: no ADF node carries an HTML comment')
   assert.equal(code(markdownToAdf('A <a href="/x" disabled\nid=y> b\n')), 'unmappable-html')
   assert.equal(code(markdownToAdf('Part.\n<span>\n')), 'unmappable-html')
   assert.deepEqual(path(markdownToAdf('Part.\n\nA <b>b</b>.\n')), ['content', 1])
@@ -315,6 +318,7 @@ test('refuses the raw inline HTML no element mapping carries, naming it', () =>
 test('leaves the angle bracket that opens no HTML construct to the text it sits in', () => {
   assert.deepEqual(content(markdownToAdf('3 < 4 and 5 <b 6\n')), [paragraph('3 < 4 and 5 <b 6')])
   assert.deepEqual(content(markdownToAdf('a <b"c> d\n')), [paragraph('a <b"c> d')])
+  assert.deepEqual(content(markdownToAdf('a <!-- b\n')), [paragraph('a <!-- b')])
   assert.deepEqual(content(markdownToAdf('a </b c> d\n')), [paragraph('a </b c> d')])
   assert.deepEqual(content(markdownToAdf('`<span>`\n')), [{ content: [codeSpan('<span>')], type: 'paragraph' }])
   assert.deepEqual(content(markdownToAdf('\\<span>\n')), [paragraph('<span>')])