Read deep list items' blank lines and a directive's unclosed backtick runs in time linear in the markdown

src/markdown/commonmark/backtick-runs.ts

@@ -1,12 +1,14 @@
 import { runLength } from './emphasis-matching.ts'
 
+export type CodeSpanCloser = (from: number, opener: number) => number | undefined
+
 export function backtickRun(text: string, index: number): number {
   return text.charAt(index) === '`' ? runLength(text, index) : 0
 }
 
 // Where the first run of exactly `opener` backticks at or after `from` begins, `from` never inside a run. Ask with
 // `from` rising, so each length's cursor only moves forward and a walk stays linear; an earlier `from` rescans that length.
-export function backtickCloser(text: string): (from: number, opener: number) => number | undefined {
+export function backtickCloser(text: string): CodeSpanCloser {
   const counts = new Map<number, number>()
   eachRun(text, (_, length) => counts.set(length, (counts.get(length) ?? 0) + 1))
   const firsts = new Map<number, number>()
@@ -35,17 +37,6 @@ export function backtickCloser(text: string): (from: number, opener: number) =>
   }
 }
 
-// Where the run of exactly `opener` backticks closing a code span begins, `undefined` where none does.
-export function closingBacktickRun(text: string, from: number, opener: number): number | undefined {
-  let cursor = from
-  while (cursor < text.length) {
-    const run = backtickRun(text, cursor)
-    if (run === opener) return cursor
-    cursor += run === 0 ? 1 : run
-  }
-  return undefined
-}
-
 export function fencedCodeBlock(info: string, body: string): string {
   const fence = '`'.repeat(Math.max(3, longestBacktickRun(body) + 1))
   return body === '' ? `${fence}${info}\n${fence}` : `${fence}${info}\n${body}\n${fence}`

src/markdown/directive-syntax.test.ts

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 import test from 'node:test'
 
 import type { DirectiveAttributes, DirectiveLine } from './directive-syntax.ts'
+import { backtickCloser } from './commonmark/backtick-runs.ts'
 import { claimDirectivePrefix, readDirectiveLine, readInlineDirective } from './directive-syntax.ts'
 import { largestNesting } from '../nesting.ts'
 
@@ -24,7 +25,7 @@ function fault(line: string): string {
 function inline(text: string): unknown {
   const prefix = claimDirectivePrefix(text, 0)
   if (prefix === undefined) return 'unclaimed'
-  const read = readInlineDirective(text, prefix)
+  const read = readInlineDirective(text, prefix, backtickCloser(text))
   if (read.fault !== undefined) return read.fault.message
   return { attributes: read.value.attributes, content: read.value.content, length: read.value.length, name: read.value.name }
 }
@@ -156,3 +157,8 @@ test('refuses inline directives nested deeper than the parser carries', () => {
   spans(nest(largestNesting), 'underline', nest(largestNesting - 1))
   assert.equal(inline(nest(largestNesting + 1)), `the input nests inline directives deeper than the ${largestNesting} levels the parser carries`)
 })
+
+test('reads a directive whose content holds many unclosed backtick runs in time linear in its length', () => {
+  const runs = Array.from({ length: 3000 }, (_, index) => '`'.repeat(index + 1)).join(' ')
+  spans(`!adf:x[${runs}]`, 'x', runs)
+})

src/markdown/directive-syntax.ts

@@ -1,8 +1,9 @@
 import type { AttributeKind, VocabularyPair, VocabularyValue } from '../adf/attribute-vocabulary.ts'
+import type { CodeSpanCloser } from './commonmark/backtick-runs.ts'
 import type { ConvertFault } from '../result.ts'
 import type { JsonValue } from '../json-value.ts'
 import { backslashEscape, holdsNullCharacter } from './commonmark/grammar.ts'
-import { backtickRun, closingBacktickRun } from './commonmark/backtick-runs.ts'
+import { backtickRun } from './commonmark/backtick-runs.ts'
 import { isNumberList } from '../adf/attribute-vocabulary.ts'
 import { overNested } from '../json-value.ts'
 import { largestNesting } from '../nesting.ts'
@@ -100,8 +101,8 @@ export function readDirectiveLine(line: string): Read<DirectiveLine> | undefined
   return line.charAt(directivePrefix.length) === '/' ? readCloserLine(line) : readOpenerLine(line)
 }
 
-export function readInlineDirective(text: string, prefix: ClaimedPrefix): Read<DirectiveSpan> {
-  return readClaimedDirective(text, prefix.index, 1)
+export function readInlineDirective(text: string, prefix: ClaimedPrefix, codeSpanCloser: CodeSpanCloser): Read<DirectiveSpan> {
+  return readClaimedDirective(text, prefix.index, 1, codeSpanCloser)
 }
 
 export function readSoleStringAttribute(span: DirectiveSpan, key: string): Read<string> {
@@ -227,26 +228,26 @@ function readOpenerAttributes(rest: string, index: number): Read<Attributes> {
   return rest.charAt(index) === ' ' ? readAttributesAt(rest, index + 1, false) : { value: { attributes: noAttributes, end: index } }
 }
 
-function readNestedDirective(text: string, index: number, depth: number): Read<DirectiveSpan> | undefined {
-  return claimsDirectivePrefix(text, index) ? readClaimedDirective(text, index, depth) : undefined
+function readNestedDirective(text: string, index: number, depth: number, codeSpanCloser: CodeSpanCloser): Read<DirectiveSpan> | undefined {
+  return claimsDirectivePrefix(text, index) ? readClaimedDirective(text, index, depth, codeSpanCloser) : undefined
 }
 
-function readClaimedDirective(text: string, index: number, depth: number): Read<DirectiveSpan> {
+function readClaimedDirective(text: string, index: number, depth: number, codeSpanCloser: CodeSpanCloser): Read<DirectiveSpan> {
   const name = inlineDirectiveName(text, index)
   if (name === undefined) return { fault: malformedDirective(prefixFault) }
   if (depth > largestNesting) {
     return { fault: { code: 'unsupported-nesting-depth', message: `the input nests inline directives deeper than the ${largestNesting} levels the parser carries` } }
   }
-  const slot = readContentSlot(text, index + directivePrefix.length + name.length, depth)
+  const slot = readContentSlot(text, index + directivePrefix.length + name.length, depth, codeSpanCloser)
   if (slot.fault !== undefined) return { fault: slot.fault }
   const attributes = readAttributesAt(text, slot.value.end, slot.value.content === undefined)
   if (attributes.fault !== undefined) return { fault: attributes.fault }
   return { value: { attributes: attributes.value.attributes, content: slot.value.content, length: attributes.value.end - index, name, spans: slot.value.spans } }
 }
 
-function readContentSlot(text: string, index: number, depth: number): Read<Content> {
+function readContentSlot(text: string, index: number, depth: number, codeSpanCloser: CodeSpanCloser): Read<Content> {
   if (text.charAt(index) !== '[') return { value: { content: undefined, end: index, spans: noSpans } }
-  const close = readDirectiveContent(text, index + 1, depth)
+  const close = readDirectiveContent(text, index + 1, depth, codeSpanCloser)
   if (close.fault !== undefined) return { fault: close.fault }
   return { value: { content: text.slice(index + 1, close.value.end), end: close.value.end + 1, spans: close.value.spans } }
 }
@@ -259,7 +260,7 @@ function readAttributesAt(text: string, index: number, braceClaims: boolean): Re
 }
 
 // A code span, an escape and a nested directive each bind before the content's own closing bracket.
-function readDirectiveContent(text: string, start: number, depth: number): Read<DirectiveContent> {
+function readDirectiveContent(text: string, start: number, depth: number, codeSpanCloser: CodeSpanCloser): Read<DirectiveContent> {
   const spans = new Map<number, DirectiveSpan>()
   let brackets = 0
   let cursor = start
@@ -270,12 +271,12 @@ function readDirectiveContent(text: string, start: number, depth: number): Read<
       continue
     }
     if (character === '`') {
-      const span = readCodeSpanEnd(text, cursor)
+      const span = readCodeSpanEnd(text, cursor, codeSpanCloser)
       if (span === undefined) break
       cursor = span
       continue
     }
-    const nested = readNestedDirective(text, cursor, depth + 1)
+    const nested = readNestedDirective(text, cursor, depth + 1, codeSpanCloser)
     if (nested?.fault !== undefined) return { fault: nested.fault }
     if (nested !== undefined) {
       spans.set(cursor - start, nested.value)
@@ -291,9 +292,9 @@ function readDirectiveContent(text: string, start: number, depth: number): Read<
 }
 
 // `undefined` where the span crosses the line ending an inline directive may not cross.
-function readCodeSpanEnd(text: string, index: number): number | undefined {
+function readCodeSpanEnd(text: string, index: number, codeSpanCloser: CodeSpanCloser): number | undefined {
   const opener = backtickRun(text, index)
-  const closer = closingBacktickRun(text, index + opener, opener)
+  const closer = codeSpanCloser(index + opener, opener)
   if (closer === undefined) return index + opener
   return text.slice(index, closer + opener).includes('\n') ? undefined : closer + opener
 }

src/markdown/inline-tokens.ts

@@ -1,7 +1,7 @@
 import type { ClaimedPrefix } from './directive-syntax.ts'
 import type { Claims } from './portable/conventions.ts'
 import { backslashEscape, inlineHtmlConstruct, readBracketedAutolink, readEmailAutolink } from './commonmark/grammar.ts'
-import { backtickCloser, backtickRun } from './commonmark/backtick-runs.ts'
+import { backtickCloser, backtickRun, type CodeSpanCloser } from './commonmark/backtick-runs.ts'
 import { claimDirectivePrefix } from './directive-syntax.ts'
 import { delimiterFlags, runLength } from './commonmark/emphasis-matching.ts'
 import { highlightDelimiter, highlightFlanking } from './portable/conventions.ts'
@@ -20,7 +20,7 @@ export type InlineToken =
 
 // Build one per text and pass it to every `readInlineToken` call on that text, so a walk stays linear in the text's length.
 export type InlineSource = {
-  readonly codeSpanCloser: (from: number, opener: number) => number | undefined
+  readonly codeSpanCloser: CodeSpanCloser
   // Where each HTML terminator last starts in the text, -1 for none; filled the first time an opener asks.
   readonly terminators: Map<string, number>
   readonly text: string

src/markdown/parse/blocks.test.ts

@@ -122,3 +122,31 @@ test('closes the containers a closer names past as unclosed, and crosses no list
 test("releases an indented code block's held blank lines as a walk rather than as one call's arguments", () => {
   assert.deepEqual(kinds(`    a\n${'\n'.repeat(200000)}    b\n`), ['code'])
 })
+
+// A level is one list item. The gate times nothing; a quadratic walk outlasts Bun's 5 s test timeout.
+function deepestItem(markdown: string): { depth: number; kinds: string[] } {
+  let blocks = parseBlocks(markdown, flavourClaims.lossless).blocks
+  let depth = 0
+  for (let list = blocks[0]; list?.kind === 'bulletList'; list = blocks[0]) {
+    blocks = list.items.at(-1) ?? []
+    depth += 1
+  }
+  return { depth, kinds: blocks.map((block) => block.kind) }
+}
+
+test('continues a blank line through a deep run of list items in time linear in the lines', () => {
+  const depth = 20000
+  assert.deepEqual(deepestItem(`${'- '.repeat(depth)}a\n${'\n'.repeat(depth)}`), { depth, kinds: ['paragraph'] })
+  assert.deepEqual(deepestItem(`${'- '.repeat(depth)}a\n${'\n'.repeat(depth)}${'  '.repeat(depth)}b\n`), { depth, kinds: ['paragraph', 'paragraph'] })
+})
+
+test('stops a blank line at the first blockquote and at an empty item', () => {
+  assert.deepEqual(deepestItem('- > a\n\n  b\n'), { depth: 1, kinds: ['blockquote', 'paragraph'] })
+  assert.deepEqual(deepestItem('- - \n\n  a\n'), { depth: 2, kinds: [] })
+  assert.deepEqual(deepestItem('- > - a\n\n  > b\n'), { depth: 1, kinds: ['blockquote', 'blockquote'] })
+})
+
+test('continues a line through a deep run of list items in time linear in the line', () => {
+  const depth = 20000
+  assert.deepEqual(deepestItem(`${'- '.repeat(depth)}a\n${`${'  '.repeat(depth)}b\n`.repeat(3)}`), { depth, kinds: ['paragraph'] })
+})

src/markdown/parse/blocks.ts

@@ -73,6 +73,8 @@ type ContainerStack = {
   edges: readonly { container: EdgeContainer; depth: number }[]
   open: readonly OpenContainer[]
   push: (container: OpenContainer) => void
+  // The index in `edges` of each blockquote.
+  quotes: readonly number[]
 }
 
 type Walk = ParsedBlocks & {
@@ -112,13 +114,17 @@ function containerStack(): ContainerStack {
   const depthsOf = (name: string): number[] => entryOf(directiveDepths, name, () => [])
   const edges: { container: EdgeContainer; depth: number }[] = []
   const open: OpenContainer[] = []
+  const quotes: number[] = []
   return {
     directiveDepth: (name) => directiveDepths.get(name)?.at(-1),
     drop: (depth) => {
       const dropped = open.splice(depth)
       for (const container of dropped) {
         if (container.kind === 'directive') depthsOf(container.name).pop()
-        else edges.pop()
+        else {
+          edges.pop()
+          if (container.kind === 'blockquote') quotes.pop()
+        }
       }
       return dropped
     },
@@ -127,8 +133,12 @@ function containerStack(): ContainerStack {
     push: (container) => {
       const depth = open.push(container) - 1
       if (container.kind === 'directive') depthsOf(container.name).push(depth)
-      else edges.push({ container, depth })
+      else {
+        if (container.kind === 'blockquote') quotes.push(edges.length)
+        edges.push({ container, depth })
+      }
     },
+    quotes,
   }
 }
 
@@ -157,22 +167,34 @@ function swallowsLines(leaf: OpenLeaf | undefined): boolean {
 
 // A directive container has no continuation marker, so every line continues it.
 function matchContainers(walk: Walk, line: Line): { depth: number; rest: Line } {
+  if (isBlankLine(line.text)) return matchBlankLine(walk, line, 0, 0)
+  let quotes = 0
   let rest = line
-  for (const { container, depth } of walk.stack.edges) {
-    const next = continuesContainer(walk, container, rest)
+  for (const [index, { container, depth }] of walk.stack.edges.entries()) {
+    const next = continuesContainer(container, rest)
     if (next === undefined) return { depth, rest }
     rest = next
+    if (container.kind !== 'blockquote') continue
+    quotes += 1
+    if (isBlankLine(rest.text)) return matchBlankLine(walk, rest, index + 1, quotes)
   }
   return { depth: walk.stack.open.length, rest }
 }
 
-function continuesContainer(walk: Walk, container: EdgeContainer, line: Line): Line | undefined {
+// A blank line continues every list item up to the next blockquote, so it is matched in one step rather than one per item.
+// A list item begins with at most one blank line: an empty one gives the second up. Only the innermost item is empty:
+// opening a container inside an item gives it a block.
+function matchBlankLine(walk: Walk, line: Line, from: number, quote: number): { depth: number; rest: Line } {
+  const { edges, quotes } = walk.stack
+  const last = edges.length - 1
+  const emptyItem = last >= from && edges[last]?.container.blocks.length === 0 && walk.leaf === undefined
+  const stop = Math.min(quotes[quote] ?? edges.length, emptyItem ? last : edges.length)
+  return { depth: edges[stop]?.depth ?? walk.stack.open.length, rest: stop === from ? line : { column: line.column, text: '' } }
+}
+
+function continuesContainer(container: EdgeContainer, line: Line): Line | undefined {
   if (container.kind === 'blockquote') return blockquoteRest(removeColumns(line, largestOpenerIndentation))
-  // A list item begins with at most one blank line: an empty one gives the second up.
-  if (isBlankLine(line.text)) {
-    return container.blocks.length === 0 && walk.leaf === undefined ? undefined : { column: line.column, text: '' }
-  }
-  return leadingColumns(line) < container.indentation ? undefined : removeColumns(line, container.indentation)
+  return leadingColumns(line, container.indentation) < container.indentation ? undefined : removeColumns(line, container.indentation)
 }
 
 function blockquoteRest(opener: Line): Line | undefined {
@@ -583,9 +605,11 @@ function sourceLine(markdown: string, line: number, start: number, end: number):
   return { position: { line, offset: start }, text: replaceNullCharacters(markdown.slice(start, end)) }
 }
 
-function leadingColumns(line: Line): number {
+// Counting stops at `upTo` columns, so a test against a width costs the width, not the line's whole indentation.
+function leadingColumns(line: Line, upTo = Infinity): number {
   let columns = 0
   for (const character of line.text) {
+    if (columns >= upTo) break
     if (character === ' ') columns += 1
     else if (character === '\t') columns += tabStop - ((line.column + columns) % tabStop)
     else break

src/markdown/parse/inline-content.ts

@@ -2,7 +2,7 @@ import type { AdfMark, AdfNode } from '../../adf/document.ts'
 import type { ClaimedPrefix, DirectiveSpan, NestedSpans } from '../directive-syntax.ts'
 import type { Claims } from '../portable/conventions.ts'
 import type { EmphasisPairing } from '../commonmark/emphasis-matching.ts'
-import type { InlineToken } from '../inline-tokens.ts'
+import type { InlineSource, InlineToken } from '../inline-tokens.ts'
 import type { LineContainer } from '../line-container.ts'
 import type { LinkDefinition } from '../commonmark/link-syntax.ts'
 import { centeredImage, externalMedia } from '../external-image.ts'
@@ -112,7 +112,7 @@ function scanInline(scan: Scan): Result<Scanned> {
   const inline = inlineSource(scan.source)
   let index = 0
   while (index < scan.source.length) {
-    const read = readToken(scan, index, readInlineToken(inline, index, scan.claims))
+    const read = readToken(scan, inline, index, readInlineToken(inline, index, scan.claims))
     if (!read.ok) return read
     index = read.value
   }
@@ -120,8 +120,8 @@ function scanInline(scan: Scan): Result<Scanned> {
   return assemble(scan)
 }
 
-function readToken(scan: Scan, index: number, token: InlineToken): Result<number> {
-  if (token.kind === 'directive') return readDirective(scan, token.prefix)
+function readToken(scan: Scan, inline: InlineSource, index: number, token: InlineToken): Result<number> {
+  if (token.kind === 'directive') return readDirective(scan, inline, token.prefix)
   if (token.kind === 'html') return failure('unmappable-html', `no raw HTML converts at this version: ${token.construct}`, scan.path)
   const end = index + token.width
   switch (token.kind) {
@@ -170,10 +170,10 @@ function readLineEnding(scan: Scan, index: number): number {
   return index + 1
 }
 
-function readDirective(scan: Scan, prefix: ClaimedPrefix): Result<number> {
+function readDirective(scan: Scan, inline: InlineSource, prefix: ClaimedPrefix): Result<number> {
   const held = scan.spans.get(prefix.index)
   if (held !== undefined) return pushDirective(scan, held, prefix.index)
-  const directive = readInlineDirective(scan.source, prefix)
+  const directive = readInlineDirective(scan.source, prefix, inline.codeSpanCloser)
   if (directive.fault !== undefined) return faulted(directive.fault, scan.path)
   return pushDirective(scan, directive.value, prefix.index)
 }