Answer the stability review: a code or HTML block swallows the marker line too

src/markdown/commonmark-grammar.ts

@@ -16,10 +16,11 @@ const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/
 const codeFenceOpener = /^(`{3,}|~{3,})/
 const directiveClaim = /^:{2,}(?:[A-Za-z0-9]|[ \t]*$)/
 const pipeClaim = /^\|/
+const bulletListOpener = /^[*+-](?:[ \t]|$)/
 // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip.
-const firstCharacterOpeners = [atxHeadingOpener, /^>/, /^[*+-](?:[ \t]|$)/, codeFenceOpener, /^:{2,}/, pipeClaim]
+const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim]
 const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/]
-const orderedListOpener = /^(\d{1,9})[.)](?:[ \t]|$)/
+const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/
 const setextUnderline = /^(=+|-+)[ \t]*$/
 const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
 const unicodeWhitespace = /[\t\n\f\r \p{Zs}]/u
@@ -86,8 +87,16 @@ export function isUnicodeWhitespace(character: string): boolean {
   return unicodeWhitespace.test(character)
 }
 
-// The first marker of a list, `undefined` for a bullet: one answer both directions read, or the emitter
-// spells a list the parser folds into the paragraph above it.
+// `start` is the list's first number, `undefined` for a bullet.
+export function listMarker(line: string): { delimiter: string; start: number | undefined; width: number } | undefined {
+  const ordered = orderedListOpener.exec(line)
+  if (ordered !== null) {
+    const digits = ordered[1] ?? ''
+    return { delimiter: ordered[2] ?? '', start: Number(digits), width: digits.length + 1 }
+  }
+  return bulletListOpener.test(line) ? { delimiter: line.charAt(0), start: undefined, width: 1 } : undefined
+}
+
 export function markerInterruptsParagraph(start: number | undefined, empty: boolean): boolean {
   return !empty && (start === undefined || start === 1)
 }

src/markdown/parse/blocks.ts

@@ -5,6 +5,7 @@ import {
   claimsPipeLine,
   closingCodeFence,
   isThematicBreak,
+  listMarker,
   markerInterruptsParagraph,
   openingCodeFence,
   setextHeadingLevel,
@@ -39,13 +40,9 @@ type OpenLeaf =
 
 type ContainerStart = { kind: 'blockquote'; rest: string } | { fresh: boolean; indentation: number; kind: 'item'; list: ListBlock; marker: string; rest: string }
 
-type ItemMarker = { list: ListBlock; marker: string; width: number }
-
 type Walk = ParsedBlocks & { leaf: OpenLeaf | undefined; stack: OpenContainer[] }
 
 const blankLine = /^[ \t]*$/
-const bulletMarker = /^[-*+](?=[ \t]|$)/
-const orderedMarker = /^(\d{1,9})([.)])(?=[ \t]|$)/
 const indentedCodeColumns = 4
 const largestOpenerIndentation = 3
 const tabStop = 4
@@ -59,6 +56,11 @@ export function parseBlocks(markdown: string): ParsedBlocks {
 
 function readLine(walk: Walk, line: string): void {
   const matched = matchContainers(walk, line)
+  // A leaf that swallows whole lines takes the marker too: no container opens inside a code or HTML block.
+  if (matched.depth === walk.stack.length && swallowsLines(walk.leaf)) {
+    readBlockLine(walk, matched.rest)
+    return
+  }
   const paragraphOpen = matched.depth === walk.stack.length && walk.leaf?.kind === 'paragraph'
   const opened = openContainers(walk, matched.rest, paragraphOpen, matched.depth)
   if (!opened.opened && matched.depth < walk.stack.length) {
@@ -71,6 +73,10 @@ function readLine(walk: Walk, line: string): void {
   readBlockLine(walk, opened.rest)
 }
 
+function swallowsLines(leaf: OpenLeaf | undefined): boolean {
+  return leaf?.kind === 'fenced-code' || leaf?.kind === 'html'
+}
+
 function matchContainers(walk: Walk, line: string): { depth: number; rest: string } {
   let depth = 0
   let rest = line
@@ -111,38 +117,32 @@ function openContainers(walk: Walk, line: string, paragraphOpen: boolean, depth:
 function containerStart(line: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined {
   const opener = removeColumns(line, largestOpenerIndentation)
   if (opener.startsWith('>')) return { kind: 'blockquote', rest: removeColumns(opener.slice(1), 1) }
-  if (isThematicBreak(opener) || (paragraphOpen && setextHeadingLevel(opener) !== undefined)) return undefined
+  if (isThematicBreak(opener)) return undefined
   return itemStart(line, opener, paragraphOpen, enclosing)
 }
 
 function itemStart(line: string, opener: string, paragraphOpen: boolean, enclosing: OpenContainer | undefined): ContainerStart | undefined {
-  const marker = itemMarker(opener)
+  const marker = listMarker(opener)
   if (marker === undefined) return undefined
   const after = opener.slice(marker.width)
   const blank = blankLine.test(after)
-  if (paragraphOpen && !markerInterruptsParagraph(marker.list.kind === 'orderedList' ? marker.list.start : undefined, blank)) return undefined
+  if (paragraphOpen && !markerInterruptsParagraph(marker.start, blank)) return undefined
   const spaces = leadingColumns(after)
   const padding = blank || spaces > indentedCodeColumns ? 1 : spaces
-  const continued = enclosing?.kind === 'item' && enclosing.list.kind === marker.list.kind && enclosing.marker === marker.marker
+  const kind = marker.start === undefined ? 'bulletList' : 'orderedList'
+  const continued = enclosing?.kind === 'item' && enclosing.list.kind === kind && enclosing.marker === marker.delimiter
   return {
     fresh: !continued,
     indentation: leadingColumns(line) + marker.width + padding,
     kind: 'item',
-    list: continued ? enclosing.list : marker.list,
-    marker: marker.marker,
+    list: continued ? enclosing.list : openList(marker.start),
+    marker: marker.delimiter,
     rest: blank ? '' : removeColumns(after, padding),
   }
 }
 
-function itemMarker(opener: string): ItemMarker | undefined {
-  const ordered = orderedMarker.exec(opener)
-  if (ordered !== null) {
-    const digits = ordered[1] ?? ''
-    const delimiter = ordered[2] ?? ''
-    return { list: { items: [], kind: 'orderedList', start: Number(digits) }, marker: delimiter, width: digits.length + 1 }
-  }
-  const bullet = bulletMarker.exec(opener)?.[0]
-  return bullet === undefined ? undefined : { list: { items: [], kind: 'bulletList' }, marker: bullet, width: 1 }
+function openList(start: number | undefined): ListBlock {
+  return start === undefined ? { items: [], kind: 'bulletList' } : { items: [], kind: 'orderedList', start }
 }
 
 function openContainer(walk: Walk, start: ContainerStart): void {

src/markdown/parse/markdown-to-adf.test.ts

@@ -85,6 +85,7 @@ test('reads a fenced code block, its info string the language', () => {
   assert.deepEqual(content(markdownToAdf('~~~ a`b\n```\n~~~\n')), [{ attrs: { language: 'a`b' }, content: [text('```')], type: 'codeBlock' }])
   assert.deepEqual(content(markdownToAdf('``` a`b\n')), [paragraph('``` a`b')])
   assert.deepEqual(content(markdownToAdf('```\n``` x\n```\n')), [{ content: [text('``` x')], type: 'codeBlock' }])
+  assert.deepEqual(content(markdownToAdf('```\n- x\n> y\n```\n')), [{ content: [text('- x\n> y')], type: 'codeBlock' }])
 })
 
 test('strips the opening fence indentation from the content lines it holds', () => {
@@ -124,6 +125,7 @@ test('refuses the raw HTML no element mapping carries', () => {
   assert.equal(code(markdownToAdf('<span foo="bar">\n')), 'unmappable-html')
   assert.deepEqual(path(markdownToAdf('Part.\n\n<div>\n')), ['content', 1])
   assert.equal(code(markdownToAdf('<div>\nx\n\n:::\n')), 'unmappable-html')
+  assert.equal(code(markdownToAdf('<div>\n- x\n</div>\n')), 'unmappable-html')
 })
 
 test('swallows an HTML block ahead of the claim a line inside it would make', () => {

src/markdown/parse/markdown-to-adf.ts

@@ -22,7 +22,6 @@ function blockNodes(blocks: readonly Block[], path: ConvertErrorPath, depth: num
   return success(content)
 }
 
-// Switched, not chained: `noImplicitReturns` then refuses the kind a later milestone adds and forgets.
 function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result<AdfNode> {
   switch (block.kind) {
     case 'blockquote':