Spell a portable link title holding a backslash, an entity reference or, in a table cell, a pipe

CHANGELOG.md

@@ -14,6 +14,9 @@
   holding another mark as `!adf:carry{json="…"}`, and `adfToPortableMarkdown` drops that mark.
 - `adfToPortableMarkdown` keeps both targets of a linked external image standing inline, as in a
   table cell: `[alt](href) ([image](url))`, where it wrote `[alt](url)` and dropped the link's target.
+- `adfToPortableMarkdown` keeps a link's title holding a backslash or an entity reference, and in a
+  table cell one holding a `|`, escaping each: `[docs](https://example.com "C:\\dir")`, where it
+  dropped the title. `portableMarkdownToAdf` reads it back to the same title.
 - `losslessMarkdownToAdf` names the CommonMark spelling to write when it refuses a directive whose
   node CommonMark spells, such as `[![alt](url)](href)` for a linked `!adf:mediaSingle`.
 - Every reader and writer takes time linear in the text when it holds many `<!--`, `<?`,

src/markdown/commonmark/link-syntax.ts

@@ -1,10 +1,13 @@
 import { backslashEscape, decodeTextEscapes, holdsControlCharacter, holdsNullCharacter } from './grammar.ts'
-import { holdsEntityReference } from './entity-references.ts'
+import { holdsEntityReference, readEntityReference } from './entity-references.ts'
 
 export type LinkDefinition = { destination: string; title?: string }
 
 export type LinkPart = { length: number; value: string }
 
+// spec/flavour.md, Marks: the lossless flavour escapes a quote alone and writes any other title CommonMark reads otherwise as a directive; the portable flavour has none.
+export type TitleEscapes = 'lossless' | 'portable' | 'portable-table-cell'
+
 const bracketedDestination = /<((?:[^\n<>\\]|\\[^\n])*)>/y
 const linkLabel = /\[((?:[^[\]\\]|\\[\s\S]){0,999})\]/y
 const titleClosers: Readonly<Record<string, string>> = { '"': '"', "'": "'", '(': ')' }
@@ -95,10 +98,10 @@ export function spellDestination(href: string): string | undefined {
   return href.startsWith('<') ? undefined : escapeUnbalanced(href, '(', ')')
 }
 
-export function spellLinkTarget(href: string, title: string | undefined): string | undefined {
+export function spellLinkTarget(href: string, title: string | undefined, escapes: TitleEscapes): string | undefined {
   const destination = spellDestination(href)
   if (destination === undefined || title === undefined) return destination
-  const spelledTitle = spellTitle(title)
+  const spelledTitle = spellTitle(title, escapes)
   return spelledTitle === undefined ? undefined : `${destination === '' ? '<>' : destination}${spelledTitle}`
 }
 
@@ -116,6 +119,14 @@ export function escapeUnbalanced(spelling: string, opener: string, closer: strin
   return spelled
 }
 
-function spellTitle(title: string): string | undefined {
-  return /[\n\r\\]/.test(title) || holdsNullCharacter(title) || holdsEntityReference(title) ? undefined : ` "${title.replaceAll('"', '\\"')}"`
+function spellTitle(title: string, escapes: TitleEscapes): string | undefined {
+  if (/[\n\r]/.test(title) || holdsNullCharacter(title)) return undefined
+  if (escapes === 'lossless') return title.includes('\\') || holdsEntityReference(title) ? undefined : ` "${title.replaceAll('"', '\\"')}"`
+  let spelled = ''
+  for (let index = 0; index < title.length; index += 1) {
+    const character = title.charAt(index)
+    const escaped = character === '\\' || character === '"' || (character === '&' && readEntityReference(title, index) !== undefined) || (character === '|' && escapes === 'portable-table-cell')
+    spelled += escaped ? `\\${character}` : character
+  }
+  return ` "${spelled}"`
 }

src/markdown/emit/adf-to-markdown.ts

@@ -145,7 +145,7 @@ function spellReadableBlock(node: AdfNode, path: ConvertErrorPath, depth: number
   if (node.type === 'bulletList' || node.type === 'orderedList') return tryList(node, path, depth, writing)
   if (node.type === 'codeBlock') return readableText(tryCodeBlock(node, writing.flavour))
   if (node.type === 'heading') return tryHeading(node, path, writing.flavour)
-  if (node.type === 'mediaSingle') return readableText(tryImage(node))
+  if (node.type === 'mediaSingle') return readableText(tryImage(node, writing.flavour))
   if (node.type === 'paragraph') return tryParagraph(node, path, writing.flavour)
   if (node.type === 'rule') return readableText(tryRule(node))
   if (node.type === 'table') return readableText(tryPipeTable(node, path, writing.flavour))

src/markdown/emit/image.ts

@@ -1,11 +1,12 @@
 import type { AdfNode } from '../../adf/document.ts'
+import type { WrittenFlavour } from '../flavour.ts'
 import { centeredImage, externalMedia } from '../external-image.ts'
 import { linkDefinition } from '../mark-spellings.ts'
 import { nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
 import { serializeCanonicalJson } from '../../canonical-json.ts'
 import { tryImageLine } from './inline-line.ts'
 
-export function tryImage(node: AdfNode): string | undefined {
+export function tryImage(node: AdfNode, flavour: WrittenFlavour): string | undefined {
   const [media] = nodeContent(node)
   if (media === undefined) return undefined
   const { alt, url } = nodeAttrs(media)
@@ -13,5 +14,5 @@ export function tryImage(node: AdfNode): string | undefined {
   const [mark] = nodeMarks(media)
   const link = mark === undefined ? undefined : linkDefinition(nodeAttrs(mark))
   const read = centeredImage([externalMedia(alt ?? '', url, link)])
-  return serializeCanonicalJson(node, 'compact') === serializeCanonicalJson(read, 'compact') ? tryImageLine(alt, url, link) : undefined
+  return serializeCanonicalJson(node, 'compact') === serializeCanonicalJson(read, 'compact') ? tryImageLine(alt, url, flavour, link) : undefined
 }

src/markdown/emit/inline-line.ts

@@ -1,9 +1,10 @@
 import type { AdfMark, AdfNode } from '../../adf/document.ts'
 import type { InlineNodeModel } from '../../adf/inline-nodes.ts'
 import type { LineContainer } from '../line-container.ts'
-import type { LinkDefinition } from '../commonmark/link-syntax.ts'
+import type { LinkDefinition, TitleEscapes } from '../commonmark/link-syntax.ts'
 import type { WrittenFlavour } from '../flavour.ts'
 import { assembleInlineLine, isSyntax, type InlineEscaping, type InlineSegment, type MarkRun, type NodeRange } from './line-escaping.ts'
+import { barePipeCells } from '../pipe-table-syntax.ts'
 import { carriedInline } from '../opaque-carry.ts'
 import { commonMarkLink, linkHref, markSpelling, spellMarkAttributes } from '../mark-spellings.ts'
 import { escapeUnbalanced, spellDestination, spellLinkTarget } from '../commonmark/link-syntax.ts'
@@ -28,6 +29,7 @@ type InlineContext = {
   atBlockEnd: boolean
   bracketed: boolean
   carried: ReadonlySet<number>
+  container: LineContainer
   flavour: WrittenFlavour
   openingLinkAsDirective: boolean
   path: ConvertErrorPath
@@ -65,17 +67,19 @@ export function portableLineFallback(nodes: readonly AdfNode[], container: LineC
 export function tryPipeCell(nodes: readonly AdfNode[], path: ConvertErrorPath, flavour: WrittenFlavour): string | undefined {
   const emitted = emitLine(nodes, 'table-cell', path, flavour)
   if (!emitted.ok) return undefined
-  if (emitted.value.segments.some((segment) => isSyntax(segment.escaping) && segment.text.includes('|'))) return undefined
+  // spec/flavour.md, Tables: the lossless flavour writes a table holding a `|` in syntax as directives; the portable flavour's only such `|` is in a title, escaped.
+  const splitsCell = (syntax: string): boolean => (flavour === 'lossless' ? syntax.includes('|') : barePipeCells(syntax) !== undefined)
+  if (emitted.value.segments.some((segment) => isSyntax(segment.escaping) && splitsCell(segment.text))) return undefined
   return emitted.value.line
 }
 
-export function tryImageLine(alt: string | undefined, url: string, link?: LinkDefinition): string | undefined {
+export function tryImageLine(alt: string | undefined, url: string, flavour: WrittenFlavour, link?: LinkDefinition): string | undefined {
   if (alt !== undefined && (/^[ \t]|[ \t]$|[\n\r]/.test(alt) || holdsNullCharacter(alt))) return undefined
   const destination = spellDestination(url)
   if (destination === undefined) return undefined
   const description: InlineSegment[] = alt === undefined ? [] : [{ escaping: 'bracketed', text: alt }]
   if (link === undefined) return attemptLine([syntax('!['), ...description, syntax(`](${destination})`)], 'paragraph', 'lossless').line
-  const target = spellLinkTarget(link.destination, link.title)
+  const target = spellLinkTarget(link.destination, link.title, titleEscapes(flavour, 'paragraph'))
   if (target === undefined) return undefined
   return attemptLine([syntax('[!['), ...description, syntax(']('), bracketedLinkTarget(destination), syntax(`)](${target})`)], 'paragraph', 'lossless').line
 }
@@ -123,7 +127,7 @@ function takeFallback(fallbacks: LineFallbacks, fallback: NodeRange | 'opening-l
 }
 
 function lineSegments(nodes: readonly AdfNode[], container: LineContainer, path: ConvertErrorPath, fallbacks: LineFallbacks): Result<Emission> {
-  const context: InlineContext = { atBlockEnd: true, bracketed: false, ...fallbacks, path, spansLines: container === 'paragraph' }
+  const context: InlineContext = { atBlockEnd: true, bracketed: false, ...fallbacks, container, path, spansLines: container === 'paragraph' }
   const emission = emitRun(nodes, 0, 0, context)
   if (!emission.ok) return emission
   if (emission.value.carry !== undefined) return emission
@@ -179,6 +183,11 @@ function syntax(text: string): InlineSegment {
   return { escaping: 'none', text }
 }
 
+function titleEscapes(flavour: WrittenFlavour, container: LineContainer): TitleEscapes {
+  if (flavour === 'lossless') return 'lossless'
+  return container === 'table-cell' ? 'portable-table-cell' : 'portable'
+}
+
 function bracketedLinkTarget(target: string): InlineSegment {
   return { escaping: 'bracketed-link-target', text: escapeUnbalanced(target, '[', ']') }
 }
@@ -345,7 +354,7 @@ function tryLink(nodes: readonly AdfNode[], mark: AdfMark, depth: number, range:
   const href = linkHref(nodeAttrs(mark))
   if (href === undefined) return success({ carry: range })
   const opening = depth === 0 && range.first === 0 && context.openingLinkAsDirective
-  const commonMark = opening ? undefined : commonMarkLink(nodeAttrs(mark), href, nodes, depth + 1, context.bracketed)
+  const commonMark = opening ? undefined : commonMarkLink(nodeAttrs(mark), href, nodes, depth + 1, context.bracketed, titleEscapes(context.flavour, context.container))
   if (commonMark === undefined) return undefined
   if (commonMark.form === 'autolink') return success({ segments: [syntax(`<${href}>`)] })
   const inner = emitRun(nodes, depth + 1, range.first, { ...context, bracketed: true })

src/markdown/mark-spellings.ts

@@ -1,6 +1,6 @@
 import type { AdfAttributes, AdfMark, AdfNode } from '../adf/document.ts'
 import type { AttributeVocabulary } from '../adf/attribute-vocabulary.ts'
-import type { LinkDefinition } from './commonmark/link-syntax.ts'
+import type { LinkDefinition, TitleEscapes } from './commonmark/link-syntax.ts'
 import type { MarkType } from '../adf/mark-attributes.ts'
 import { emptyKeys, nodeAttrs, nodeMarks } from '../adf/document.ts'
 import { escapeUnbalanced, spellLinkTarget } from './commonmark/link-syntax.ts'
@@ -55,12 +55,12 @@ export function linkHref(attrs: AdfAttributes): string | undefined {
 }
 
 // spec/flavour.md, Marks. `marksInside` counts the marks the link's nodes carry within it: the emitter's depth + 1, the parser's 0.
-export function commonMarkLink(attrs: AdfAttributes, href: string, nodes: readonly AdfNode[], marksInside: number, bracketed: boolean): CommonMarkLink | undefined {
+export function commonMarkLink(attrs: AdfAttributes, href: string, nodes: readonly AdfNode[], marksInside: number, bracketed: boolean, escapes: TitleEscapes): CommonMarkLink | undefined {
   if (Object.keys(attrs).some((key) => key !== 'href' && key !== 'title')) return undefined
   const title = attrs['title']
   const autolinkHolds = !bracketed || (!href.includes('`') && !holdsDirectivePrefix(href) && escapeUnbalanced(href, '[', ']') === href)
   if (isBareLink(nodes, href, marksInside) && autolinkHolds && title === undefined && isAutolink(href) && !holdsEntityReference(href)) return { form: 'autolink' }
-  const target = spellLinkTarget(href, typeof title === 'string' ? title : undefined)
+  const target = spellLinkTarget(href, typeof title === 'string' ? title : undefined, escapes)
   return target === undefined ? undefined : { form: 'inline', target }
 }
 

src/markdown/parse/inline-content.ts

@@ -225,7 +225,7 @@ function refuseLinkDirective(scan: Scan, mark: AdfMark, nodes: readonly Inline[]
   const href = linkHref(nodeAttrs(mark))
   if (href === undefined) return failure('unsupported-node-shape', hreflessLink, scan.path)
   if (marksLink(nodes)) return failure('unsupported-node-shape', linkInLink, scan.path)
-  if (commonMarkLink(nodeAttrs(mark), href, adfNodes(nodes), 0, scan.container === undefined) === undefined) return undefined
+  if (commonMarkLink(nodeAttrs(mark), href, adfNodes(nodes), 0, scan.container === undefined, 'lossless') === undefined) return undefined
   if (index !== 0 || scan.container !== 'paragraph') return failure('unsupported-node-shape', spellableLink, scan.path)
   scan.openingSpellableLink = true
   return undefined

src/markdown/portable/adf-to-portable-markdown.test.ts

@@ -319,10 +319,26 @@ test('keeps a table as a pipe table headed by its first row, one line per cell',
   assert.equal(portable(node('table', {}, said('stray'))), '| stray |\n| --- |\n')
   const titled = paragraph(text('t', link('https://e.com', 'a|b')))
   const folded = [node('expand', { title: 'Log' }, said('x')), node('nestedExpand', {}, said('y'))]
-  assert.equal(portable(node('table', {}, row(cell('tableHeader', titled), cell('tableHeader', ...folded)))), '| [t](https://e.com) | Log x y |\n| --- | --- |\n')
+  assert.equal(portable(node('table', {}, row(cell('tableHeader', titled), cell('tableHeader', ...folded)))), '| [t](https://e.com "a\\|b") | Log x y |\n| --- | --- |\n')
   assert.equal(portable(node('table', {}, row())), '')
 })
 
+test('escapes a backslash, an ampersand opening an entity reference and, in a table cell, a pipe in a link title, which reads back', () => {
+  const titles = ['C:\\dir\\', 'AT&T & & &x', 'say "hi"', 'a|b', 'a\\|b']
+  const spelled = ['"C:\\\\dir\\\\"', '"AT\\&T & \\& &x"', '"say \\"hi\\""', '"a|b"', '"a\\\\|b"']
+  const inCell = ['"C:\\\\dir\\\\"', '"AT\\&T & \\& &x"', '"say \\"hi\\""', '"a\\|b"', '"a\\\\\\|b"']
+  for (const [index, title] of titles.entries()) {
+    const linked = paragraph(text('t', link('https://e.com', title)))
+    const table: AdfNode = { content: [row(cell('tableHeader', linked))], type: 'table' }
+    assert.equal(portable(linked), `[t](https://e.com ${spelled[index]})\n`)
+    assert.equal(portable(table), `| [t](https://e.com ${inCell[index]}) |\n| --- |\n`)
+    assert.deepEqual(portableMarkdownToAdf(portable(linked)), { ok: true, value: document(linked) })
+    assert.deepEqual(portableMarkdownToAdf(portable(table)), { ok: true, value: document(table) })
+  }
+  const image: AdfNode = { attrs: { alt: 'Moon', type: 'external', url: 'https://e.com/moon.png' }, marks: [link('https://e.com', 'C:\\dir')], type: 'media' }
+  assert.equal(portable(node('mediaSingle', { layout: 'center' }, image)), '[![Moon](https://e.com/moon.png)](https://e.com "C:\\\\dir")\n')
+})
+
 test('keeps code, em, link, strike and strong and drops every other mark, and code text every mark but link, keeping its text', () => {
   const marks: AdfMark[] = [{ type: 'strike' }, { attrs: { type: 'sub' }, type: 'subsup' }, { type: 'underline' }, { attrs: { color: '#f00' }, type: 'textColor' }]
   assert.equal(portable(paragraph(text('H'), text('2', ...marks), text('O', em, strong), text('!', { attrs: { size: 1 }, type: 'border' }))), 'H~~2~~_**O**_!\n')
@@ -333,7 +349,7 @@ test('keeps code, em, link, strike and strong and drops every other mark, and co
 
 test('percent-encodes a link href no CommonMark escape writes until one does', () => {
   assert.equal(portable(paragraph(text('a', link('a\\b')), text(' '), text('b', { attrs: { id: 'i' }, type: 'link' }), text(' '), text('c', link('/&')))), '[a](a%5Cb) b [c](/%26amp;)\n')
-  assert.equal(portable(paragraph(text('t', link('https://e.com', 'two\nlines')), text(' '), text('u', link('https://e.com', 'a\\b')))), '[t](https://e.com "two lines") [u](https://e.com)\n')
+  assert.equal(portable(paragraph(text('t', link('https://e.com', 'two\nlines')), text(' '), text('u', link('https://e.com', 'a\\b')))), '[t](https://e.com "two lines") [u](https://e.com "a\\\\b")\n')
   assert.equal(portable(paragraph(text(']: a', link('/u'), code))), '[\\]: a](/u)\n')
 })
 

src/markdown/portable/inline-reduction.ts

@@ -8,7 +8,7 @@ import { joinsWhenEditorNormal, sameMarkWhenEditorNormal } from './editor-normal
 import { largestNesting } from '../../nesting.ts'
 import { mergeAdjacentText, nodeAttrs, nodeContent, nodeMarks } from '../../adf/document.ts'
 import { portableLineFallback, type PortableLineFallback } from '../emit/inline-line.ts'
-import { spellDestination, spellLinkTarget } from '../commonmark/link-syntax.ts'
+import { spellDestination } from '../commonmark/link-syntax.ts'
 
 const highlight = 'backgroundColor'
 const highlightMark: AdfMark = { type: highlight }
@@ -158,8 +158,7 @@ export function portableLink(mark: AdfMark, container: LineContainer): AdfMark |
   const title = typeof attrs['title'] === 'string' ? attrs['title'].replace(/[\r\0]/g, '').replace(/\n/g, ' ') : undefined
   if (typeof held !== 'string') return undefined
   const href = writableHref(container === 'table-cell' ? held.replaceAll('|', '%7C') : held)
-  if (title === undefined || spellLinkTarget(href, title) === undefined || (container === 'table-cell' && title.includes('|'))) return { attrs: { href }, type: 'link' }
-  return { attrs: { href, title }, type: 'link' }
+  return { attrs: title === undefined ? { href } : { href, title }, type: 'link' }
 }
 
 // spec/flavour.md, Links: the characters no destination spelling holds, then an ampersand an entity reference would read.

todo.md

@@ -12,7 +12,6 @@
 
 | ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
 |---|---|---|---|---|---|---|---|---|---|---|---|
-| 123 | 0.3.0 |  | defect | **Spell a link title holding a backslash, an entity reference or, in a table cell, a pipe, instead of dropping it.** | 3 | 1 | 2 | 4 | 3 | 7 by 2 | 13.1 |
 | 122 | 0.3.0 |  | defect | **Keep an image's URL and title when a link's text holds the image, instead of dropping them.** | 3 | 1 | 2 | 4 | 5 | 7 by 4 | 10.5 |
 | 95 | 0.3.0 |  | decision | **Find the code-span closers an escape forms, the mark runs a line cannot spell and the text `[` that opens link text by reading the line back, in place of `escapeClosedRuns`, `unspellableRuns` and `lastLinkClose` predicting them.** | 4 | 1 | 3 | 6 | 6 | code health by 7 | 10.2 |
 | 125 | 0.3.0 |  |  | **Anchor each level of L, E and I in `docs/score-anchors.md` with two or three shipped items.** | 3 | 2 | 3 | 6 | 4 | code health by 2 | 8.7 |
@@ -20,7 +19,7 @@
 | 109 | 0.3.0 |  | defect | **Spell nested lists and quotes in time linear in the markdown.** | 3 | 1 | 2 | 4 | 5 | 9 by 3 | 7.8 |
 | 115 | 0.3.0 |  | defect | **Build text holding no `border` mark in `losslessMarkdownToAdf`, and have `adfToLosslessMarkdown` carry text holding one.** | 2 | 1 | 2 | 3 | 3 | 4 by 3 | 4.6 |
 | 91 | 0.3.0 | 95 |  | **Spell an emphasis once across the code spans, an emphasis or link once across the hard breaks, dates, emoji, inline cards, mentions and statuses between two nodes it marks, and an emphasis across inline media, in both writers.** | 3 | 2 | 2 | 5 | 5 | 6 by 6 | 3.6 |
-| 73 | 0.3.0 | 33, 91, 95, 109, 115, 122, 123, 125 |  | **Announce the package where someone needing an ADF converter already reads: JRACLOUD-77436, the Atlassian developer community and Stack Overflow's ADF-to-markdown questions.** | 1 | 2 | 2 | 3 | 2 | 3 by 8 | 16.9 |
+| 73 | 0.3.0 | 33, 91, 95, 109, 115, 122, 125 |  | **Announce the package where someone needing an ADF converter already reads: JRACLOUD-77436, the Atlassian developer community and Stack Overflow's ADF-to-markdown questions.** | 1 | 2 | 2 | 3 | 2 | 3 by 8 | 16.9 |
 | 128 | 0.4.0 |  |  | **Pin the portable flavour's block reading and writing with tests: callout titles past one line, a link beside other marks in a title, the nesting limit of callouts and task lists, refusals inside tasks and their paths, and empty or non-paragraph tasks.** | 2 | 1 | 1 | 2 | 3 | code health by 3 | 19.6 |
 | 75 | 0.4.0 |  |  | **Make the generated-markdown property write block carries as `adf:` fences, canonical and hostile.** | 1 | 1 | 1 | 1 | 2 | code health by 2 | 17.4 |
 | 45 | 0.4.0 |  | defect | **Replace `isAdfDocument` with a reader returning `Result<AdfDocument>`.** | 2 | 1 | 3 | 4 | 3 | 1 by 2 | 13.1 |
@@ -56,16 +55,6 @@
 
 ## Details
 
-### 123. Spell a link title holding a backslash, an entity reference or, in a table cell, a pipe, instead of dropping it.
-
-The portable writer drops a link's title when CommonMark cannot spell it as written: a title
-holding a backslash or an entity reference (`spellTitle` in `src/markdown/commonmark/link-syntax.ts`),
-or, in a table cell, one holding a pipe (`portableLink` in `src/markdown/portable/inline-reduction.ts`).
-README `## Goals` Goal 7 counts a title as content, so each drop breaks a Maintain goal today.
-Escaping keeps the title: `\\` for a backslash, `\&` for an ampersand that would open an entity
-reference, and `\|` for a pipe in a table cell. `portableMarkdownToAdf` must read each spelling
-back to the same title.
-
 ### 122. Keep an image's URL and title when a link's text holds the image, instead of dropping them.
 
 When a link's text holds an image, the reader turns the image into its alt text, linked, and drops