3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-18 22:25:44 +01:00

Read adfToHtml's HTML in htmlToAdf as a browser serializes it too

This commit is contained in:
2026-10-31 01:48:10 +01:00
committed by Lillem4n
parent 088494511d
commit 6cc59f8574
13 changed files with 199 additions and 57 deletions
+48 -12
View File
@@ -1,17 +1,35 @@
const entities: ReadonlyMap<string, string> = new Map([
export type Spelling = 'attribute' | 'text'
const escapable = /[&<>"]/g
const escapes: ReadonlyMap<string, string> = new Map([
['"', '&quot;'],
['&', '&amp;'],
['<', '&lt;'],
['>', '&gt;'],
])
const writtenEntities = [...escapes.values()]
const entity = /&(?:amp|gt|lt|nbsp|quot);/g
const entityValues: ReadonlyMap<string, string> = new Map([
['&amp;', '&'],
['&gt;', '>'],
['&lt;', '<'],
['&nbsp;', '\u00a0'],
['&quot;', '"'],
])
const entity = new RegExp([...entities.keys()].join('|'), 'g')
const escapable = new RegExp(`[${[...entities.values()].join('')}]`, 'g')
const escapes: ReadonlyMap<string, string> = new Map([...entities].map(([spelled, character]) => [character, spelled]))
// What HTML text and attribute values cannot hold (WHATWG HTML §13.1.3): any control but a tab, line feed or form feed, a carriage return among them, since a parser reads it as a line feed; a noncharacter; and a lone surrogate, which UTF-8 cannot encode.
const unwritable = /(?![\t\n\f ])[\p{Cc}\p{Cs}\p{Noncharacter_Code_Point}]/u
const unwritables = new RegExp(unwritable.source, 'gu')
// A character `escapeHtml` escapes, other than the `&` opening an entity, or one HTML cannot hold.
const misspelling = new RegExp(`(?!${entity.source})${escapable.source}|${unwritable.source}`, 'u')
const strayAmpersand = '&(?!(?:amp|gt|lt|nbsp|quot);)'
// What reads wrongly in a value: `&` outside an entity, a character HTML cannot hold, and in text a `<` or `>`. An attribute value may hold `<` and `>` raw, as older browsers write it, and text may hold `"`.
const misspellings: Readonly<Record<Spelling, RegExp>> = {
attribute: new RegExp(`${strayAmpersand}|${unwritable.source}`, 'u'),
text: new RegExp(`${strayAmpersand}|[<>]|${unwritable.source}`, 'u'),
}
// What `adfToHtml` spells with an entity, a browser (WHATWG HTML §13.3) spells raw in text, or in an attribute; a no-break space it spells `&nbsp;` in both.
const browserSpellings: Readonly<Record<Spelling, ReadonlyMap<string, string>>> = {
attribute: new Map([['&gt;', '>'], ['&lt;', '<'], ['\u00a0', '&nbsp;']]),
text: new Map([['&quot;', '"'], ['\u00a0', '&nbsp;']]),
}
// The carry's JSON, which spells each character HTML cannot hold as a JSON escape reading back to the same string.
export function holdableJson(json: string): string {
@@ -22,9 +40,9 @@ export function jsonValue(json: string): string {
return escapeHtml(holdableJson(json))
}
// The index of the first character `writableValue` would not write so, -1 where none stands in `html`.
export function misspelledAt(html: string): number {
return html.search(misspelling)
// The index of the first character that reads wrongly in `html`, -1 where none stands in it.
export function misspelledAt(html: string, spelling: Spelling): number {
return html.search(misspellings[spelling])
}
// ` name="value"`, `''` where the value is undefined, `undefined` where HTML cannot hold it.
@@ -38,9 +56,27 @@ export function shownValue(value: string, lineFeed: string): string {
return escapeHtml(value.replace(unwritables, '')).replaceAll('\n', lineFeed)
}
// The value an escaped HTML text or attribute stands for, `undefined` where `writableValue` would not write it so.
export function unescapedValue(html: string): string | undefined {
return misspelledAt(html) < 0 ? html.replace(entity, (spelled) => entities.get(spelled) ?? spelled) : undefined
// Where `html` reads as `shown` from `index`: `whole`, and the index after it, or where it first reads otherwise. `shown` is spelled as `adfToHtml` writes it, `html` as a browser may too.
export function readsAsShown(html: string, index: number, shown: string): { at: number; whole: boolean } {
if (html.startsWith(shown, index)) return { at: index + shown.length, whole: true }
let at = index
let spelling: Spelling = 'text'
for (let next = 0; next < shown.length; ) {
const token = writtenEntities.find((spelled) => shown.startsWith(spelled, next)) ?? shown.charAt(next)
// Only a tag holds a raw `<` or `>`, and only its attribute values an escape.
if (token === '<') spelling = 'attribute'
if (token === '>') spelling = 'text'
const read = [token, browserSpellings[spelling].get(token)].find((spelled) => spelled !== undefined && html.startsWith(spelled, at))
if (read === undefined) return { at, whole: false }
at += read.length
next += token.length
}
return { at, whole: true }
}
// The value an escaped HTML text or attribute stands for, `undefined` where `misspelledAt` finds a character that reads wrongly.
export function unescapedValue(html: string, spelling: Spelling): string | undefined {
return misspelledAt(html, spelling) < 0 ? html.replace(entity, (spelled) => entityValues.get(spelled) ?? spelled) : undefined
}
// `undefined` where HTML cannot hold the value, so its node rides the carry element.
+4 -5
View File
@@ -4,7 +4,7 @@ import { carriedNesting, carriedNode, carriedValue } from '../../adf/carried-jso
import { carryContent } from '../carry-content.ts'
import { carryJsonOffset, readCarryAttributes } from '../carry-attributes.ts'
import { foundAt, refused, type Place, type ReadNode, type StartTag } from './source.ts'
import { holdableJson } from '../escaping.ts'
import { holdableJson, readsAsShown } from '../escaping.ts'
import { spellsCanonicalJson } from '../../canonical-json.ts'
import { success } from '../../result.ts'
@@ -18,10 +18,9 @@ export function readCarry(html: string, tag: StartTag, element: 'div' | 'span',
const node = heldNode(json, levels, html, jsonAt)
if (node.fault !== undefined) return refused(node.fault.code, node.fault.message, html, jsonAt, place)
const shown = `${carryContent(node.value, element)}</${element}>`
if (html.startsWith(shown, tag.end)) return success({ end: tag.end + shown.length, node: node.value })
let at = tag.end
while (html[at] === shown[at - tag.end]) at += 1
return refused('unmappable-html', `a carry element shows exactly what adfToHtml writes for its node, then </${element}>; to change the node, edit its ADF and write the HTML again: ${foundAt(html, at)}`, html, at, place)
const read = readsAsShown(html, tag.end, shown)
if (read.whole) return success({ end: read.at, node: node.value })
return refused('unmappable-html', `a carry element shows exactly what adfToHtml writes for its node, then </${element}>; to change the node, edit its ADF and write the HTML again: ${foundAt(html, read.at)}`, html, read.at, place)
}
function heldNode(json: string, levels: number, html: string, jsonAt: number): Read<AdfNode> {
+50 -3
View File
@@ -52,7 +52,7 @@ const rule = JSON.stringify({ type: 'unknownBlock' }).replaceAll('"', '&quot;')
// The carry element of `node`, showing `shown`.
function carry(tag: 'div' | 'span', node: unknown, shown: string): string {
return `<${tag} class="adf-carry" data-adf="${JSON.stringify(node).replaceAll('"', '&quot;')}">${shown}</${tag}>`
return `<${tag} class="adf-carry" data-adf="${JSON.stringify(node).replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;')}">${shown}</${tag}>`
}
function nested(value: unknown, levels: number, wrap: (inner: unknown) => unknown): unknown {
@@ -83,6 +83,53 @@ test('reads escaped text, and keeps tabs, line feeds and form feeds', () => {
assert.deepEqual(read(htmlToAdf('<p>a &lt; b &amp;&amp; c &gt; &quot;d&quot; \'e\'\tf\ng\fh &amp;lt;</p>\n')), document(paragraph(text('a < b && c > "d" \'e\'\tf\ng\fh &lt;'))))
})
test('reads text as a browser\'s innerHTML spells it: a raw ", &nbsp; for a no-break space, and in an attribute a raw < or >', () => {
assert.deepEqual(read(htmlToAdf('<p>a "b" &quot;c&quot;&nbsp;d e</p>\n')), document(paragraph(text('a "b" "c" d e'))))
assert.deepEqual(read(htmlToAdf('<pre><code>say "a"&nbsp;b</code></pre>\n')), document({ content: [text('say "a" b')], type: 'codeBlock' }))
const link: AdfMark = { attrs: { href: 'https://x.test/?a<b>c&d', title: 'a b' }, type: 'link' }
assert.deepEqual(read(htmlToAdf('<p><a href="https://x.test/?a<b>c&amp;d" title="a&nbsp;b">x</a></p>\n')), document(paragraph(text('x', link))))
assert.deepEqual(read(htmlToAdf('<p><a href="https://x.test/?a&lt;b&gt;c&amp;d" title="a&nbsp;b">x</a></p>\n')), document(paragraph(text('x', link))))
assert.deepEqual(read(htmlToAdf('<hr data-style="a&nbsp;&quot;b">\n')), document({ attrs: { style: 'a "b' }, type: 'rule' }))
})
test('reads a line ending of CRLF as a line feed wherever the dialect has one, and keeps a lone carriage return refused', () => {
const crlf = (html: string): string => html.replaceAll('\n', '\r\n')
assert.deepEqual(read(htmlToAdf(crlf('<p>a\nb</p>\n<hr>\n<pre><code>c\n d</code></pre>\n'))), document(paragraph(text('a\nb')), { type: 'rule' }, { content: [text('c\n d')], type: 'codeBlock' }))
assert.deepEqual(read(htmlToAdf(crlf('<blockquote>\n<ul>\n<li>Bolt\n<p>b</p>\n</li>\n</ul>\n</blockquote>\n'))), document({ content: [{ content: [item(paragraph(text('Bolt')), paragraph(text('b')))], type: 'bulletList' }], type: 'blockquote' }))
assert.deepEqual(read(htmlToAdf(crlf(`${carry('div', panel, '<p>Check the collation.</p>')}\n`))), document(panel))
assert.deepEqual(read(htmlToAdf(crlf(`${documentCarry}\n`))), { type: 'doc', version: 1 })
assert.deepEqual(read(htmlToAdf('<hr data-style="a\r\nb">\n')), document({ attrs: { style: 'a\nb' }, type: 'rule' }))
for (const html of ['<p>a\r</p>\n', '<p>a</p>\r', '<p>a</p>\r\r\n', '<p>a\r\r\nb</p>\n']) assert.equal(refusal(htmlToAdf(html)).startsWith('unmappable-html'), true, JSON.stringify(html))
})
test('places a refusal in HTML ending its lines in CRLF at the start of the line it stands on, counted in the HTML handed in', () => {
assert.equal(refusal(htmlToAdf('<p>a</p>\r\n<b>x</b>\r\n')), 'unmappable-html at /content/1 line 2 offset 10')
assert.equal(refusal(htmlToAdf('<p>a</p>\r\n<p>b</p>\r\n<p>c<b>d</b></p>\r\n')), 'unmappable-html at /content/2 line 3 offset 20')
assert.equal(refusal(htmlToAdf('<p>a</p>\r\n<p>b</p>')), 'unmappable-html at / line 2 offset 10')
})
test('reads a carry element showing what a browser spells it as, in any mix with the spelling adfToHtml writes', () => {
const said: AdfNode = { attrs: { id: ids.mention, text: 'Say "hi" now' }, type: 'mention' }
for (const shown of ['Say &quot;hi&quot; now', 'Say "hi"&nbsp;now', 'Say "hi" now', 'Say &quot;hi&quot;&nbsp;now']) {
assert.deepEqual(read(htmlToAdf(`<p>${carry('span', said, shown)}</p>\n`)), document(paragraph(said)), shown)
}
const underlined: AdfNode = text('a<b', { type: 'underline' }, { attrs: { href: 'https://x.test/?a<b&c>d', title: 'T" ' }, type: 'link' })
for (const opener of ['<a href="https://x.test/?a&lt;b&amp;c&gt;d" title="T&quot; ">', '<a href="https://x.test/?a<b&amp;c>d" title="T&quot;&nbsp;">']) {
assert.deepEqual(read(htmlToAdf(`<p>${carry('span', underlined, `${opener}a&lt;b</a>`)}</p>\n`)), document(paragraph(underlined)), opener)
}
const block: AdfNode = { attrs: { panelType: 'info' }, content: [paragraph(text('A "quote" here'))], type: 'panel' }
assert.deepEqual(read(htmlToAdf(`${carry('div', block, '<p>A "quote"&nbsp;here</p>')}\r\n`)), document(block))
})
test('refuses an edit to what a carry element shows even where a browser would spell the same text so', () => {
const said: AdfNode = { attrs: { id: ids.mention, text: 'Say "hi"' }, type: 'mention' }
const underlined: AdfNode = text('a', { type: 'underline' }, { attrs: { href: 'x', title: 'T"' }, type: 'link' })
assert.equal(refusal(htmlToAdf(`<p>${carry('span', said, 'Say "hi')}</p>\n`)), 'unmappable-html at /content/0/content/0 line 1 offset 0')
assert.equal(refusal(htmlToAdf(`<p>${carry('span', said, 'Say &amp;quot;hi&amp;quot;')}</p>\n`)), 'unmappable-html at /content/0/content/0 line 1 offset 0')
assert.equal(refusal(htmlToAdf(`<p>${carry('span', underlined, '<a href="x" title="T"">a</a>')}</p>\n`)), 'unmappable-html at /content/0/content/0 line 1 offset 0')
assert.equal(refusal(htmlToAdf(`<p>${carry('span', underlined, '<a href="x" title="T&quot;">a</a>')}</p>\n`)), 'read')
})
test('reads lists, an item\'s first paragraph on its line and its other blocks a line each', () => {
assert.deepEqual(read(htmlToAdf('<ul>\n<li>Bolt</li>\n<li>Nut</li>\n</ul>\n')), document({ content: [item(paragraph(text('Bolt'))), item(paragraph(text('Nut')))], type: 'bulletList' }))
const listed: AdfNode = { attrs: { order: 3 }, content: [item(paragraph(text('Bolt')), { content: [item(paragraph(text('M8')))], type: 'bulletList' }), item(paragraph(text('a')), paragraph(text('b')))], type: 'orderedList' }
@@ -279,9 +326,9 @@ test('refuses a tag of three million attributes, past what one pattern over them
const messages: readonly (readonly [string, string])[] = [
['<p>a<b>b</b></p>\n', '<b> as written here is no inline element of the dialect: found "<b>b</b></p>\\n"'],
['<p>a > b</p>\n', 'text escapes &, <, > and " as &amp;, &lt;, &gt; and &quot;, and holds no character HTML cannot hold: found "> b</p>\\n"'],
['<p>a > b</p>\n', 'text escapes & as &amp;, < as &lt; and > as &gt;, and holds no character HTML cannot hold: found "> b</p>\\n"'],
['<p><a href="a"b">a</a></p>\n', '<a> holds its attributes as name="value", each value escaped as text is, then >: found "b\\">a</a></p>\\n"'],
['<p>a&nbsp;\u00a0</p>\n', 'text escapes &, <, > and " as &amp;, &lt;, &gt; and &quot;, and holds no character HTML cannot hold: found "&nbsp;\\u00a0</p>\\n"'],
['<p>a&nbsp</p>\n', 'text escapes & as &amp;, < as &lt; and > as &gt;, and holds no character HTML cannot hold: found "&nbsp</p>\\n"'],
['<pre><code>a</code></pre\n', 'a code block\'s text ends at </code></pre>: found "</code></pre\\n"'],
['<pre><code class="hljs language-js">x</code></pre>\n', '<code> as written here is no code element of the dialect, which writes <code> or <code class="language-…">: found "<code class=\\"hljs langua"…'],
['<ul>\n<li>a</ul>\n', 'an item\'s line ends at </li>, or at a line feed before the item\'s next block, and its text spells < as &lt;: found "</ul>\\n"'],
+10 -3
View File
@@ -7,7 +7,7 @@ import { carryContent } from '../carry-content.ts'
import { centeredImage, externalMedia } from '../../adf/plain-image.ts'
import { containerElements, headingElements, headingLevel, readListStart, type ContainerType } from '../block-elements.ts'
import { firstParagraphForm } from '../first-paragraph.ts'
import { foundAt, misread, readStartTag, refused, startTagName, type Place, type ReadNode, type StartTag } from './source.ts'
import { foundAt, misread, positionIn, readStartTag, refused, startTagName, type Place, type ReadNode, type StartTag } from './source.ts'
import { largestNesting } from '../../nesting.ts'
import { misspelledAt, unescapedValue } from '../escaping.ts'
import { misspelledText, readInline } from './inline.ts'
@@ -59,8 +59,15 @@ export function htmlToAdf(html: string): Result<AdfDocument, ParseError> {
)
}
// A form post ends each line in `\r\n`, which reads as the line feed `adfToHtml` writes; a refusal's position still counts the HTML as handed in.
function readHtml(html: string): Result<AdfDocument> {
if (typeof html !== 'string') return notAString(html, 'HTML')
const read = readLineFeeds(html.replaceAll('\r\n', '\n'))
if (read.ok || read.error.position === undefined) return read
return { error: { ...read.error, position: positionIn(html, read.error.position) }, ok: false }
}
function readLineFeeds(html: string): Result<AdfDocument> {
if (html === '') return success({ content: [], type: 'doc', version: 1 })
return readDocumentCarry(html) ?? readBlocks(html)
}
@@ -191,8 +198,8 @@ function readCodeBlock(html: string, tag: StartTag, place: Place): Result<BlockS
const next = html.indexOf('<', code.end)
const end = next < 0 ? html.length : next
const spelled = html.slice(code.end, end)
const text = unescapedValue(spelled)
if (text === undefined) return misspelledText(html, code.end + misspelledAt(spelled), place)
const text = unescapedValue(spelled, 'text')
if (text === undefined) return misspelledText(html, code.end + misspelledAt(spelled, 'text'), place)
if (!html.startsWith(codeCloser, end)) return refused('unmappable-html', `a code block's text ends at ${codeCloser}: ${foundAt(html, end)}`, html, end, place)
const { language } = named
return success({ end: end + codeCloser.length, kind: 'node', node: built('codeBlock', language === undefined ? data.value : { ...data.value, language }, text === '' ? [] : [{ text, type: 'text' }]) })
+4 -3
View File
@@ -63,13 +63,14 @@ function readMarked(html: string, index: number, place: Place): ReadNode {
function readText(html: string, index: number, marks: AdfMark[], place: Place): ReadNode {
const next = html.indexOf('<', index)
const end = next < 0 ? html.length : next
const text = unescapedValue(html.slice(index, end))
if (text === undefined) return misspelledText(html, index + misspelledAt(html.slice(index, end)), place)
const spelled = html.slice(index, end)
const text = unescapedValue(spelled, 'text')
if (text === undefined) return misspelledText(html, index + misspelledAt(spelled, 'text'), place)
return success({ end, node: marks.length === 0 ? { text, type: 'text' } : { marks, text, type: 'text' } })
}
export function misspelledText<T>(html: string, index: number, place: Place): Result<T, ParseError> {
return refused('unmappable-html', `text escapes &, <, > and " as &amp;, &lt;, &gt; and &quot;, and holds no character HTML cannot hold: ${foundAt(html, index)}`, html, index, place)
return refused('unmappable-html', `text escapes & as &amp;, < as &lt; and > as &gt;, and holds no character HTML cannot hold: ${foundAt(html, index)}`, html, index, place)
}
function markOf(tag: StartTag): AdfMark | undefined {
+10 -3
View File
@@ -45,6 +45,13 @@ export function foundAt(html: string, index: number): string {
return `found ${spelled}${index + found.length < html.length ? '…' : ''}`
}
// The start of the same line in `html`, where the position was counted after its `\r\n` became line feeds.
export function positionIn(html: string, { line }: SourcePosition): SourcePosition {
let offset = 0
for (let passed = 1; passed < line; passed += 1) offset = html.indexOf('\n', offset) + 1
return { line, offset }
}
// At the start of the line `index` stands on.
export function refused<T>(code: ConvertErrorCode, message: string, html: string, index: number, place: Place | undefined): Result<T, ParseError> {
return { error: { code, message, path: pathOf(place), position: lineStart(html, index) }, ok: false }
@@ -61,14 +68,14 @@ function scanStartTag(html: string, index: number): CutTag | StartTag | undefine
const match = attribute.exec(html)
if (match === null) return html.startsWith('>', at) ? { attributes, end: at + 1, name, start: index } : { name, stopped: at }
const [whole, key = '', spelled = ''] = match
const value = unescapedValue(spelled)
if (value === undefined) return { name, stopped: at + whole.length - 1 - spelled.length + misspelledAt(spelled) }
const value = unescapedValue(spelled, 'attribute')
if (value === undefined) return { name, stopped: at + whole.length - 1 - spelled.length + misspelledAt(spelled, 'attribute') }
attributes.push([key, value])
at += whole.length
}
}
// Only a line feed ends a line: the reader refuses a carriage return wherever it stands.
// Only a line feed ends a line: readHtml has made each \r\n one, and a lone carriage return is refused.
function lineStart(html: string, index: number): SourcePosition {
let line = 1
let offset = 0