From 3ff1278b711a92da66421a1186fb680b9ea87251 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 30 Aug 2026 21:42:02 +0200 Subject: [PATCH 1/5] Read the inline text, and decode the escapes and references CommonMark spells --- AGENTS.md | 6 +- corpus/errors/inline-html.error | 1 + corpus/errors/inline-html.md | 1 + corpus/normalization/entity-references.json | 37 +++ corpus/normalization/entity-references.md | 3 + corpus/normalization/soft-wraps.json | 22 ++ corpus/normalization/soft-wraps.md | 3 + src/markdown/commonmark-grammar.ts | 83 +++++-- src/markdown/emit/adf-to-markdown.test.ts | 3 + src/markdown/emit/adf-to-markdown.ts | 3 +- src/markdown/emit/destination-spelling.ts | 7 +- src/markdown/emit/inline-line.ts | 3 +- src/markdown/emit/line-escaping.ts | 14 +- src/markdown/entity-references.ts | 237 ++++++++++++++++++++ src/markdown/parse/blocks.ts | 3 +- src/markdown/parse/html-blocks.ts | 16 +- src/markdown/parse/inline-content.ts | 93 ++++++++ src/markdown/parse/markdown-to-adf.test.ts | 89 +++++++- src/markdown/parse/markdown-to-adf.ts | 21 +- todo.md | 9 +- 20 files changed, 593 insertions(+), 61 deletions(-) create mode 100644 corpus/errors/inline-html.error create mode 100644 corpus/errors/inline-html.md create mode 100644 corpus/normalization/entity-references.json create mode 100644 corpus/normalization/entity-references.md create mode 100644 corpus/normalization/soft-wraps.json create mode 100644 corpus/normalization/soft-wraps.md create mode 100644 src/markdown/entity-references.ts create mode 100644 src/markdown/parse/inline-content.ts diff --git a/AGENTS.md b/AGENTS.md index 3527e67..21ce233 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,8 +53,10 @@ Round-trip equality is a property tested over a corpus, not a claim made in pros `dependencies` is empty. A runtime dependency enters only through a decision entry here stating why ~20 lines of own code cannot do the job, who maintains it, and what auditing it costs. So the -CommonMark and HTML parsers are written in this repo. `devDependencies`: few, each earning its -keep; they never reach a consumer. +CommonMark and HTML parsers are written in this repo. A table a standard fixes is data rather than +a dependency: HTML5's 2125 semicolon-terminated character references ship packed in +`markdown/entity-references.ts`, so entity decoding is complete without one. `devDependencies`: +few, each earning its keep; they never reach a consumer. ## 6. The package contract diff --git a/corpus/errors/inline-html.error b/corpus/errors/inline-html.error new file mode 100644 index 0000000..847e0f9 --- /dev/null +++ b/corpus/errors/inline-html.error @@ -0,0 +1 @@ +unmappable-html diff --git a/corpus/errors/inline-html.md b/corpus/errors/inline-html.md new file mode 100644 index 0000000..12c3825 --- /dev/null +++ b/corpus/errors/inline-html.md @@ -0,0 +1 @@ +A paragraph carrying a raw tag in its text. diff --git a/corpus/normalization/entity-references.json b/corpus/normalization/entity-references.json new file mode 100644 index 0000000..0afd7b4 --- /dev/null +++ b/corpus/normalization/entity-references.json @@ -0,0 +1,37 @@ +{ + "content": [ + { + "content": [ + { + "text": "© 2026 — the № you asked for, & nothing else.", + "type": "text" + } + ], + "type": "paragraph" + }, + { + "content": [ + { + "text": "Escaped, the reference stays literal: © and ", + "type": "text" + }, + { + "marks": [ + { + "type": "code" + } + ], + "text": "©", + "type": "text" + }, + { + "text": " alike.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/entity-references.md b/corpus/normalization/entity-references.md new file mode 100644 index 0000000..2287481 --- /dev/null +++ b/corpus/normalization/entity-references.md @@ -0,0 +1,3 @@ +© 2026 — the № you asked for, & nothing else. + +Escaped, the reference stays literal: \© and `©` alike. diff --git a/corpus/normalization/soft-wraps.json b/corpus/normalization/soft-wraps.json new file mode 100644 index 0000000..8a6f25f --- /dev/null +++ b/corpus/normalization/soft-wraps.json @@ -0,0 +1,22 @@ +{ + "content": [ + { + "content": [ + { + "text": "A paragraph the author soft-wrapped across three lines, holding a hard break", + "type": "text" + }, + { + "type": "hardBreak" + }, + { + "text": "the backslash spells.", + "type": "text" + } + ], + "type": "paragraph" + } + ], + "type": "doc", + "version": 1 +} diff --git a/corpus/normalization/soft-wraps.md b/corpus/normalization/soft-wraps.md new file mode 100644 index 0000000..b0a159f --- /dev/null +++ b/corpus/normalization/soft-wraps.md @@ -0,0 +1,3 @@ +A paragraph the author soft-wrapped +across three lines, holding a hard break\ +the backslash spells. diff --git a/src/markdown/commonmark-grammar.ts b/src/markdown/commonmark-grammar.ts index 09dbdf7..c3f4c64 100644 --- a/src/markdown/commonmark-grammar.ts +++ b/src/markdown/commonmark-grammar.ts @@ -1,16 +1,37 @@ +import { readEntityReference } from './entity-references.ts' + +export type HtmlConstruct = { length: number; name: string } + export type LinePosition = 'first' | 'later' +export const htmlConstructNames = { + cdata: 'a CDATA section', + comment: 'an HTML comment', + declaration: 'an HTML declaration', + processingInstruction: 'an HTML processing instruction', +} + const controlCharacterRange = '\\u0000-\\u001f\\u007f' const autolinkSource = `[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\\s<>${controlCharacterRange}]*` const nullCharacterSource = '\\u0000' -const entityReferenceSource = '&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\\d{1,7}|#[Xx][A-Fa-f0-9]{1,6});' +const tagNameSource = '[A-Za-z][A-Za-z0-9-]*' +const htmlSpaceSource = '[ \\t\\n]' +const attributeSource = `(?:${htmlSpaceSource}+[A-Za-z_:][A-Za-z0-9_.:-]*(?:${htmlSpaceSource}*=${htmlSpaceSource}*(?:[^ \\t\\n"'=<>\`]+|'[^']*'|"[^"]*"))?)` + +export const htmlTagSource = `(?:<${tagNameSource}${attributeSource}*${htmlSpaceSource}*/?>|)` -const anchoredEntityReference = new RegExp(`^(?:${entityReferenceSource})`) const autolink = new RegExp(`^(?:${autolinkSource})$`) const bracketedAutolink = new RegExp(`^<(?:${autolinkSource})>`) const controlCharacter = new RegExp(`[${controlCharacterRange}]`) -const entityReference = new RegExp(entityReferenceSource) +const htmlTag = new RegExp(`^${htmlTagSource}`) const nullCharacter = new RegExp(nullCharacterSource) +const tagName = new RegExp(`^/ }, + { name: htmlConstructNames.comment, pattern: /^(?:||)/ }, + { name: htmlConstructNames.declaration, pattern: /^]*>/ }, + { name: htmlConstructNames.processingInstruction, pattern: /^<\?[\s\S]*?\?>/ }, +] const asciiPunctuation = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/ const atxHeadingOpener = /^(#{1,6})(?:[ \t]|$)/ const codeFenceOpener = /^(`{3,}|~{3,})/ @@ -19,7 +40,7 @@ const pipeClaim = /^\|/ const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] -const htmlConstructs = [/^<[!?]/, /^<\/?[A-Za-z][A-Za-z0-9-]*(?:[\s/>]|$)/, /^<[^\s<>@]+@[^\s<>@]+>/] +const emailAutolink = /^<[^\s<>@]+@[^\s<>@]+>/ const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ @@ -50,6 +71,29 @@ export function closingCodeFence(line: string, marker: string): boolean { return /^[ \t]*$/.test(line.slice(closing.length)) } +// Text where only backslash escapes and entity references are processed: an info string, a link title. +export function decodeTextEscapes(text: string): string { + let decoded = '' + let index = 0 + while (index < text.length) { + const escaped = backslashEscape(text, index) + if (escaped !== undefined) { + decoded += escaped + index += 2 + continue + } + const reference = text.charAt(index) === '&' ? readEntityReference(text.slice(index)) : undefined + if (reference !== undefined) { + decoded += reference.text + index += reference.length + continue + } + decoded += text.charAt(index) + index += 1 + } + return decoded +} + export function escapesLineClaim(line: string, offset: number, position: LinePosition): boolean { if (offset === 0) { if (firstCharacterOpeners.some((opener) => opener.test(line)) || thematicBreak.test(line)) return true @@ -63,14 +107,23 @@ export function holdsControlCharacter(text: string): boolean { return controlCharacter.test(text) } -export function holdsEntityReference(text: string): boolean { - return entityReference.test(text) -} - export function holdsNullCharacter(text: string): boolean { return nullCharacter.test(text) } +export function htmlTagName(text: string): string { + return text.replace(tagName, '<$1>') +} + +export function inlineHtmlConstruct(text: string): HtmlConstruct | undefined { + for (const construct of inlineHtmlConstructs) { + const matched = construct.pattern.exec(text)?.[0] + if (matched !== undefined) return { length: matched.length, name: construct.name } + } + const tag = htmlTag.exec(text)?.[0] + return tag === undefined ? undefined : { length: tag.length, name: htmlTagName(tag) } +} + export function isAsciiPunctuation(character: string): boolean { return asciiPunctuation.test(character) } @@ -112,8 +165,8 @@ export function opensBracketedAutolink(text: string): boolean { return bracketedAutolink.test(text) } -export function opensHtmlConstruct(text: string): boolean { - return htmlConstructs.some((construct) => construct.test(text)) +export function opensEmailAutolink(text: string): boolean { + return emailAutolink.test(text) } export function setextHeadingLevel(line: string): number | undefined { @@ -122,10 +175,12 @@ export function setextHeadingLevel(line: string): number | undefined { return underline.startsWith('=') ? 1 : 2 } -export function startsEntityReference(text: string): boolean { - return anchoredEntityReference.test(text) -} - export function trimSpace(text: string): string { return text.replace(/^[ \t]+|[ \t]+$/g, '') } + +function backslashEscape(text: string, index: number): string | undefined { + if (text.charAt(index) !== '\\') return undefined + const escaped = text.charAt(index + 1) + return isAsciiPunctuation(escaped) ? escaped : undefined +} diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 80b106e..3979a67 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -204,6 +204,9 @@ test('escapes only text that would otherwise open a construct', () => { assert.equal(emitted('
'), '\\
\n') assert.equal(emitted('a < b'), 'a < b\n') assert.equal(emitted('& & x'), '\\& & x\n') + assert.equal(emitted('¬areference; x'), '¬areference; x\n') + assert.equal(emitted('a e'), 'a \\ e\n') + assert.equal(emitted('a { if (holdsControlCharacter(href)) return failure('unspellable-link-destination', 'a link destination holds a control character', path) if (href.includes('\\')) return failure('unspellable-link-destination', 'no canonical escape spells a backslash in a link destination', path) if (holdsEntityReference(href)) { - return failure('unspellable-link-destination', 'a link destination shaped like an entity reference decodes on the way back', path) + return failure('unspellable-link-destination', 'a link destination holds an entity reference that decodes on the way back', path) } if (href.includes(' ')) { if (/[<>]/.test(href)) { @@ -22,7 +23,7 @@ export function spellTitle(title: string, path: ConvertErrorPath): Result Gamma~Γ Gammad~Ϝ Gbreve~Ğ Gcedil~Ģ\ + Gcirc~Ĝ Gcy~Г Gdot~Ġ Gfr~𝔊 Gg~⋙ Gopf~𝔾 GreaterEqual~≥ GreaterEqualLess~⋛ GreaterFullEqual~≧ GreaterGreater~⪢\ + GreaterLess~≷ GreaterSlantEqual~⩾ GreaterTilde~≳ Gscr~𝒢 Gt~≫ HARDcy~Ъ Hacek~ˇ Hat~^ Hcirc~Ĥ Hfr~ℌ HilbertSpace~ℋ\ + Hopf~ℍ HorizontalLine~─ Hscr~ℋ Hstrok~Ħ HumpDownHump~≎ HumpEqual~≏ IEcy~Е IJlig~IJ IOcy~Ё Iacute~Í Icirc~Î Icy~И\ + Idot~İ Ifr~ℑ Igrave~Ì Im~ℑ Imacr~Ī ImaginaryI~ⅈ Implies~⇒ Int~∬ Integral~∫ Intersection~⋂ InvisibleComma~⁣\ + InvisibleTimes~⁢ Iogon~Į Iopf~𝕀 Iota~Ι Iscr~ℐ Itilde~Ĩ Iukcy~І Iuml~Ï Jcirc~Ĵ Jcy~Й Jfr~𝔍 Jopf~𝕁 Jscr~𝒥\ + Jsercy~Ј Jukcy~Є KHcy~Х KJcy~Ќ Kappa~Κ Kcedil~Ķ Kcy~К Kfr~𝔎 Kopf~𝕂 Kscr~𝒦 LJcy~Љ LT~< Lacute~Ĺ Lambda~Λ Lang~⟪\ + Laplacetrf~ℒ Larr~↞ Lcaron~Ľ Lcedil~Ļ Lcy~Л LeftAngleBracket~⟨ LeftArrow~← LeftArrowBar~⇤ LeftArrowRightArrow~⇆\ + LeftCeiling~⌈ LeftDoubleBracket~⟦ LeftDownTeeVector~⥡ LeftDownVector~⇃ LeftDownVectorBar~⥙ LeftFloor~⌊\ + LeftRightArrow~↔ LeftRightVector~⥎ LeftTee~⊣ LeftTeeArrow~↤ LeftTeeVector~⥚ LeftTriangle~⊲ LeftTriangleBar~⧏\ + LeftTriangleEqual~⊴ LeftUpDownVector~⥑ LeftUpTeeVector~⥠ LeftUpVector~↿ LeftUpVectorBar~⥘ LeftVector~↼\ + LeftVectorBar~⥒ Leftarrow~⇐ Leftrightarrow~⇔ LessEqualGreater~⋚ LessFullEqual~≦ LessGreater~≶ LessLess~⪡\ + LessSlantEqual~⩽ LessTilde~≲ Lfr~𝔏 Ll~⋘ Lleftarrow~⇚ Lmidot~Ŀ LongLeftArrow~⟵ LongLeftRightArrow~⟷\ + LongRightArrow~⟶ Longleftarrow~⟸ Longleftrightarrow~⟺ Longrightarrow~⟹ Lopf~𝕃 LowerLeftArrow~↙ LowerRightArrow~↘\ + Lscr~ℒ Lsh~↰ Lstrok~Ł Lt~≪ Map~⤅ Mcy~М MediumSpace~  Mellintrf~ℳ Mfr~𝔐 MinusPlus~∓ Mopf~𝕄 Mscr~ℳ Mu~Μ NJcy~Њ\ + Nacute~Ń Ncaron~Ň Ncedil~Ņ Ncy~Н NegativeMediumSpace~​ NegativeThickSpace~​ NegativeThinSpace~​\ + NegativeVeryThinSpace~​ NestedGreaterGreater~≫ NestedLessLess~≪ NewLine~\n Nfr~𝔑 NoBreak~⁠ NonBreakingSpace~ \ + Nopf~ℕ Not~⫬ NotCongruent~≢ NotCupCap~≭ NotDoubleVerticalBar~∦ NotElement~∉ NotEqual~≠ NotEqualTilde~≂̸\ + NotExists~∄ NotGreater~≯ NotGreaterEqual~≱ NotGreaterFullEqual~≧̸ NotGreaterGreater~≫̸ NotGreaterLess~≹\ + NotGreaterSlantEqual~⩾̸ NotGreaterTilde~≵ NotHumpDownHump~≎̸ NotHumpEqual~≏̸ NotLeftTriangle~⋪\ + NotLeftTriangleBar~⧏̸ NotLeftTriangleEqual~⋬ NotLess~≮ NotLessEqual~≰ NotLessGreater~≸ NotLessLess~≪̸\ + NotLessSlantEqual~⩽̸ NotLessTilde~≴ NotNestedGreaterGreater~⪢̸ NotNestedLessLess~⪡̸ NotPrecedes~⊀\ + NotPrecedesEqual~⪯̸ NotPrecedesSlantEqual~⋠ NotReverseElement~∌ NotRightTriangle~⋫ NotRightTriangleBar~⧐̸\ + NotRightTriangleEqual~⋭ NotSquareSubset~⊏̸ NotSquareSubsetEqual~⋢ NotSquareSuperset~⊐̸ NotSquareSupersetEqual~⋣\ + NotSubset~⊂⃒ NotSubsetEqual~⊈ NotSucceeds~⊁ NotSucceedsEqual~⪰̸ NotSucceedsSlantEqual~⋡ NotSucceedsTilde~≿̸\ + NotSuperset~⊃⃒ NotSupersetEqual~⊉ NotTilde~≁ NotTildeEqual~≄ NotTildeFullEqual~≇ NotTildeTilde~≉ NotVerticalBar~∤\ + Nscr~𝒩 Ntilde~Ñ Nu~Ν OElig~Œ Oacute~Ó Ocirc~Ô Ocy~О Odblac~Ő Ofr~𝔒 Ograve~Ò Omacr~Ō Omega~Ω Omicron~Ο Oopf~𝕆\ + OpenCurlyDoubleQuote~“ OpenCurlyQuote~‘ Or~⩔ Oscr~𝒪 Oslash~Ø Otilde~Õ Otimes~⨷ Ouml~Ö OverBar~‾ OverBrace~⏞\ + OverBracket~⎴ OverParenthesis~⏜ PartialD~∂ Pcy~П Pfr~𝔓 Phi~Φ Pi~Π PlusMinus~± Poincareplane~ℌ Popf~ℙ Pr~⪻\ + Precedes~≺ PrecedesEqual~⪯ PrecedesSlantEqual~≼ PrecedesTilde~≾ Prime~″ Product~∏ Proportion~∷ Proportional~∝\ + Pscr~𝒫 Psi~Ψ QUOT~" Qfr~𝔔 Qopf~ℚ Qscr~𝒬 RBarr~⤐ REG~® Racute~Ŕ Rang~⟫ Rarr~↠ Rarrtl~⤖ Rcaron~Ř Rcedil~Ŗ Rcy~Р\ + Re~ℜ ReverseElement~∋ ReverseEquilibrium~⇋ ReverseUpEquilibrium~⥯ Rfr~ℜ Rho~Ρ RightAngleBracket~⟩ RightArrow~→\ + RightArrowBar~⇥ RightArrowLeftArrow~⇄ RightCeiling~⌉ RightDoubleBracket~⟧ RightDownTeeVector~⥝ RightDownVector~⇂\ + RightDownVectorBar~⥕ RightFloor~⌋ RightTee~⊢ RightTeeArrow~↦ RightTeeVector~⥛ RightTriangle~⊳ RightTriangleBar~⧐\ + RightTriangleEqual~⊵ RightUpDownVector~⥏ RightUpTeeVector~⥜ RightUpVector~↾ RightUpVectorBar~⥔ RightVector~⇀\ + RightVectorBar~⥓ Rightarrow~⇒ Ropf~ℝ RoundImplies~⥰ Rrightarrow~⇛ Rscr~ℛ Rsh~↱ RuleDelayed~⧴ SHCHcy~Щ SHcy~Ш\ + SOFTcy~Ь Sacute~Ś Sc~⪼ Scaron~Š Scedil~Ş Scirc~Ŝ Scy~С Sfr~𝔖 ShortDownArrow~↓ ShortLeftArrow~← ShortRightArrow~→\ + ShortUpArrow~↑ Sigma~Σ SmallCircle~∘ Sopf~𝕊 Sqrt~√ Square~□ SquareIntersection~⊓ SquareSubset~⊏\ + SquareSubsetEqual~⊑ SquareSuperset~⊐ SquareSupersetEqual~⊒ SquareUnion~⊔ Sscr~𝒮 Star~⋆ Sub~⋐ Subset~⋐\ + SubsetEqual~⊆ Succeeds~≻ SucceedsEqual~⪰ SucceedsSlantEqual~≽ SucceedsTilde~≿ SuchThat~∋ Sum~∑ Sup~⋑ Superset~⊃\ + SupersetEqual~⊇ Supset~⋑ THORN~Þ TRADE~™ TSHcy~Ћ TScy~Ц Tab~\t Tau~Τ Tcaron~Ť Tcedil~Ţ Tcy~Т Tfr~𝔗 Therefore~∴\ + Theta~Θ ThickSpace~   ThinSpace~  Tilde~∼ TildeEqual~≃ TildeFullEqual~≅ TildeTilde~≈ Topf~𝕋 TripleDot~⃛ Tscr~𝒯\ + Tstrok~Ŧ Uacute~Ú Uarr~↟ Uarrocir~⥉ Ubrcy~Ў Ubreve~Ŭ Ucirc~Û Ucy~У Udblac~Ű Ufr~𝔘 Ugrave~Ù Umacr~Ū UnderBar~_\ + UnderBrace~⏟ UnderBracket~⎵ UnderParenthesis~⏝ Union~⋃ UnionPlus~⊎ Uogon~Ų Uopf~𝕌 UpArrow~↑ UpArrowBar~⤒\ + UpArrowDownArrow~⇅ UpDownArrow~↕ UpEquilibrium~⥮ UpTee~⊥ UpTeeArrow~↥ Uparrow~⇑ Updownarrow~⇕ UpperLeftArrow~↖\ + UpperRightArrow~↗ Upsi~ϒ Upsilon~Υ Uring~Ů Uscr~𝒰 Utilde~Ũ Uuml~Ü VDash~⊫ Vbar~⫫ Vcy~В Vdash~⊩ Vdashl~⫦ Vee~⋁\ + Verbar~‖ Vert~‖ VerticalBar~∣ VerticalLine~| VerticalSeparator~❘ VerticalTilde~≀ VeryThinSpace~  Vfr~𝔙 Vopf~𝕍\ + Vscr~𝒱 Vvdash~⊪ Wcirc~Ŵ Wedge~⋀ Wfr~𝔚 Wopf~𝕎 Wscr~𝒲 Xfr~𝔛 Xi~Ξ Xopf~𝕏 Xscr~𝒳 YAcy~Я YIcy~Ї YUcy~Ю Yacute~Ý\ + Ycirc~Ŷ Ycy~Ы Yfr~𝔜 Yopf~𝕐 Yscr~𝒴 Yuml~Ÿ ZHcy~Ж Zacute~Ź Zcaron~Ž Zcy~З Zdot~Ż ZeroWidthSpace~​ Zeta~Ζ Zfr~ℨ\ + Zopf~ℤ Zscr~𝒵 aacute~á abreve~ă ac~∾ acE~∾̳ acd~∿ acirc~â acute~´ acy~а aelig~æ af~⁡ afr~𝔞 agrave~à alefsym~ℵ\ + aleph~ℵ alpha~α amacr~ā amalg~⨿ amp~& and~∧ andand~⩕ andd~⩜ andslope~⩘ andv~⩚ ang~∠ ange~⦤ angle~∠ angmsd~∡\ + angmsdaa~⦨ angmsdab~⦩ angmsdac~⦪ angmsdad~⦫ angmsdae~⦬ angmsdaf~⦭ angmsdag~⦮ angmsdah~⦯ angrt~∟ angrtvb~⊾\ + angrtvbd~⦝ angsph~∢ angst~Å angzarr~⍼ aogon~ą aopf~𝕒 ap~≈ apE~⩰ apacir~⩯ ape~≊ apid~≋ apos~\' approx~≈ approxeq~≊\ + aring~å ascr~𝒶 ast~* asymp~≈ asympeq~≍ atilde~ã auml~ä awconint~∳ awint~⨑ bNot~⫭ backcong~≌ backepsilon~϶\ + backprime~‵ backsim~∽ backsimeq~⋍ barvee~⊽ barwed~⌅ barwedge~⌅ bbrk~⎵ bbrktbrk~⎶ bcong~≌ bcy~б bdquo~„ becaus~∵\ + because~∵ bemptyv~⦰ bepsi~϶ bernou~ℬ beta~β beth~ℶ between~≬ bfr~𝔟 bigcap~⋂ bigcirc~◯ bigcup~⋃ bigodot~⨀\ + bigoplus~⨁ bigotimes~⨂ bigsqcup~⨆ bigstar~★ bigtriangledown~▽ bigtriangleup~△ biguplus~⨄ bigvee~⋁ bigwedge~⋀\ + bkarow~⤍ blacklozenge~⧫ blacksquare~▪ blacktriangle~▴ blacktriangledown~▾ blacktriangleleft~◂ blacktriangleright~▸\ + blank~␣ blk12~▒ blk14~░ blk34~▓ block~█ bne~=⃥ bnequiv~≡⃥ bnot~⌐ bopf~𝕓 bot~⊥ bottom~⊥ bowtie~⋈ boxDL~╗ boxDR~╔\ + boxDl~╖ boxDr~╓ boxH~═ boxHD~╦ boxHU~╩ boxHd~╤ boxHu~╧ boxUL~╝ boxUR~╚ boxUl~╜ boxUr~╙ boxV~║ boxVH~╬ boxVL~╣\ + boxVR~╠ boxVh~╫ boxVl~╢ boxVr~╟ boxbox~⧉ boxdL~╕ boxdR~╒ boxdl~┐ boxdr~┌ boxh~─ boxhD~╥ boxhU~╨ boxhd~┬ boxhu~┴\ + boxminus~⊟ boxplus~⊞ boxtimes~⊠ boxuL~╛ boxuR~╘ boxul~┘ boxur~└ boxv~│ boxvH~╪ boxvL~╡ boxvR~╞ boxvh~┼ boxvl~┤\ + boxvr~├ bprime~‵ breve~˘ brvbar~¦ bscr~𝒷 bsemi~⁏ bsim~∽ bsime~⋍ bsol~\\ bsolb~⧅ bsolhsub~⟈ bull~• bullet~• bump~≎\ + bumpE~⪮ bumpe~≏ bumpeq~≏ cacute~ć cap~∩ capand~⩄ capbrcup~⩉ capcap~⩋ capcup~⩇ capdot~⩀ caps~∩︀ caret~⁁ caron~ˇ\ + ccaps~⩍ ccaron~č ccedil~ç ccirc~ĉ ccups~⩌ ccupssm~⩐ cdot~ċ cedil~¸ cemptyv~⦲ cent~¢ centerdot~· cfr~𝔠 chcy~ч\ + check~✓ checkmark~✓ chi~χ cir~○ cirE~⧃ circ~ˆ circeq~≗ circlearrowleft~↺ circlearrowright~↻ circledR~® circledS~Ⓢ\ + circledast~⊛ circledcirc~⊚ circleddash~⊝ cire~≗ cirfnint~⨐ cirmid~⫯ cirscir~⧂ clubs~♣ clubsuit~♣ colon~: colone~≔\ + coloneq~≔ comma~, commat~@ comp~∁ compfn~∘ complement~∁ complexes~ℂ cong~≅ congdot~⩭ conint~∮ copf~𝕔 coprod~∐\ + copy~© copysr~℗ crarr~↵ cross~✗ cscr~𝒸 csub~⫏ csube~⫑ csup~⫐ csupe~⫒ ctdot~⋯ cudarrl~⤸ cudarrr~⤵ cuepr~⋞ cuesc~⋟\ + cularr~↶ cularrp~⤽ cup~∪ cupbrcap~⩈ cupcap~⩆ cupcup~⩊ cupdot~⊍ cupor~⩅ cups~∪︀ curarr~↷ curarrm~⤼ curlyeqprec~⋞\ + curlyeqsucc~⋟ curlyvee~⋎ curlywedge~⋏ curren~¤ curvearrowleft~↶ curvearrowright~↷ cuvee~⋎ cuwed~⋏ cwconint~∲\ + cwint~∱ cylcty~⌭ dArr~⇓ dHar~⥥ dagger~† daleth~ℸ darr~↓ dash~‐ dashv~⊣ dbkarow~⤏ dblac~˝ dcaron~ď dcy~д dd~ⅆ\ + ddagger~‡ ddarr~⇊ ddotseq~⩷ deg~° delta~δ demptyv~⦱ dfisht~⥿ dfr~𝔡 dharl~⇃ dharr~⇂ diam~⋄ diamond~⋄ diamondsuit~♦\ + diams~♦ die~¨ digamma~ϝ disin~⋲ div~÷ divide~÷ divideontimes~⋇ divonx~⋇ djcy~ђ dlcorn~⌞ dlcrop~⌍ dollar~$ dopf~𝕕\ + dot~˙ doteq~≐ doteqdot~≑ dotminus~∸ dotplus~∔ dotsquare~⊡ doublebarwedge~⌆ downarrow~↓ downdownarrows~⇊\ + downharpoonleft~⇃ downharpoonright~⇂ drbkarow~⤐ drcorn~⌟ drcrop~⌌ dscr~𝒹 dscy~ѕ dsol~⧶ dstrok~đ dtdot~⋱ dtri~▿\ + dtrif~▾ duarr~⇵ duhar~⥯ dwangle~⦦ dzcy~џ dzigrarr~⟿ eDDot~⩷ eDot~≑ eacute~é easter~⩮ ecaron~ě ecir~≖ ecirc~ê\ + ecolon~≕ ecy~э edot~ė ee~ⅇ efDot~≒ efr~𝔢 eg~⪚ egrave~è egs~⪖ egsdot~⪘ el~⪙ elinters~⏧ ell~ℓ els~⪕ elsdot~⪗\ + emacr~ē empty~∅ emptyset~∅ emptyv~∅ emsp~  emsp13~  emsp14~  eng~ŋ ensp~  eogon~ę eopf~𝕖 epar~⋕ eparsl~⧣ eplus~⩱\ + epsi~ε epsilon~ε epsiv~ϵ eqcirc~≖ eqcolon~≕ eqsim~≂ eqslantgtr~⪖ eqslantless~⪕ equals~= equest~≟ equiv~≡ equivDD~⩸\ + eqvparsl~⧥ erDot~≓ erarr~⥱ escr~ℯ esdot~≐ esim~≂ eta~η eth~ð euml~ë euro~€ excl~! exist~∃ expectation~ℰ\ + exponentiale~ⅇ fallingdotseq~≒ fcy~ф female~♀ ffilig~ffi fflig~ff ffllig~ffl ffr~𝔣 filig~fi fjlig~fj flat~♭ fllig~fl\ + fltns~▱ fnof~ƒ fopf~𝕗 forall~∀ fork~⋔ forkv~⫙ fpartint~⨍ frac12~½ frac13~⅓ frac14~¼ frac15~⅕ frac16~⅙ frac18~⅛\ + frac23~⅔ frac25~⅖ frac34~¾ frac35~⅗ frac38~⅜ frac45~⅘ frac56~⅚ frac58~⅝ frac78~⅞ frasl~⁄ frown~⌢ fscr~𝒻 gE~≧\ + gEl~⪌ gacute~ǵ gamma~γ gammad~ϝ gap~⪆ gbreve~ğ gcirc~ĝ gcy~г gdot~ġ ge~≥ gel~⋛ geq~≥ geqq~≧ geqslant~⩾ ges~⩾\ + gescc~⪩ gesdot~⪀ gesdoto~⪂ gesdotol~⪄ gesl~⋛︀ gesles~⪔ gfr~𝔤 gg~≫ ggg~⋙ gimel~ℷ gjcy~ѓ gl~≷ glE~⪒ gla~⪥ glj~⪤\ + gnE~≩ gnap~⪊ gnapprox~⪊ gne~⪈ gneq~⪈ gneqq~≩ gnsim~⋧ gopf~𝕘 grave~` gscr~ℊ gsim~≳ gsime~⪎ gsiml~⪐ gt~> gtcc~⪧\ + gtcir~⩺ gtdot~⋗ gtlPar~⦕ gtquest~⩼ gtrapprox~⪆ gtrarr~⥸ gtrdot~⋗ gtreqless~⋛ gtreqqless~⪌ gtrless~≷ gtrsim~≳\ + gvertneqq~≩︀ gvnE~≩︀ hArr~⇔ hairsp~  half~½ hamilt~ℋ hardcy~ъ harr~↔ harrcir~⥈ harrw~↭ hbar~ℏ hcirc~ĥ hearts~♥\ + heartsuit~♥ hellip~… hercon~⊹ hfr~𝔥 hksearow~⤥ hkswarow~⤦ hoarr~⇿ homtht~∻ hookleftarrow~↩ hookrightarrow~↪\ + hopf~𝕙 horbar~― hscr~𝒽 hslash~ℏ hstrok~ħ hybull~⁃ hyphen~‐ iacute~í ic~⁣ icirc~î icy~и iecy~е iexcl~¡ iff~⇔\ + ifr~𝔦 igrave~ì ii~ⅈ iiiint~⨌ iiint~∭ iinfin~⧜ iiota~℩ ijlig~ij imacr~ī image~ℑ imagline~ℐ imagpart~ℑ imath~ı\ + imof~⊷ imped~Ƶ in~∈ incare~℅ infin~∞ infintie~⧝ inodot~ı int~∫ intcal~⊺ integers~ℤ intercal~⊺ intlarhk~⨗ intprod~⨼\ + iocy~ё iogon~į iopf~𝕚 iota~ι iprod~⨼ iquest~¿ iscr~𝒾 isin~∈ isinE~⋹ isindot~⋵ isins~⋴ isinsv~⋳ isinv~∈ it~⁢\ + itilde~ĩ iukcy~і iuml~ï jcirc~ĵ jcy~й jfr~𝔧 jmath~ȷ jopf~𝕛 jscr~𝒿 jsercy~ј jukcy~є kappa~κ kappav~ϰ kcedil~ķ\ + kcy~к kfr~𝔨 kgreen~ĸ khcy~х kjcy~ќ kopf~𝕜 kscr~𝓀 lAarr~⇚ lArr~⇐ lAtail~⤛ lBarr~⤎ lE~≦ lEg~⪋ lHar~⥢ lacute~ĺ\ + laemptyv~⦴ lagran~ℒ lambda~λ lang~⟨ langd~⦑ langle~⟨ lap~⪅ laquo~« larr~← larrb~⇤ larrbfs~⤟ larrfs~⤝ larrhk~↩\ + larrlp~↫ larrpl~⤹ larrsim~⥳ larrtl~↢ lat~⪫ latail~⤙ late~⪭ lates~⪭︀ lbarr~⤌ lbbrk~❲ lbrace~{ lbrack~[ lbrke~⦋\ + lbrksld~⦏ lbrkslu~⦍ lcaron~ľ lcedil~ļ lceil~⌈ lcub~{ lcy~л ldca~⤶ ldquo~“ ldquor~„ ldrdhar~⥧ ldrushar~⥋ ldsh~↲\ + le~≤ leftarrow~← leftarrowtail~↢ leftharpoondown~↽ leftharpoonup~↼ leftleftarrows~⇇ leftrightarrow~↔\ + leftrightarrows~⇆ leftrightharpoons~⇋ leftrightsquigarrow~↭ leftthreetimes~⋋ leg~⋚ leq~≤ leqq~≦ leqslant~⩽ les~⩽\ + lescc~⪨ lesdot~⩿ lesdoto~⪁ lesdotor~⪃ lesg~⋚︀ lesges~⪓ lessapprox~⪅ lessdot~⋖ lesseqgtr~⋚ lesseqqgtr~⪋ lessgtr~≶\ + lesssim~≲ lfisht~⥼ lfloor~⌊ lfr~𝔩 lg~≶ lgE~⪑ lhard~↽ lharu~↼ lharul~⥪ lhblk~▄ ljcy~љ ll~≪ llarr~⇇ llcorner~⌞\ + llhard~⥫ lltri~◺ lmidot~ŀ lmoust~⎰ lmoustache~⎰ lnE~≨ lnap~⪉ lnapprox~⪉ lne~⪇ lneq~⪇ lneqq~≨ lnsim~⋦ loang~⟬\ + loarr~⇽ lobrk~⟦ longleftarrow~⟵ longleftrightarrow~⟷ longmapsto~⟼ longrightarrow~⟶ looparrowleft~↫\ + looparrowright~↬ lopar~⦅ lopf~𝕝 loplus~⨭ lotimes~⨴ lowast~∗ lowbar~_ loz~◊ lozenge~◊ lozf~⧫ lpar~( lparlt~⦓\ + lrarr~⇆ lrcorner~⌟ lrhar~⇋ lrhard~⥭ lrm~‎ lrtri~⊿ lsaquo~‹ lscr~𝓁 lsh~↰ lsim~≲ lsime~⪍ lsimg~⪏ lsqb~[ lsquo~‘\ + lsquor~‚ lstrok~ł lt~< ltcc~⪦ ltcir~⩹ ltdot~⋖ lthree~⋋ ltimes~⋉ ltlarr~⥶ ltquest~⩻ ltrPar~⦖ ltri~◃ ltrie~⊴ ltrif~◂\ + lurdshar~⥊ luruhar~⥦ lvertneqq~≨︀ lvnE~≨︀ mDDot~∺ macr~¯ male~♂ malt~✠ maltese~✠ map~↦ mapsto~↦ mapstodown~↧\ + mapstoleft~↤ mapstoup~↥ marker~▮ mcomma~⨩ mcy~м mdash~— measuredangle~∡ mfr~𝔪 mho~℧ micro~µ mid~∣ midast~*\ + midcir~⫰ middot~· minus~− minusb~⊟ minusd~∸ minusdu~⨪ mlcp~⫛ mldr~… mnplus~∓ models~⊧ mopf~𝕞 mp~∓ mscr~𝓂\ + mstpos~∾ mu~μ multimap~⊸ mumap~⊸ nGg~⋙̸ nGt~≫⃒ nGtv~≫̸ nLeftarrow~⇍ nLeftrightarrow~⇎ nLl~⋘̸ nLt~≪⃒ nLtv~≪̸\ + nRightarrow~⇏ nVDash~⊯ nVdash~⊮ nabla~∇ nacute~ń nang~∠⃒ nap~≉ napE~⩰̸ napid~≋̸ napos~ʼn napprox~≉ natur~♮\ + natural~♮ naturals~ℕ nbsp~  nbump~≎̸ nbumpe~≏̸ ncap~⩃ ncaron~ň ncedil~ņ ncong~≇ ncongdot~⩭̸ ncup~⩂ ncy~н ndash~–\ + ne~≠ neArr~⇗ nearhk~⤤ nearr~↗ nearrow~↗ nedot~≐̸ nequiv~≢ nesear~⤨ nesim~≂̸ nexist~∄ nexists~∄ nfr~𝔫 ngE~≧̸ nge~≱\ + ngeq~≱ ngeqq~≧̸ ngeqslant~⩾̸ nges~⩾̸ ngsim~≵ ngt~≯ ngtr~≯ nhArr~⇎ nharr~↮ nhpar~⫲ ni~∋ nis~⋼ nisd~⋺ niv~∋ njcy~њ\ + nlArr~⇍ nlE~≦̸ nlarr~↚ nldr~‥ nle~≰ nleftarrow~↚ nleftrightarrow~↮ nleq~≰ nleqq~≦̸ nleqslant~⩽̸ nles~⩽̸ nless~≮\ + nlsim~≴ nlt~≮ nltri~⋪ nltrie~⋬ nmid~∤ nopf~𝕟 not~¬ notin~∉ notinE~⋹̸ notindot~⋵̸ notinva~∉ notinvb~⋷ notinvc~⋶\ + notni~∌ notniva~∌ notnivb~⋾ notnivc~⋽ npar~∦ nparallel~∦ nparsl~⫽⃥ npart~∂̸ npolint~⨔ npr~⊀ nprcue~⋠ npre~⪯̸\ + nprec~⊀ npreceq~⪯̸ nrArr~⇏ nrarr~↛ nrarrc~⤳̸ nrarrw~↝̸ nrightarrow~↛ nrtri~⋫ nrtrie~⋭ nsc~⊁ nsccue~⋡ nsce~⪰̸\ + nscr~𝓃 nshortmid~∤ nshortparallel~∦ nsim~≁ nsime~≄ nsimeq~≄ nsmid~∤ nspar~∦ nsqsube~⋢ nsqsupe~⋣ nsub~⊄ nsubE~⫅̸\ + nsube~⊈ nsubset~⊂⃒ nsubseteq~⊈ nsubseteqq~⫅̸ nsucc~⊁ nsucceq~⪰̸ nsup~⊅ nsupE~⫆̸ nsupe~⊉ nsupset~⊃⃒ nsupseteq~⊉\ + nsupseteqq~⫆̸ ntgl~≹ ntilde~ñ ntlg~≸ ntriangleleft~⋪ ntrianglelefteq~⋬ ntriangleright~⋫ ntrianglerighteq~⋭ nu~ν\ + num~# numero~№ numsp~  nvDash~⊭ nvHarr~⤄ nvap~≍⃒ nvdash~⊬ nvge~≥⃒ nvgt~>⃒ nvinfin~⧞ nvlArr~⤂ nvle~≤⃒ nvlt~<⃒\ + nvltrie~⊴⃒ nvrArr~⤃ nvrtrie~⊵⃒ nvsim~∼⃒ nwArr~⇖ nwarhk~⤣ nwarr~↖ nwarrow~↖ nwnear~⤧ oS~Ⓢ oacute~ó oast~⊛ ocir~⊚\ + ocirc~ô ocy~о odash~⊝ odblac~ő odiv~⨸ odot~⊙ odsold~⦼ oelig~œ ofcir~⦿ ofr~𝔬 ogon~˛ ograve~ò ogt~⧁ ohbar~⦵ ohm~Ω\ + oint~∮ olarr~↺ olcir~⦾ olcross~⦻ oline~‾ olt~⧀ omacr~ō omega~ω omicron~ο omid~⦶ ominus~⊖ oopf~𝕠 opar~⦷ operp~⦹\ + oplus~⊕ or~∨ orarr~↻ ord~⩝ order~ℴ orderof~ℴ ordf~ª ordm~º origof~⊶ oror~⩖ orslope~⩗ orv~⩛ oscr~ℴ oslash~ø osol~⊘\ + otilde~õ otimes~⊗ otimesas~⨶ ouml~ö ovbar~⌽ par~∥ para~¶ parallel~∥ parsim~⫳ parsl~⫽ part~∂ pcy~п percnt~%\ + period~. permil~‰ perp~⊥ pertenk~‱ pfr~𝔭 phi~φ phiv~ϕ phmmat~ℳ phone~☎ pi~π pitchfork~⋔ piv~ϖ planck~ℏ planckh~ℎ\ + plankv~ℏ plus~+ plusacir~⨣ plusb~⊞ pluscir~⨢ plusdo~∔ plusdu~⨥ pluse~⩲ plusmn~± plussim~⨦ plustwo~⨧ pm~±\ + pointint~⨕ popf~𝕡 pound~£ pr~≺ prE~⪳ prap~⪷ prcue~≼ pre~⪯ prec~≺ precapprox~⪷ preccurlyeq~≼ preceq~⪯\ + precnapprox~⪹ precneqq~⪵ precnsim~⋨ precsim~≾ prime~′ primes~ℙ prnE~⪵ prnap~⪹ prnsim~⋨ prod~∏ profalar~⌮\ + profline~⌒ profsurf~⌓ prop~∝ propto~∝ prsim~≾ prurel~⊰ pscr~𝓅 psi~ψ puncsp~  qfr~𝔮 qint~⨌ qopf~𝕢 qprime~⁗\ + qscr~𝓆 quaternions~ℍ quatint~⨖ quest~? questeq~≟ quot~" rAarr~⇛ rArr~⇒ rAtail~⤜ rBarr~⤏ rHar~⥤ race~∽̱ racute~ŕ\ + radic~√ raemptyv~⦳ rang~⟩ rangd~⦒ range~⦥ rangle~⟩ raquo~» rarr~→ rarrap~⥵ rarrb~⇥ rarrbfs~⤠ rarrc~⤳ rarrfs~⤞\ + rarrhk~↪ rarrlp~↬ rarrpl~⥅ rarrsim~⥴ rarrtl~↣ rarrw~↝ ratail~⤚ ratio~∶ rationals~ℚ rbarr~⤍ rbbrk~❳ rbrace~}\ + rbrack~] rbrke~⦌ rbrksld~⦎ rbrkslu~⦐ rcaron~ř rcedil~ŗ rceil~⌉ rcub~} rcy~р rdca~⤷ rdldhar~⥩ rdquo~” rdquor~”\ + rdsh~↳ real~ℜ realine~ℛ realpart~ℜ reals~ℝ rect~▭ reg~® rfisht~⥽ rfloor~⌋ rfr~𝔯 rhard~⇁ rharu~⇀ rharul~⥬ rho~ρ\ + rhov~ϱ rightarrow~→ rightarrowtail~↣ rightharpoondown~⇁ rightharpoonup~⇀ rightleftarrows~⇄ rightleftharpoons~⇌\ + rightrightarrows~⇉ rightsquigarrow~↝ rightthreetimes~⋌ ring~˚ risingdotseq~≓ rlarr~⇄ rlhar~⇌ rlm~‏ rmoust~⎱\ + rmoustache~⎱ rnmid~⫮ roang~⟭ roarr~⇾ robrk~⟧ ropar~⦆ ropf~𝕣 roplus~⨮ rotimes~⨵ rpar~) rpargt~⦔ rppolint~⨒ rrarr~⇉\ + rsaquo~› rscr~𝓇 rsh~↱ rsqb~] rsquo~’ rsquor~’ rthree~⋌ rtimes~⋊ rtri~▹ rtrie~⊵ rtrif~▸ rtriltri~⧎ ruluhar~⥨ rx~℞\ + sacute~ś sbquo~‚ sc~≻ scE~⪴ scap~⪸ scaron~š sccue~≽ sce~⪰ scedil~ş scirc~ŝ scnE~⪶ scnap~⪺ scnsim~⋩ scpolint~⨓\ + scsim~≿ scy~с sdot~⋅ sdotb~⊡ sdote~⩦ seArr~⇘ searhk~⤥ searr~↘ searrow~↘ sect~§ semi~; seswar~⤩ setminus~∖ setmn~∖\ + sext~✶ sfr~𝔰 sfrown~⌢ sharp~♯ shchcy~щ shcy~ш shortmid~∣ shortparallel~∥ shy~­ sigma~σ sigmaf~ς sigmav~ς sim~∼\ + simdot~⩪ sime~≃ simeq~≃ simg~⪞ simgE~⪠ siml~⪝ simlE~⪟ simne~≆ simplus~⨤ simrarr~⥲ slarr~← smallsetminus~∖ smashp~⨳\ + smeparsl~⧤ smid~∣ smile~⌣ smt~⪪ smte~⪬ smtes~⪬︀ softcy~ь sol~/ solb~⧄ solbar~⌿ sopf~𝕤 spades~♠ spadesuit~♠ spar~∥\ + sqcap~⊓ sqcaps~⊓︀ sqcup~⊔ sqcups~⊔︀ sqsub~⊏ sqsube~⊑ sqsubset~⊏ sqsubseteq~⊑ sqsup~⊐ sqsupe~⊒ sqsupset~⊐\ + sqsupseteq~⊒ squ~□ square~□ squarf~▪ squf~▪ srarr~→ sscr~𝓈 ssetmn~∖ ssmile~⌣ sstarf~⋆ star~☆ starf~★\ + straightepsilon~ϵ straightphi~ϕ strns~¯ sub~⊂ subE~⫅ subdot~⪽ sube~⊆ subedot~⫃ submult~⫁ subnE~⫋ subne~⊊ subplus~⪿\ + subrarr~⥹ subset~⊂ subseteq~⊆ subseteqq~⫅ subsetneq~⊊ subsetneqq~⫋ subsim~⫇ subsub~⫕ subsup~⫓ succ~≻ succapprox~⪸\ + succcurlyeq~≽ succeq~⪰ succnapprox~⪺ succneqq~⪶ succnsim~⋩ succsim~≿ sum~∑ sung~♪ sup~⊃ sup1~¹ sup2~² sup3~³\ + supE~⫆ supdot~⪾ supdsub~⫘ supe~⊇ supedot~⫄ suphsol~⟉ suphsub~⫗ suplarr~⥻ supmult~⫂ supnE~⫌ supne~⊋ supplus~⫀\ + supset~⊃ supseteq~⊇ supseteqq~⫆ supsetneq~⊋ supsetneqq~⫌ supsim~⫈ supsub~⫔ supsup~⫖ swArr~⇙ swarhk~⤦ swarr~↙\ + swarrow~↙ swnwar~⤪ szlig~ß target~⌖ tau~τ tbrk~⎴ tcaron~ť tcedil~ţ tcy~т tdot~⃛ telrec~⌕ tfr~𝔱 there4~∴\ + therefore~∴ theta~θ thetasym~ϑ thetav~ϑ thickapprox~≈ thicksim~∼ thinsp~  thkap~≈ thksim~∼ thorn~þ tilde~˜ times~×\ + timesb~⊠ timesbar~⨱ timesd~⨰ tint~∭ toea~⤨ top~⊤ topbot~⌶ topcir~⫱ topf~𝕥 topfork~⫚ tosa~⤩ tprime~‴ trade~™\ + triangle~▵ triangledown~▿ triangleleft~◃ trianglelefteq~⊴ triangleq~≜ triangleright~▹ trianglerighteq~⊵ tridot~◬\ + trie~≜ triminus~⨺ triplus~⨹ trisb~⧍ tritime~⨻ trpezium~⏢ tscr~𝓉 tscy~ц tshcy~ћ tstrok~ŧ twixt~≬\ + twoheadleftarrow~↞ twoheadrightarrow~↠ uArr~⇑ uHar~⥣ uacute~ú uarr~↑ ubrcy~ў ubreve~ŭ ucirc~û ucy~у udarr~⇅\ + udblac~ű udhar~⥮ ufisht~⥾ ufr~𝔲 ugrave~ù uharl~↿ uharr~↾ uhblk~▀ ulcorn~⌜ ulcorner~⌜ ulcrop~⌏ ultri~◸ umacr~ū\ + uml~¨ uogon~ų uopf~𝕦 uparrow~↑ updownarrow~↕ upharpoonleft~↿ upharpoonright~↾ uplus~⊎ upsi~υ upsih~ϒ upsilon~υ\ + upuparrows~⇈ urcorn~⌝ urcorner~⌝ urcrop~⌎ uring~ů urtri~◹ uscr~𝓊 utdot~⋰ utilde~ũ utri~▵ utrif~▴ uuarr~⇈ uuml~ü\ + uwangle~⦧ vArr~⇕ vBar~⫨ vBarv~⫩ vDash~⊨ vangrt~⦜ varepsilon~ϵ varkappa~ϰ varnothing~∅ varphi~ϕ varpi~ϖ varpropto~∝\ + varr~↕ varrho~ϱ varsigma~ς varsubsetneq~⊊︀ varsubsetneqq~⫋︀ varsupsetneq~⊋︀ varsupsetneqq~⫌︀ vartheta~ϑ\ + vartriangleleft~⊲ vartriangleright~⊳ vcy~в vdash~⊢ vee~∨ veebar~⊻ veeeq~≚ vellip~⋮ verbar~| vert~| vfr~𝔳 vltri~⊲\ + vnsub~⊂⃒ vnsup~⊃⃒ vopf~𝕧 vprop~∝ vrtri~⊳ vscr~𝓋 vsubnE~⫋︀ vsubne~⊊︀ vsupnE~⫌︀ vsupne~⊋︀ vzigzag~⦚ wcirc~ŵ\ + wedbar~⩟ wedge~∧ wedgeq~≙ weierp~℘ wfr~𝔴 wopf~𝕨 wp~℘ wr~≀ wreath~≀ wscr~𝓌 xcap~⋂ xcirc~◯ xcup~⋃ xdtri~▽ xfr~𝔵\ + xhArr~⟺ xharr~⟷ xi~ξ xlArr~⟸ xlarr~⟵ xmap~⟼ xnis~⋻ xodot~⨀ xopf~𝕩 xoplus~⨁ xotime~⨂ xrArr~⟹ xrarr~⟶ xscr~𝓍\ + xsqcup~⨆ xuplus~⨄ xutri~△ xvee~⋁ xwedge~⋀ yacute~ý yacy~я ycirc~ŷ ycy~ы yen~¥ yfr~𝔶 yicy~ї yopf~𝕪 yscr~𝓎 yucy~ю\ + yuml~ÿ zacute~ź zcaron~ž zcy~з zdot~ż zeetrf~ℨ zeta~ζ zfr~𝔷 zhcy~ж zigrarr~⇝ zopf~𝕫 zscr~𝓏 zwj~‍ zwnj~‌' + +const namedReferences = new Map(packedReferences.split(/ +/).map(namedReference)) + +export function holdsEntityReference(text: string): boolean { + for (let index = text.indexOf('&'); index !== -1; index = text.indexOf('&', index + 1)) { + if (readEntityReference(text.slice(index)) !== undefined) return true + } + return false +} + +// The reference text opens with, `undefined` where it opens with none. +export function readEntityReference(text: string): EntityReference | undefined { + const reference = anchoredEntityReference.exec(text)?.[0] + if (reference === undefined) return undefined + const decoded = decodeReference(reference) + return decoded === undefined ? undefined : { length: reference.length, text: decoded } +} + +function decodeReference(reference: string): string | undefined { + const decimal = decimalReference.exec(reference)?.[1] + if (decimal !== undefined) return characterOf(Number.parseInt(decimal, 10)) + const hexadecimal = hexadecimalReference.exec(reference)?.[1] + if (hexadecimal !== undefined) return characterOf(Number.parseInt(hexadecimal, 16)) + return namedReferences.get(reference.slice(1, -1)) +} + +function characterOf(codePoint: number): string { + if (codePoint === 0 || codePoint > largestCodePoint) return replacementCharacter + if (codePoint >= surrogates.first && codePoint <= surrogates.last) return replacementCharacter + return String.fromCodePoint(codePoint) +} + +function namedReference(entry: string): [string, string] { + const separator = entry.indexOf('~') + return [entry.slice(0, separator), entry.slice(separator + 1)] +} diff --git a/src/markdown/parse/blocks.ts b/src/markdown/parse/blocks.ts index 616bc1e..34e54f5 100644 --- a/src/markdown/parse/blocks.ts +++ b/src/markdown/parse/blocks.ts @@ -4,6 +4,7 @@ import { claimsDirectiveLine, claimsPipeLine, closingCodeFence, + decodeTextEscapes, isThematicBreak, listMarker, markerInterruptsParagraph, @@ -281,7 +282,7 @@ function closeLeaf(walk: Walk): void { } walk.leaf = undefined if (leaf.kind === 'html') currentBlocks(walk).push({ construct: leaf.construct, kind: 'html' }) - else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? leaf.info : '', text: leaf.lines.join('\n') }) + else currentBlocks(walk).push({ kind: 'code', language: leaf.kind === 'fenced-code' ? decodeTextEscapes(leaf.info) : '', text: leaf.lines.join('\n') }) } function takeParagraph(walk: Walk): string | undefined { diff --git a/src/markdown/parse/html-blocks.ts b/src/markdown/parse/html-blocks.ts index ad5ef61..80db4a6 100644 --- a/src/markdown/parse/html-blocks.ts +++ b/src/markdown/parse/html-blocks.ts @@ -1,3 +1,5 @@ +import { htmlConstructNames, htmlTagName, htmlTagSource } from '../commonmark-grammar.ts' + export type OpenHtmlBlock = { closer: RegExp | undefined; construct: string } type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | undefined; interrupts: boolean; start: RegExp } @@ -5,16 +7,14 @@ type HtmlBlockCondition = { closer: RegExp | undefined; construct: string | unde // CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. const blockTagNames = 'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul' -const attributeSource = '(?:[ \\t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \\t]*=[ \\t]*(?:[^ \\t"\'=<>`]+|\'[^\']*\'|"[^"]*"))?)' -const completeTag = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*${attributeSource}*[ \\t]*/?>|)[ \\t]*$`) -const tagName = /^<\/?([A-Za-z][A-Za-z0-9-]*).*$/ +const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`) const conditions: HtmlBlockCondition[] = [ { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i }, - { closer: /-->/, construct: 'an HTML comment', interrupts: true, start: /^/, construct: htmlConstructNames.comment, interrupts: true, start: /^ here.\n')), 'unmappable-html: no ADF node carries an HTML comment') + assert.equal(content(markdownToAdf('Part here.\n')), 'unmappable-html: no ADF node carries an HTML processing instruction') + assert.equal(content(markdownToAdf('Part here.\n')), 'unmappable-html: no ADF node carries an HTML declaration') + assert.equal(content(markdownToAdf('Part here.\n')), 'unmappable-html: no ADF node carries a CDATA section') + assert.equal(content(markdownToAdf('Part here.\n')), 'unmappable-html: no ADF node carries an HTML comment') + assert.equal(code(markdownToAdf('A b\n')), 'unmappable-html') + assert.equal(code(markdownToAdf('Part.\n\n')), 'unmappable-html') + assert.deepEqual(path(markdownToAdf('Part.\n\nA b.\n')), ['content', 1]) +}) + +test('leaves the angle bracket that opens no HTML construct to the text it sits in', () => { + assert.deepEqual(content(markdownToAdf('3 < 4 and 5 d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('a d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('``\n')), [{ content: [codeSpan('')], type: 'paragraph' }]) + assert.deepEqual(content(markdownToAdf('\\\n')), [paragraph('')]) +}) diff --git a/src/markdown/parse/markdown-to-adf.ts b/src/markdown/parse/markdown-to-adf.ts index b3d4dcf..fc2c1ed 100644 --- a/src/markdown/parse/markdown-to-adf.ts +++ b/src/markdown/parse/markdown-to-adf.ts @@ -3,7 +3,7 @@ import type { Block, ClaimedConstruct } from './blocks.ts' import { failure, success, type ConvertErrorPath, type Result } from '../../result.ts' import { largestNesting } from '../../nesting.ts' import { parseBlocks } from './blocks.ts' -import { trimSpace } from '../commonmark-grammar.ts' +import { parseInlineContent } from './inline-content.ts' export function markdownToAdf(markdown: string): Result { const content = blockNodes(parseBlocks(markdown).blocks, [], 0) @@ -33,13 +33,13 @@ function blockNode(block: Block, path: ConvertErrorPath, depth: number): Result< case 'code': return success(codeBlockNode(block.language, block.text)) case 'heading': - return success(withContent({ attrs: { level: block.level }, type: 'heading' }, block.text)) + return contentNode({ attrs: { level: block.level }, type: 'heading' }, block.text, path) case 'html': return failure('unmappable-html', `no ADF node carries ${block.construct}`, path) case 'orderedList': return listNode({ attrs: { order: block.start }, type: 'orderedList' }, block.items, path, depth) case 'paragraph': - return success(withContent({ type: 'paragraph' }, block.text)) + return contentNode({ type: 'paragraph' }, block.text, path) case 'rule': return success({ type: 'rule' }) } @@ -75,15 +75,8 @@ function codeBlockNode(language: string, text: string): AdfNode { return text === '' ? node : { ...node, content: [{ text, type: 'text' }] } } -function withContent(node: AdfNode, text: string): AdfNode { - const content = inlineContent(text) - return content.length === 0 ? node : { ...node, content } -} - -function inlineContent(text: string): AdfNode[] { - const line = text - .split('\n') - .map((part) => trimSpace(part)) - .join(' ') - return line === '' ? [] : [{ text: line, type: 'text' }] +function contentNode(node: AdfNode, text: string, path: ConvertErrorPath): Result { + const content = parseInlineContent(text, path) + if (!content.ok) return content + return success(content.value.length === 0 ? node : { ...node, content: content.value }) } diff --git a/todo.md b/todo.md index ae732dc..35924d7 100644 --- a/todo.md +++ b/todo.md @@ -226,13 +226,20 @@ detail is settled at its own milestone. into the separation it names, and `spec/flavour.md`'s "none between a nested list and a CommonMark block above it" gaining that exception. Every fixture spelled tight today keeps its bytes, and `nested-list-tight` becomes the round-trip pair `nested-list-separation`. - - [ ] **3d — Inline text.** The inline scanner over a block's content: backslash escapes, entity + - [x] **3d — Inline text.** The inline scanner over a block's content: backslash escapes, entity references decoding to their characters, code spans and the literal they hold — directive syntax and `~~` included — CommonMark's own hard breaks, a trailing backslash and two trailing spaces alike, a soft line break as one space, the fenced info string's own decoding the block walk leaves raw, and the raw inline tag, comment and processing instruction refused by name, recognized by the `commonmark-grammar.ts` predicates the emitter already escapes against, under 3b's one-table rule. + **Settled** (the maintainer, 2026-08-30): entity references decode against HTML5's whole + named table, checked in packed (§5) — a curated subset leaves 3k an exception class and a + cutoff line nobody can defend. And the escape superset the emitter reads for raw HTML + tightens into one precise CommonMark inline reader both directions share, the email + autolink parting off as 3e's own predicate: refusing on the superset would refuse + `1 Date: Sun, 30 Aug 2026 22:04:41 +0200 Subject: [PATCH 2/5] Answer the architecture review: one table for the HTML block start, and one code-span reader --- AGENTS.md | 6 +- .../commonmark-subset/text-escaping.json | 18 +++++ .../commonmark-subset/text-escaping.md | 4 + src/markdown/backtick-runs.ts | 17 ++++ src/markdown/commonmark-grammar.ts | 40 ++++++++-- src/markdown/emit/adf-to-markdown.test.ts | 12 +++ src/markdown/emit/line-escaping.ts | 5 +- src/markdown/parse/blocks.ts | 2 +- src/markdown/parse/html-blocks.ts | 28 ------- src/markdown/parse/inline-content.ts | 79 +++++++++---------- src/markdown/parse/markdown-to-adf.test.ts | 3 + 11 files changed, 130 insertions(+), 84 deletions(-) delete mode 100644 src/markdown/parse/html-blocks.ts diff --git a/AGENTS.md b/AGENTS.md index 21ce233..c2fdf95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,9 +54,9 @@ Round-trip equality is a property tested over a corpus, not a claim made in pros `dependencies` is empty. A runtime dependency enters only through a decision entry here stating why ~20 lines of own code cannot do the job, who maintains it, and what auditing it costs. So the CommonMark and HTML parsers are written in this repo. A table a standard fixes is data rather than -a dependency: HTML5's 2125 semicolon-terminated character references ship packed in -`markdown/entity-references.ts`, so entity decoding is complete without one. `devDependencies`: -few, each earning its keep; they never reach a consumer. +a dependency: HTML5's 2125 semicolon-terminated character references ship packed in their own +module, so entity decoding is complete without one. `devDependencies`: few, each earning its keep; +they never reach a consumer. ## 6. The package contract diff --git a/corpus/round-trip/commonmark-subset/text-escaping.json b/corpus/round-trip/commonmark-subset/text-escaping.json index 5481877..9350b74 100644 --- a/corpus/round-trip/commonmark-subset/text-escaping.json +++ b/corpus/round-trip/commonmark-subset/text-escaping.json @@ -44,6 +44,24 @@ } ], "type": "paragraph" + }, + { + "content": [ + { + "text": "
]*>/ }, { name: htmlConstructNames.processingInstruction, pattern: /^<\?[\s\S]*?\?>/ }, ] +// CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. +const blockTagNames = + 'address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul' +const completeTag = new RegExp(`^${htmlTagSource}[ \\t]*$`) +const htmlBlockConditions: HtmlBlockCondition[] = [ + { closer: /<\/(?:pre|script|style|textarea)>/i, construct: undefined, interrupts: true, start: /^<(?:pre|script|style|textarea)(?:[ \t>]|$)/i }, + { closer: /-->/, construct: htmlConstructNames.comment, interrupts: true, start: /^/, construct: htmlConstructNames.comment, interrupts: true, start: /^`). const inlineHtmlConstructs = [ - { name: htmlConstructNames.cdata, pattern: /^/ }, - { name: htmlConstructNames.comment, pattern: /^(?:||)/ }, - { name: htmlConstructNames.declaration, pattern: /^]*>/ }, - { name: htmlConstructNames.processingInstruction, pattern: /^<\?[\s\S]*?\?>/ }, + { name: htmlConstructNames.cdata, opener: /' }, + { name: htmlConstructNames.comment, opener: /|-->|--)/y, terminator: '-->' }, + { name: htmlConstructNames.declaration, opener: /' }, + { name: htmlConstructNames.processingInstruction, opener: /<\?/y, terminator: '?>' }, ] // CommonMark 0.31.2, HTML blocks: the tag names start condition 6 lists. const blockTagNames = @@ -57,7 +56,7 @@ const pipeClaim = /^\|/ const bulletListOpener = /^[*+-](?:[ \t]|$)/ // A superset of what the parser claims: over-escaping a line is safe, under-escaping one breaks the round-trip. const firstCharacterOpeners = [atxHeadingOpener, /^>/, bulletListOpener, codeFenceOpener, /^:{2,}/, pipeClaim] -const emailAutolink = /^<[^\s<>@]+@[^\s<>@]+>/ +const emailAutolink = /<[^\s<>@]+@[^\s<>@]+>/y const orderedListOpener = /^(\d{1,9})([.)])(?:[ \t]|$)/ const setextUnderline = /^(=+|-+)[ \t]*$/ const thematicBreak = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/ @@ -105,7 +104,7 @@ export function decodeTextEscapes(text: string): string { index += 2 continue } - const reference = text.charAt(index) === '&' ? readEntityReference(text.slice(index)) : undefined + const reference = readEntityReference(text, index) if (reference !== undefined) { decoded += reference.text index += reference.length @@ -135,17 +134,21 @@ export function holdsNullCharacter(text: string): boolean { return nullCharacter.test(text) } -export function htmlTagName(text: string): string { +function htmlTagName(text: string): string { return text.replace(tagName, '<$1>') } -export function inlineHtmlConstruct(text: string): HtmlConstruct | undefined { +export function inlineHtmlConstruct(text: string, index: number): string | undefined { for (const construct of inlineHtmlConstructs) { - const matched = construct.pattern.exec(text)?.[0] - if (matched !== undefined) return { length: matched.length, name: construct.name } + construct.opener.lastIndex = index + const opened = construct.opener.exec(text)?.[0] + if (opened === undefined) continue + if (opened.endsWith(construct.terminator)) return construct.name + return text.includes(construct.terminator, index + opened.length) ? construct.name : undefined } + htmlTag.lastIndex = index const tag = htmlTag.exec(text)?.[0] - return tag === undefined ? undefined : { length: tag.length, name: htmlTagName(tag) } + return tag === undefined ? undefined : htmlTagName(tag) } export function isAsciiPunctuation(character: string): boolean { @@ -193,11 +196,13 @@ export function openingHtmlBlock(line: string, interrupting: boolean): OpenHtmlB return undefined } -export function opensBracketedAutolink(text: string): boolean { +export function opensBracketedAutolink(text: string, index: number): boolean { + bracketedAutolink.lastIndex = index return bracketedAutolink.test(text) } -export function opensEmailAutolink(text: string): boolean { +export function opensEmailAutolink(text: string, index: number): boolean { + emailAutolink.lastIndex = index return emailAutolink.test(text) } diff --git a/src/markdown/directive-attributes.ts b/src/markdown/directive-attributes.ts index 246a216..405ba07 100644 --- a/src/markdown/directive-attributes.ts +++ b/src/markdown/directive-attributes.ts @@ -3,7 +3,7 @@ import type { VocabularyPair } from '../adf/attribute-vocabulary.ts' import { serializeCanonicalJson } from '../canonical-json.ts' const bareToken = /^[A-Za-z0-9_-]+$/ -const inlineDirectiveOpener = /^:[a-z][A-Za-z0-9]*[[{]/ +const inlineDirectiveOpener = /:[a-z][A-Za-z0-9]*[[{]/y // spec/flavour.md, Attributes. const quotedEscapes = /[&<`|]/g @@ -12,7 +12,8 @@ export function isBareToken(text: string): boolean { return bareToken.test(text) } -export function opensInlineDirective(text: string): boolean { +export function opensInlineDirective(text: string, index: number): boolean { + inlineDirectiveOpener.lastIndex = index return inlineDirectiveOpener.test(text) } diff --git a/src/markdown/emit/adf-to-markdown.test.ts b/src/markdown/emit/adf-to-markdown.test.ts index 9375bf4..7ab0676 100644 --- a/src/markdown/emit/adf-to-markdown.test.ts +++ b/src/markdown/emit/adf-to-markdown.test.ts @@ -216,6 +216,8 @@ test('escapes only text that would otherwise open a construct', () => { assert.equal(emitted('a e'), 'a \\ e\n') assert.equal(emitted('a c'), 'a \\ c\n') const later = paragraph({ text: 'a', type: 'text' }, { type: 'hardBreak' }, { text: ' \\ here.\n')), 'unmappable-html: no ADF node carries an HTML comment') + assert.equal(content(markdownToAdf('Part here.\n')), 'unmappable-html: no ADF node carries an HTML comment') assert.equal(code(markdownToAdf('A b\n')), 'unmappable-html') assert.equal(code(markdownToAdf('Part.\n\n')), 'unmappable-html') assert.deepEqual(path(markdownToAdf('Part.\n\nA b.\n')), ['content', 1]) @@ -315,6 +318,7 @@ test('refuses the raw inline HTML no element mapping carries, naming it', () => test('leaves the angle bracket that opens no HTML construct to the text it sits in', () => { assert.deepEqual(content(markdownToAdf('3 < 4 and 5 d\n')), [paragraph('a d')]) + assert.deepEqual(content(markdownToAdf('a