3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-20 15:15:44 +01:00

Match a reference label to its definition under Unicode case folding

This commit is contained in:
2026-11-15 22:04:28 +01:00
committed by Lillem4n
parent 3b89637f2c
commit fe3bf6e268
13 changed files with 1890 additions and 22 deletions
+8
View File
@@ -47,6 +47,14 @@ for (const element of document.querySelectorAll('[data-text-color]')) element.st
`losslessMarkdownToAdf` reads the first two as before. `losslessMarkdownToAdf` reads the first two as before.
### Readings
Markdown that `0.4.0` reads as a different document.
| Input | `0.3.1` | `0.4.0` |
| --- | --- | --- |
| a reference matching its definition only under Unicode case folding (`[ẞ]` with `[SS]: /url`, `[Straße]` with `[STRASSE]: /url`) | literal text, the definition dropped | the link; escape the opening bracket (`\[ẞ]`) to keep the text literal |
## From `0.2.0` to `0.3.0` ## From `0.2.0` to `0.3.0`
Every reader skips code text when it spells a mark other than a link, and `losslessMarkdownToAdf` Every reader skips code text when it spells a mark other than a link, and `losslessMarkdownToAdf`
+1 -2
View File
@@ -771,8 +771,7 @@ Serves [Goals](#goals) 1, 4 and 5.
- An image's title becomes its caption when the image stays an image, and otherwise the title of - An image's title becomes its caption when the image stays an image, and otherwise the title of
the link to the image. the link to the image.
And these, for other reasons: a shortcut reference matching its definition only under Unicode case And this, for another reason: a link whose text holds an autolink keeps the inner link and leaves
folding stays unresolved, and a link whose text holds an autolink keeps the inner link and leaves
the outer brackets literal text, which the spec requires and the reference itself breaks, nesting the outer brackets literal text, which the spec requires and the reference itself breaks, nesting
one `<a>` in the other. one `<a>` in the other.
- Not every document converts back: `adfToLosslessMarkdown` refuses a document nesting past 500 - Not every document converts back: `adfToLosslessMarkdown` refuses a document nesting past 500
+2 -2
View File
@@ -24,8 +24,8 @@ One directory per contract kind:
`spec.json` is the suite; `refusals.json` pins each refusing example to its error `code`; `spec.json` is the suite; `refusals.json` pins each refusing example to its error `code`;
`exceptions.json` pins each known divergence by `check`, `example`, `kind` and the exact `exceptions.json` pins each known divergence by `check`, `example`, `kind` and the exact
`divergence`, with a `reason`. `kind` is `mark-model` (the permanent count divergence from ADF's `divergence`, with a `reason`. `kind` is `mark-model` (the permanent count divergence from ADF's
mark-per-node model), `node-model` (a permanent divergence where ADF holds no node for what mark-per-node model) or `node-model` (a permanent divergence where ADF holds no node for what
CommonMark renders) or `pending` (a parser gap). CommonMark renders).
JSON is two-space indent, keys sorted, and a document read back must deep-equal the fixture's JSON is two-space indent, keys sorted, and a document read back must deep-equal the fixture's
(`docs/decisions.md` §Equality is deep). `spec.json` is the vendored, upstream machine-readable (`docs/decisions.md` §Equality is deep). `spec.json` is the vendored, upstream machine-readable
-14
View File
@@ -300,20 +300,6 @@
"kind": "mark-model", "kind": "mark-model",
"reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element." "reason": "The mark spans adjacent text nodes, counted once per node where CommonMark nests one element."
}, },
{
"check": "count",
"divergence": "a 1/0",
"example": 540,
"kind": "pending",
"reason": "The case-folding shortcut reference is unresolved; CommonMark folds case and links."
},
{
"check": "text",
"divergence": "\"ẞ\" against \"[ẞ]\"",
"example": 540,
"kind": "pending",
"reason": "A case-folding shortcut reference is unresolved; CommonMark folds case and links."
},
{ {
"check": "count", "check": "count",
"divergence": "a 1/2", "divergence": "a 1/2",
+3 -1
View File
@@ -570,7 +570,9 @@ renumbered suite. A spec bump is a deliberate re-pin, exceptions re-derived by h
Atlassian's ADF JSON Schemas ship vendored the same way, at `spec/adf-schema/`, rather than as the Atlassian's ADF JSON Schemas ship vendored the same way, at `spec/adf-schema/`, rather than as the
`@atlaskit/adf-schema` dev dependency — CommonJS-only, some fifty packages with React among them, `@atlaskit/adf-schema` dev dependency — CommonJS-only, some fifty packages with React among them,
and a release most days for Renovate to automerge — re-pinned by hand when a payload or a report and a release most days for Renovate to automerge — re-pinned by hand when a payload or a report
shows the need. shows the need. Unicode's `CaseFolding.txt` (18.0.0) is vendored at `spec/unicode/` and packed in
`src/markdown/commonmark/case-folding.ts` the same way; its test holds every C and F mapping to the
file, so a re-pin is a new file and a green test.
## fast-check ## fast-check
+15
View File
@@ -1214,3 +1214,18 @@ Read most (opens and searches):
- `ci.sh` 1 - `ci.sh` 1
- `docs/decisions.md` 1 - `docs/decisions.md` 1
- `package.json` 1 - `package.json` 1
## Reading cost, chunk 2bba00ce50cb, finished 2026-11-15T21:01:07Z
117645 new input tokens before the first commit.
Read most (opens and searches):
- `src/markdown/portable/inline-reduction.ts` 6
- `src/markdown/portable/inline-reduction.test.ts` 5
- `ci.sh` 2
- `.gitignore` 1
- `README.md` 1
- `docker-runner.sh` 1
- `src/adf/document.ts` 1
- `src/markdown/emit/inline-runs.ts` 1
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -11,7 +11,7 @@ import { commonMarkToAdf, losslessMarkdownToAdf } from '../markdown/parse/markdo
const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus', 'commonmark-spec') const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'corpus', 'commonmark-spec')
const checks = ['count', 'fixpoint', 'text'] as const const checks = ['count', 'fixpoint', 'text'] as const
const kinds = ['mark-model', 'node-model', 'pending'] as const const kinds = ['mark-model', 'node-model'] as const
type Check = (typeof checks)[number] type Check = (typeof checks)[number]
type ExceptionKind = (typeof kinds)[number] type ExceptionKind = (typeof kinds)[number]
@@ -0,0 +1,60 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
import { foldCase } from './case-folding.ts'
const source = readFileSync(join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'spec', 'unicode', 'CaseFolding.txt'), 'utf8')
function characters(hexes: string): string {
return hexes
.split(' ')
.map((hex) => String.fromCodePoint(parseInt(hex, 16)))
.join('')
}
// `CaseFolding.txt`'s common and full mappings, as `0041; C; 0061;` and `00DF; F; 0073 0073;` give them.
const mappings = new Map(
source
.split('\n')
.map((line) => /^([0-9A-F]+); [CF]; ([0-9A-F ]+);/.exec(line))
.filter((match) => match !== null)
.map((match): [string, string] => [characters(match[1] ?? ''), characters(match[2] ?? '')]),
)
test('the vendored file gives the mappings the other tests hold the table to', () => {
assert.equal(mappings.size, 1606)
assert.equal(mappings.get('ẞ'), 'ss')
assert.equal(mappings.get('ß'), 'ss')
assert.equal(mappings.get('İ'), 'i̇')
})
test('folds every code point to the mapping the standard gives it', () => {
assert.deepEqual(
[...mappings].filter(([from, to]) => foldCase(from) !== to).map(([from]) => from),
[],
)
})
test('leaves every other code point as it is', () => {
const changed: number[] = []
for (let point = 0; point <= 0x10ffff; point++) {
const character = String.fromCodePoint(point)
if (!mappings.has(character) && foldCase(character) !== character) changed.push(point)
}
assert.deepEqual(changed, [])
})
test('folds a string code point by code point, a surrogate pair as one', () => {
assert.equal(foldCase('Ab\u{10400}Ç ẞ'), 'ab\u{10428}ç ss')
assert.equal(foldCase(''), '')
assert.equal(foldCase('\ud801'), '\ud801')
})
test('keeps the dotless i and the capital I apart, as the standard does', () => {
assert.equal(foldCase('I'), 'i')
assert.equal(foldCase('ı'), 'ı')
assert.notEqual(foldCase('ı'), foldCase('I'))
})
+70
View File
@@ -0,0 +1,70 @@
// Unicode's `CaseFolding.txt` (vendored at `spec/unicode/`), the common (C) and full (F) mappings that CommonMark
// folds a link label by. The Turkic (T) and simple (S) mappings stay out: the full ones supersede the simple.
// A common run reads `first:count:stride:delta` in hex: `count` code points from `first`, `stride` apart, each
// folding to its own plus `delta`. A full folding reads `code point=code points` in hex.
const commonRuns =
'41:1a:1:+20 b5:1:1:+307 c0:17:1:+20 d8:7:1:+20 100:18:2:+1 132:3:2:+1 139:8:2:+1 14a:17:2:+1 178:1:1:-79\
179:3:2:+1 17f:1:1:-10c 181:1:1:+d2 182:2:2:+1 186:1:1:+ce 187:1:1:+1 189:2:1:+cd 18b:1:1:+1 18e:1:1:+4f\
18f:1:1:+ca 190:1:1:+cb 191:1:1:+1 193:1:1:+cd 194:1:1:+cf 196:1:1:+d3 197:1:1:+d1 198:1:1:+1 19c:1:1:+d3\
19d:1:1:+d5 19f:1:1:+d6 1a0:3:2:+1 1a6:1:1:+da 1a7:1:1:+1 1a9:1:1:+da 1ac:1:1:+1 1ae:1:1:+da 1af:1:1:+1\
1b1:2:1:+d9 1b3:2:2:+1 1b7:1:1:+db 1b8:2:4:+1 1c4:1:1:+2 1c5:1:1:+1 1c7:1:1:+2 1c8:1:1:+1 1ca:1:1:+2 1cb:9:2:+1\
1de:9:2:+1 1f1:1:1:+2 1f2:2:2:+1 1f6:1:1:-61 1f7:1:1:-38 1f8:14:2:+1 220:1:1:-82 222:9:2:+1 23a:1:1:+2a2b\
23b:1:1:+1 23d:1:1:-a3 23e:1:1:+2a28 241:1:1:+1 243:1:1:-c3 244:1:1:+45 245:1:1:+47 246:5:2:+1 345:1:1:+74\
370:2:2:+1 376:1:1:+1 37f:1:1:+74 386:1:1:+26 388:3:1:+25 38c:1:1:+40 38e:2:1:+3f 391:11:1:+20 3a3:9:1:+20\
3c2:1:1:+1 3cf:1:1:+8 3d0:1:1:-1e 3d1:1:1:-19 3d5:1:1:-f 3d6:1:1:-16 3d8:c:2:+1 3f0:1:1:-36 3f1:1:1:-30\
3f4:1:1:-3c 3f5:1:1:-40 3f7:1:1:+1 3f9:1:1:-7 3fa:1:1:+1 3fd:3:1:-82 400:10:1:+50 410:20:1:+20 460:11:2:+1\
48a:1b:2:+1 4c0:1:1:+f 4c1:7:2:+1 4d0:30:2:+1 531:26:1:+30 10a0:26:1:+1c60 10c7:2:6:+1c60 13f8:6:1:-8\
1c80:1:1:-184e 1c81:1:1:-184d 1c82:1:1:-1844 1c83:2:1:-1842 1c85:1:1:-1843 1c86:1:1:-183c 1c87:1:1:-1824\
1c88:1:1:+89c3 1c89:1:1:+1 1c90:2b:1:-bc0 1cbd:3:1:-bc0 1e00:4b:2:+1 1e9b:1:1:-3a 1ea0:30:2:+1 1f08:8:1:-8\
1f18:6:1:-8 1f28:8:1:-8 1f38:8:1:-8 1f48:6:1:-8 1f59:4:2:-8 1f68:8:1:-8 1fb8:2:1:-8 1fba:2:1:-4a 1fbe:1:1:-1c05\
1fc8:4:1:-56 1fd8:2:1:-8 1fda:2:1:-64 1fe8:2:1:-8 1fea:2:1:-70 1fec:1:1:-7 1ff8:2:1:-80 1ffa:2:1:-7e\
2126:1:1:-1d5d 212a:1:1:-20bf 212b:1:1:-2046 2132:1:1:+1c 2160:10:1:+10 2183:1:1:+1 24b6:1a:1:+1a 2c00:30:1:+30\
2c60:1:1:+1 2c62:1:1:-29f7 2c63:1:1:-ee6 2c64:1:1:-29e7 2c67:3:2:+1 2c6d:1:1:-2a1c 2c6e:1:1:-29fd 2c6f:1:1:-2a1f\
2c70:1:1:-2a1e 2c72:2:3:+1 2c7e:2:1:-2a3f 2c80:32:2:+1 2ceb:2:2:+1 2cf2:2:794e:+1 a642:16:2:+1 a680:e:2:+1\
a722:7:2:+1 a732:1f:2:+1 a779:2:2:+1 a77d:1:1:-8a04 a77e:5:2:+1 a78b:1:1:+1 a78d:1:1:-a528 a790:2:2:+1\
a796:a:2:+1 a7aa:1:1:-a544 a7ab:1:1:-a54f a7ac:1:1:-a54b a7ad:1:1:-a541 a7ae:1:1:-a544 a7b0:1:1:-a512\
a7b1:1:1:-a52a a7b2:1:1:-a515 a7b3:1:1:+3a0 a7b4:8:2:+1 a7c4:1:1:-30 a7c5:1:1:-a543 a7c6:1:1:-8a38 a7c7:2:2:+1\
a7cb:1:1:-a567 a7cc:8:2:+1 a7dc:1:1:-a641 a7dd:2:5:-a566 a7f5:1:1:+1 ab6c:2:1:-21 ab70:50:1:-97d0 ff21:1a:1:+20\
10400:28:1:+28 104b0:24:1:+28 10570:b:1:+27 1057c:f:1:+27 1058c:7:1:+27 10594:2:1:+27 10c80:33:1:+40\
10d50:16:1:+20 118a0:20:1:+20 16e40:20:1:+20 16ea0:19:1:+1b 1df40:2:8:+1 1df4a:2:3:+1 1df51:2:17:+1 1df6a:3:2:+1\
1df72:7:2:+1 1e900:22:1:+22'
const fullFoldings =
'df=73,73 130=69,307 149=2bc,6e 1f0=6a,30c 390=3b9,308,301 3b0=3c5,308,301 587=565,582 1e96=68,331 1e97=74,308\
1e98=77,30a 1e99=79,30a 1e9a=61,2be 1e9e=73,73 1f50=3c5,313 1f52=3c5,313,300 1f54=3c5,313,301 1f56=3c5,313,342\
1f80=1f00,3b9 1f81=1f01,3b9 1f82=1f02,3b9 1f83=1f03,3b9 1f84=1f04,3b9 1f85=1f05,3b9 1f86=1f06,3b9 1f87=1f07,3b9\
1f88=1f00,3b9 1f89=1f01,3b9 1f8a=1f02,3b9 1f8b=1f03,3b9 1f8c=1f04,3b9 1f8d=1f05,3b9 1f8e=1f06,3b9 1f8f=1f07,3b9\
1f90=1f20,3b9 1f91=1f21,3b9 1f92=1f22,3b9 1f93=1f23,3b9 1f94=1f24,3b9 1f95=1f25,3b9 1f96=1f26,3b9 1f97=1f27,3b9\
1f98=1f20,3b9 1f99=1f21,3b9 1f9a=1f22,3b9 1f9b=1f23,3b9 1f9c=1f24,3b9 1f9d=1f25,3b9 1f9e=1f26,3b9 1f9f=1f27,3b9\
1fa0=1f60,3b9 1fa1=1f61,3b9 1fa2=1f62,3b9 1fa3=1f63,3b9 1fa4=1f64,3b9 1fa5=1f65,3b9 1fa6=1f66,3b9 1fa7=1f67,3b9\
1fa8=1f60,3b9 1fa9=1f61,3b9 1faa=1f62,3b9 1fab=1f63,3b9 1fac=1f64,3b9 1fad=1f65,3b9 1fae=1f66,3b9 1faf=1f67,3b9\
1fb2=1f70,3b9 1fb3=3b1,3b9 1fb4=3ac,3b9 1fb6=3b1,342 1fb7=3b1,342,3b9 1fbc=3b1,3b9 1fc2=1f74,3b9 1fc3=3b7,3b9\
1fc4=3ae,3b9 1fc6=3b7,342 1fc7=3b7,342,3b9 1fcc=3b7,3b9 1fd2=3b9,308,300 1fd3=3b9,308,301 1fd6=3b9,342\
1fd7=3b9,308,342 1fe2=3c5,308,300 1fe3=3c5,308,301 1fe4=3c1,313 1fe6=3c5,342 1fe7=3c5,308,342 1ff2=1f7c,3b9\
1ff3=3c9,3b9 1ff4=3ce,3b9 1ff6=3c9,342 1ff7=3c9,342,3b9 1ffc=3c9,3b9 fb00=66,66 fb01=66,69 fb02=66,6c\
fb03=66,66,69 fb04=66,66,6c fb05=73,74 fb06=73,74 fb13=574,576 fb14=574,565 fb15=574,56b fb16=57e,576\
fb17=574,56d 1df95=73,73'
const foldings = new Map<string, string>()
for (const run of commonRuns.split(/\s+/)) {
const [first = 0, count = 0, stride = 0, delta = 0] = run.split(':').map((field) => parseInt(field, 16))
for (let index = 0; index < count; index++) {
const from = first + index * stride
foldings.set(String.fromCodePoint(from), String.fromCodePoint(from + delta))
}
}
for (const folding of fullFoldings.split(/\s+/)) {
const [from = '', to = ''] = folding.split('=')
foldings.set(
String.fromCodePoint(parseInt(from, 16)),
to
.split(',')
.map((code) => String.fromCodePoint(parseInt(code, 16)))
.join(''),
)
}
export function foldCase(text: string): string {
let folded = ''
for (const character of text) folded += foldings.get(character) ?? character
return folded
}
+12 -1
View File
@@ -1,7 +1,7 @@
import assert from 'node:assert/strict' import assert from 'node:assert/strict'
import { test } from 'node:test' import { test } from 'node:test'
import { readDestination, readInlineTarget, targetMemo } from './link-syntax.ts' import { normalizeLabel, readDestination, readInlineTarget, targetMemo } from './link-syntax.ts'
// Every `](` of the text, read in order through one memo, as a reader closes its brackets. // Every `](` of the text, read in order through one memo, as a reader closes its brackets.
function targets(text: string): unknown[] { function targets(text: string): unknown[] {
@@ -47,3 +47,14 @@ test('a target reads whitespace and a title around its destination, bracketed, u
]) ])
assert.deepEqual(targets('[a](<b>"t")'), [undefined]) assert.deepEqual(targets('[a](<b>"t")'), [undefined])
}) })
test('a label folds under Unicode case folding after its whitespace collapses', () => {
assert.equal(normalizeLabel(' Foo \n\t Bar '), 'foo bar')
assert.equal(normalizeLabel('ẞ'), normalizeLabel('SS'))
assert.equal(normalizeLabel('Straße'), normalizeLabel('STRASSE'))
assert.equal(normalizeLabel('ς'), normalizeLabel('Σ'))
})
test('a dotless i and a capital I stay two labels', () => {
assert.notEqual(normalizeLabel('ı'), normalizeLabel('I'))
})
+2 -1
View File
@@ -1,4 +1,5 @@
import { backslashEscape, controlCharacterRange, decodeTextEscapes, holdsControlCharacter, holdsNullCharacter } from './grammar.ts' import { backslashEscape, controlCharacterRange, decodeTextEscapes, holdsControlCharacter, holdsNullCharacter } from './grammar.ts'
import { foldCase } from './case-folding.ts'
import { holdsEntityReference, readEntityReference } from './entity-references.ts' import { holdsEntityReference, readEntityReference } from './entity-references.ts'
export type InlineTarget = { definition: () => LinkDefinition; length: number } export type InlineTarget = { definition: () => LinkDefinition; length: number }
@@ -36,7 +37,7 @@ export function normalizeLabel(raw: string): string {
const collapsed = raw.replace(/[ \t\n]+/g, ' ') const collapsed = raw.replace(/[ \t\n]+/g, ' ')
const start = collapsed.startsWith(' ') ? 1 : 0 const start = collapsed.startsWith(' ') ? 1 : 0
const end = collapsed.endsWith(' ') ? collapsed.length - 1 : collapsed.length const end = collapsed.endsWith(' ') ? collapsed.length - 1 : collapsed.length
return collapsed.slice(start, end).toLowerCase() return foldCase(collapsed.slice(start, end))
} }
export function readDestination(text: string, offset: number): LinkPart | undefined { export function readDestination(text: string, offset: number): LinkPart | undefined {
@@ -1080,6 +1080,18 @@ test('reads the reference links a definition resolves, and leaves the rest liter
assert.deepEqual(content(losslessMarkdownToAdf('[][]\n')), [paragraph('[][]')]) assert.deepEqual(content(losslessMarkdownToAdf('[][]\n')), [paragraph('[][]')])
}) })
for (const read of [commonMarkToAdf, losslessMarkdownToAdf, portableMarkdownToAdf]) {
test(`matches a reference to its definition under Unicode case folding in ${read.name}`, () => {
assert.deepEqual(content(read('[ẞ]\n\n[SS]: /url\n')), [{ content: [marked('ẞ', link('/url'))], type: 'paragraph' }])
assert.deepEqual(content(read('[Straße][]\n\n[STRASSE]: /url\n')), [{ content: [marked('Straße', link('/url'))], type: 'paragraph' }])
assert.deepEqual(content(read('[a][ς]\n\n[Σ]: /url\n')), [{ content: [marked('a', link('/url'))], type: 'paragraph' }])
})
test(`keeps a dotless i and a capital I two labels in ${read.name}`, () => {
assert.deepEqual(content(read('[ı]\n\n[I]: /url\n')), [paragraph('[ı]')])
})
}
test('leaves a link label past 999 characters literal, an escape counting as two, in every reader', () => { test('leaves a link label past 999 characters literal, an escape counting as two, in every reader', () => {
const longest = `foo${' '.repeat(993)}bar` const longest = `foo${' '.repeat(993)}bar`
const tooLong = `foo${' '.repeat(994)}bar` const tooLong = `foo${' '.repeat(994)}bar`