A CJK, Hangul or Southeast Asian character beside a == bounds a highlight #147

Merged
lilleman merged 5 commits from cjk-highlight into main 2026-09-29 19:53:10 +02:00
6 changed files with 27 additions and 9 deletions
+1 -1
View File
@@ -133,7 +133,7 @@ read replaces mentions, attachments and macros with text.
| `panel` | a GitHub alert, `> [!WARNING]`: info `NOTE`, note `IMPORTANT`, tip and success `TIP`, warning `WARNING`, error `CAUTION`, custom `NOTE` | `NOTE` info, `IMPORTANT` note, `TIP` tip, `WARNING` warning, `CAUTION` error, and Obsidian's: hint tip; success, check, done success; attention warning; danger, failure, fail, missing, bug, error error; any other word info — in any case; the rest of the marker's line is the first paragraph | | `panel` | a GitHub alert, `> [!WARNING]`: info `NOTE`, note `IMPORTANT`, tip and success `TIP`, warning `WARNING`, error `CAUTION`, custom `NOTE` | `NOTE` info, `IMPORTANT` note, `TIP` tip, `WARNING` warning, `CAUTION` error, and Obsidian's: hint tip; success, check, done success; attention warning; danger, failure, fail, missing, bug, error error; any other word info — in any case; the rest of the marker's line is the first paragraph |
| `expand`, `nestedExpand` | Obsidian's folded callout, `> [!NOTE]- Title` | `-` or `+` after any word, the rest of the marker's line the title; an expand inside an expand is a `nestedExpand` | | `expand`, `nestedExpand` | Obsidian's folded callout, `> [!NOTE]- Title` | `-` or `+` after any word, the rest of the marker's line the title; an expand inside an expand is a `nestedExpand` |
| `taskList` | `- [x] Done`, `- [ ] Todo` | a bullet list whose every item is so marked, `[X]` too | | `taskList` | `- [x] Done`, `- [ ] Todo` | a bullet list whose every item is so marked, `[X]` too |
| `backgroundColor` | `==text==` | `==text==` on one line, the text touching both delimiters, bounded outside by whitespace, punctuation or a line edge, in the editor's default highlight `#f8e6a0` | | `backgroundColor` | `==text==` | `==text==` on one line, the text touching both delimiters, bounded outside by whitespace, punctuation or a line edge, or touching a Han, Hangul, Hiragana, Katakana, Thai, Lao, Khmer or Myanmar character on either side, in the editor's default highlight `#f8e6a0` |
| `table` | a pipe table: the first row its header, a cell's blocks on one line, a span kept under its header by empty cells | — | | `table` | a pipe table: the first row its header, a cell's blocks on one line, a span kept under its header by empty cells | — |
| `decisionList` | a bullet list | — | | `decisionList` | a bullet list | — |
| `mention`, `status`, `emoji`, `date` | their text: `@` kept, a mention with none `@` and its id, an emoji its `shortName` without, a date `2026-09-13` in UTC | — | | `mention`, `status`, `emoji`, `date` | their text: `@` kept, a mention with none `@` and its id, an emoji its `shortName` without, a date `2026-09-13` in UTC | — |
+7 -5
View File
@@ -145,14 +145,16 @@ parentheses: `> [!faq]- See [x](http://y)` reads to the title `See x (http://y)`
## The plain flavour's spellings ## The plain flavour's spellings
2026-09-14, panels 2026-09-25, the maintainer. Goal 5. Valid while GitHub's renderer is the one the 2026-09-14, panels 2026-09-25 and 2026-09-29, the maintainer. Goal 5. Valid while GitHub's
audience's markdown is read in. renderer is the one the audience's markdown is read in.
README §Plain markdown's rows come from a survey of GitHub, GitLab, Gitea, Obsidian, Pandoc, README §Plain markdown's rows come from a survey of GitHub, GitLab, Gitea, Obsidian, Pandoc,
MkDocs, Docusaurus, Typora, Joplin, Logseq, Bear, Notion, Azure DevOps and Discord, GitHub's MkDocs, Docusaurus, Typora, Joplin, Logseq, Bear, Notion, Azure DevOps and Discord, GitHub's
renderer confirming each shape. Reader panels settled `error` as an error panel and the `==` renderer confirming each shape. Reader panels settled `error` as an error panel, the `==` bounds
bounds (3 of 3), the external image's two forms (6 of 7), the omission notes and a rule opening a (3 of 3) and a Han, Hangul, kana, Thai, Lao, Khmer or Myanmar character on either side bounding a
list item dropping (3 of 3), and a list's numbering overflowing into bullets (3 of 3, 5 of 7). delimiter, so `は==日本語==で` (3 of 3), `==한국어==에서만` and `iPhone==専用==` (6 of 7) highlight,
the external image's two forms (6 of 7), the omission notes and a rule opening a list item
dropping (3 of 3), and a list's numbering overflowing into bullets (3 of 3, 5 of 7).
Reading takes other tools' spellings, since it reads their output and writes none of them. Reading takes other tools' spellings, since it reads their output and writes none of them.
Rejected: `~sub~` and `^sup^` (`~2~` is a strike on GitHub, so `subsup` drops), underline and colour Rejected: `~sub~` and `^sup^` (`~2~` is a strike on GitHub, so `subsup` drops), underline and colour
spellings, raw HTML (`<details>`, `<mark>`), MkDocs `!!!` and the `:::` admonition family, spellings, raw HTML (`<details>`, `<mark>`), MkDocs `!!!` and the `:::` admonition family,
+1 -1
View File
@@ -23,7 +23,7 @@ export const propertyTimeout = 600000
const depthIdentifier = fc.createDepthIdentifier() const depthIdentifier = fc.createDepthIdentifier()
const emptyCell: AdfNode = { content: [{ type: 'paragraph' }], type: 'tableCell' } const emptyCell: AdfNode = { content: [{ type: 'paragraph' }], type: 'tableCell' }
const flatCommonMarkShapeWeight = 4 const flatCommonMarkShapeWeight = 4
export const markdownPieces = fc.constantFrom(...'aZ09 \t\n!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~é\xa0🎉', '==', '[!NOTE]', '[x]', 'ab:', 'http://', directivePrefix, `${directivePrefix}a[`, `${directivePrefix}a{`) export const markdownPieces = fc.constantFrom(...'aZ09 \t\n!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~é\xa0🎉日ー한𠀀', '==', '[!NOTE]', '[x]', 'ab:', 'http://', directivePrefix, `${directivePrefix}a[`, `${directivePrefix}a{`)
const nestingCommonMarkShapeWeight = 21 const nestingCommonMarkShapeWeight = 21
const spelledTypes = new Set(['text', ...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...Object.keys(markAttributes)]) const spelledTypes = new Set(['text', ...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...Object.keys(markAttributes)])
@@ -148,6 +148,9 @@ test('spells a highlight as a == pair around the run, whatever its colour', () =
assert.equal(plain(paragraph(text('a', highlight('#fff'), code), text('b', highlight('#fff')))), '`a`==b==\n') assert.equal(plain(paragraph(text('a', highlight('#fff'), code), text('b', highlight('#fff')))), '`a`==b==\n')
assert.equal(plain(paragraph(text('=', highlight('#fff')), text(' '), text('a==b', highlight('#fff')))), '==\\=== ==a==b==\n') assert.equal(plain(paragraph(text('=', highlight('#fff')), text(' '), text('a==b', highlight('#fff')))), '==\\=== ==a==b==\n')
assert.equal(plain(paragraph(text('x'), text('y', highlight('#fff')), text(' z'))), 'xy z\n') assert.equal(plain(paragraph(text('x'), text('y', highlight('#fff')), text(' z'))), 'xy z\n')
assert.equal(plain(paragraph(text('この機能は'), text('日本語', highlight('#fff')), text('でのみ'))), 'この機能は==日本語==でのみ\n')
assert.equal(plain(paragraph(text('サーバー'), text('停止', highlight('#fff')), text('中 iPhone'), text('専用', highlight('#fff')), text('アプリ 기능은 '), text('한국어', highlight('#fff')), text('에서만'))), 'サーバー==停止==中 iPhone==専用==アプリ 기능은 ==한국어==에서만\n')
assert.equal(plain(paragraph(text('日==本==語'))), '日\\==本\\==語\n')
}) })
test('escapes text a renderer would take as a flavour marker, and only there', () => { test('escapes text a renderer would take as a flavour marker, and only there', () => {
@@ -149,6 +149,12 @@ test('reads a == pair to the editor default highlight, Yellow200 #f8e6a0 in @atl
assert.deepEqual(read('==`a`==\n'), [paragraph(text('a', code))]) assert.deepEqual(read('==`a`==\n'), [paragraph(text('a', code))])
assert.deepEqual(read('x==y==z ==a == b==, (==c==) _d_==e==\n'), [paragraph(text('x==y==z '), text('a == b', highlight), text(', ('), text('c', highlight), text(') '), text('d', em), text('e', highlight))]) assert.deepEqual(read('x==y==z ==a == b==, (==c==) _d_==e==\n'), [paragraph(text('x==y==z '), text('a == b', highlight), text(', ('), text('c', highlight), text(') '), text('d', em), text('e', highlight))])
assert.deepEqual(read('😀==b== ==c==😀 é==d==\n'), [paragraph(text('😀'), text('b', highlight), text(' '), text('c', highlight), text('😀 é==d=='))]) assert.deepEqual(read('😀==b== ==c==😀 é==d==\n'), [paragraph(text('😀'), text('b', highlight), text(' '), text('c', highlight), text('😀 é==d=='))])
assert.deepEqual(read('この機能は==日本語==でのみ、中文==重点==内容、ภาษา==ไทย==ดี 𠀀==𠀁==𠀂 이 기능은 ==한국어==에서만 サーバー==停止==中 このiPhone==専用==アプリ\n'), [
paragraph(
text('この機能は'), text('日本語', highlight), text('でのみ、中文'), text('重点', highlight), text('内容、ภาษา'), text('ไทย', highlight), text('ดี 𠀀'), text('𠀁', highlight),
text('𠀂 이 기능은 '), text('한국어', highlight), text('에서만 サーバー'), text('停止', highlight), text('中 このiPhone'), text('専用', highlight), text('アプリ'),
),
])
assert.deepEqual(read('# ==h==\n\n| ==c== |\n| --- |\n'), [ assert.deepEqual(read('# ==h==\n\n| ==c== |\n| --- |\n'), [
node('heading', { level: 1 }, text('h', highlight)), node('heading', { level: 1 }, text('h', highlight)),
bare('table', bare('tableRow', bare('tableHeader', paragraph(text('c', highlight))))), bare('table', bare('tableRow', bare('tableHeader', paragraph(text('c', highlight))))),
@@ -196,7 +202,7 @@ test('reads what the reduction wrote back to the node it reduced, less the attri
test('reads text the writer kept from reading as a marker back as text', () => { test('reads text the writer kept from reading as a marker back as text', () => {
const quote = bare('blockquote', said('[!NOTE] x')) const quote = bare('blockquote', said('[!NOTE] x'))
const list = bare('bulletList', bare('listItem', said('[x] a')), bare('listItem', said('[ ] b'))) const list = bare('bulletList', bare('listItem', said('[x] a')), bare('listItem', said('[ ] b')))
assert.deepEqual(roundTripped(said('==x== a==b'), quote, list), [said('==x== a==b'), quote, list]) assert.deepEqual(roundTripped(said('==x== a==b 日==本==語'), quote, list), [said('==x== a==b 日==本==語'), quote, list])
assert.deepEqual(roundTripped(bare('taskList', task('DONE', text('[x] ==a==')))), [bare('taskList', task('DONE', text('[x] ==a==')))]) assert.deepEqual(roundTripped(bare('taskList', task('DONE', text('[x] ==a==')))), [bare('taskList', task('DONE', text('[x] ==a==')))])
}) })
+8 -1
View File
@@ -16,6 +16,9 @@ const alertWords: Readonly<Record<string, string>> = {
warning: 'WARNING', warning: 'WARNING',
} }
// ー, ー and the kana voicing marks sit outside the kana scripts; Script_Extensions would also take Latin combining marks.
const boundingScript = /^[\u3099\u309a\u30fc\uff70\uff9e\uff9f\p{Script=Han}\p{Script=Hangul}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Khmer}\p{Script=Lao}\p{Script=Myanmar}\p{Script=Thai}]$/u
const panelTypesByWord: Readonly<Record<string, string>> = { const panelTypesByWord: Readonly<Record<string, string>> = {
attention: 'warning', attention: 'warning',
bug: 'error', bug: 'error',
@@ -60,7 +63,11 @@ export function highlightFlanking(source: string, index: number): { closes: bool
const end = index + highlightDelimiter.length const end = index + highlightDelimiter.length
const before = Array.from(source.slice(Math.max(0, index - 2), index)).at(-1) ?? '' const before = Array.from(source.slice(Math.max(0, index - 2), index)).at(-1) ?? ''
const after = Array.from(source.slice(end, end + 2))[0] ?? '' const after = Array.from(source.slice(end, end + 2))[0] ?? ''
return { closes: flanks(before) && !isWordCharacter(after), opens: flanks(after) && !isWordCharacter(before) } return { closes: flanks(before) && bounds(after, before), opens: flanks(after) && bounds(before, after) }
}
function bounds(outside: string, inside: string): boolean {
return !isWordCharacter(outside) || boundingScript.test(outside) || boundingScript.test(inside)
} }
function flanks(character: string): boolean { function flanks(character: string): boolean {