Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo

AGENTS.md

@@ -93,8 +93,8 @@ In `docs/decisions.md`:
 ## 1. Nothing about any consumer in the library
 
 The shipped library holds no Jira client, no HTTP, no REST shapes, no issue keys and no actual
-consumer's name. Design against the README's personas. Tooling outside the build may fetch from
-public sites and name them, as `corpus-gathering/` does.
+consumer's name. Design against the README's personas. Tooling and fixtures outside the build may
+name public sites, and tooling may fetch from them, as `corpus-gathering/` does.
 
 ## 2. Release automation
 

corpus-gathering/anonymize.mjs

@@ -1,6 +1,6 @@
 // A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
 const formatKeys = new Set([
-  '__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
+  '__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode',
   'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
   'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
 ])
@@ -10,7 +10,7 @@ const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12
 const uuid = new RegExp(`^${uuidPattern}$`, 'i')
 const uuidOrToken = new RegExp(`(?<![\\p{L}\\p{N}])${uuidPattern}(?![\\p{L}\\p{M}\\p{N}])|${token.source}`, 'giu')
 const uuidVersionAt = 14
-// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
+// A chance match below this length identifies nobody, and a document may hold every such token, so redrawing around them might never end.
 const shortestLeak = 3
 const redraws = 64
 
@@ -22,47 +22,45 @@ function tokensOf(text) {
   return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
 }
 
-// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
-function walk(value, path, visit) {
+// Calls `visit` with each leaf, its steps from the document and its kind: `kept`, or `free`, which the anonymizer replaces.
+function walk(value, steps, visit) {
   if (Array.isArray(value)) {
-    value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
+    value.forEach((item, index) => walk(item, [...steps, index], visit))
   } else if (typeof value === 'object' && value !== null) {
     for (const [key, member] of Object.entries(value)) {
-      const memberPath = `${path}/${key}`
-      if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
-      else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
-      else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
-      else walkFree(member, memberPath, visit)
+      const memberSteps = [...steps, key]
+      if (key === 'attrs') walkAttributes(member, memberSteps, visit)
+      else if (key === 'content' || key === 'marks') walk(member, memberSteps, visit)
+      else if (key === 'type' || key === 'version') visit(member, memberSteps, 'kept')
+      else walkFree(member, memberSteps, visit)
     }
   } else {
-    visit(value, path, 'free')
+    visit(value, steps, 'free')
   }
 }
 
-// `<app id>/<environment id>/...`, so one more id, where any other extension key names a macro.
+// A Forge key reads `<app id>/<environment id>/...`, so it holds ids to replace. Any other extension key names a macro and stays.
 function isForgeKey(key, value) {
   return key === 'extensionKey' && String(value).includes('/')
 }
 
-function walkAttributes(attributes, path, visit, nodeType) {
+function walkAttributes(attributes, steps, visit) {
   for (const [key, member] of Object.entries(attributes)) {
-    const memberPath = `${path}/${key}`
-    if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept')
-    else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
-    else walkFree(member, memberPath, visit)
+    const memberSteps = [...steps, key]
+    if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberSteps, 'kept')
+    else walkFree(member, memberSteps, visit)
   }
 }
 
-function walkFree(value, path, visit) {
-  if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
-  else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
-  else visit(value, path, 'free')
+function walkFree(value, steps, visit) {
+  if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, [...steps, key], visit)
+  else visit(value, steps, 'free')
 }
 
 function replacedStrings(document) {
   const strings = []
-  walk(document, '', (value, _path, kind) => {
-    if (kind !== 'kept' && typeof value === 'string') strings.push(value)
+  walk(document, [], (value, _steps, kind) => {
+    if (kind === 'free' && typeof value === 'string') strings.push(value)
   })
   return strings
 }
@@ -101,7 +99,7 @@ function wordDrawer(words) {
 }
 
 function withCaseOf(original, word) {
-  if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
+  if (original === original.toUpperCase() && original !== original.toLowerCase()) return word.toUpperCase()
   if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
   return word
 }
@@ -111,7 +109,6 @@ function replacer(document, draws) {
   const digits = stream(draws.digits)
   const hex = stream(draws.hex)
   const letters = stream(draws.letters)
-  const nextPerson = cursor(draws.persons)
   const nextUuid = cursor(draws.uuids)
   const nextWord = wordDrawer(draws.words)
   const spelled = new Map()
@@ -139,7 +136,7 @@ function replacer(document, draws) {
   function word(original) {
     if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
     let tries = 0
-    // A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
+    // A long document can hold every word of a length, so the last two of the draws `fresh` makes gain a suffix.
     return withCaseOf(original, fresh(() => {
       tries += 1
       return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
@@ -172,42 +169,34 @@ function replacer(document, draws) {
     })
   }
 
-  function person(original) {
-    return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
-  }
-
   function scalar(value) {
     if (typeof value === 'string') return string(value)
     if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
     return value
   }
 
-  return { person, scalar }
+  return scalar
 }
 
-function setAt(target, path, value) {
-  const steps = path.split('/').slice(1)
-  const last = steps.pop()
-  let holder = target
-  for (const step of steps) holder = holder[step]
-  holder[last] = value
+function setAt(target, steps, value) {
+  const holder = steps.slice(0, -1).reduce((held, step) => held[step], target)
+  holder[steps.at(-1)] = value
 }
 
-// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
+// `draws` is fejkdata's output, as run.sh renders it.
 export function anonymize(document, draws) {
-  const { person, scalar } = replacer(document, draws)
+  const scalar = replacer(document, draws)
   const written = structuredClone(document)
-  walk(document, '', (value, path, kind) => {
-    if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
-    else if (kind === 'free') setAt(written, path, scalar(value))
+  walk(document, [], (value, steps, kind) => {
+    if (kind === 'free') setAt(written, steps, scalar(value))
   })
   return written
 }
 
 function structure(document) {
   const found = new Map()
-  walk(document, '', (value, path, kind) => {
-    found.set(path, { kind, value })
+  walk(document, [], (value, steps, kind) => {
+    found.set(`/${steps.join('/')}`, { kind, value })
   })
   return found
 }
@@ -217,12 +206,12 @@ function skeleton(text) {
 }
 
 function holdsSameShape(before, after) {
-  if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
+  if (before.kind === 'kept') return JSON.stringify(before.value) === JSON.stringify(after.value)
   if (typeof before.value !== typeof after.value) return false
-  return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
+  return typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
 }
 
-// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
+// The paths where `after` differs from `before` in anything but the tokens the anonymizer replaces.
 export function shapeDifferences(before, after) {
   const beforeValues = structure(before)
   const afterValues = structure(after)
@@ -231,11 +220,19 @@ export function shapeDifferences(before, after) {
     const first = beforeValues.get(path)
     const second = afterValues.get(path)
     return first === undefined || second === undefined || !holdsSameShape(first, second)
-  }).sort()
+  })
 }
 
 // The tokens of `before`'s replaced values that `after`'s replaced values still hold.
 export function leakedTokens(before, after) {
   const held = heldTokens(after)
-  return [...heldTokens(before)].filter((found) => held.has(found)).sort()
+  return [...heldTokens(before)].filter((found) => held.has(found))
+}
+
+// The anonymized document, or what keeps it out of the corpus: a slip here would publish real text, and nothing later could see it.
+export function anonymizeChecked(document, draws) {
+  const written = anonymize(document, draws)
+  const differences = shapeDifferences(document, written)
+  const leaks = leakedTokens(document, written)
+  return differences.length > 0 || leaks.length > 0 ? { differences, leaks, ok: false } : { ok: true, written }
 }

corpus-gathering/anonymize.test.mjs

@@ -1,12 +1,12 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
+
+import { anonymize, anonymizeChecked, leakedTokens, shapeDifferences } from './anonymize.mjs'
 
 const draws = {
   digits: ['7391', '2846'],
   hex: ['9f3ac07be1d4'],
   letters: ['qkxwzj'],
-  persons: ['Pat Smith', 'Lo Berg'],
   uuids: ['0190a1b2-c3d4-4e5f-8a9b-0c1d2e3f4a5b', '0190a1b2-c3d4-4e5f-8a9b-1c1d2e3f4a5c'],
   words: ['amber', 'clever', 'cobalt', 'merry', 'otter', 'cliff', 'tundra', 'pumice', 'lantern', 'harbour'],
 }
@@ -22,7 +22,7 @@ function textOf(document) {
 test('replaces each word and keeps the whitespace and punctuation around it', () => {
   const written = textOf(anonymize(paragraph({ text: '  Deploy *the* secret-name, now!\n', type: 'text' }), draws))
   assert.match(written, /^ {2}[A-Z][a-z]+ \*[a-z]+\* [a-z]+-[a-z]+, [a-z]+!\n$/)
-  for (const word of ['Deploy', 'the', 'secret', 'name', 'now']) assert.ok(!written.includes(word))
+  assert.doesNotMatch(written, /Deploy|the|secret|name|now/)
 })
 
 test('keeps a word in capitals in capitals', () => {
@@ -81,13 +81,13 @@ test('replaces a person, an account id and a uuid, keeping each shape', () => {
   assert.ok(!mention.attrs.text.includes('Person'))
 })
 
-test('keeps the version of every uuid it replaces', () => {
 for (const [version, original] of [['1', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], ['3', '6fa459ea-ee8a-3ca4-894e-db77e160355e'], ['4', '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e']]) {
+  test(`keeps the version of a v${version} uuid it replaces`, () => {
     const [mention] = anonymize(paragraph({ attrs: { localId: original }, type: 'mention' }), draws).content[0].content
     assert.match(mention.attrs.localId, new RegExp(`^[0-9a-f]{8}-[0-9a-f]{4}-${version}[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$`))
     assert.notEqual(mention.attrs.localId, original)
-  }
   })
+}
 
 test('keeps a uuid in capitals in capitals', () => {
   const [mention] = anonymize(paragraph({ attrs: { localId: '6FA459EA-EE8A-3CA4-894E-DB77E160355E' }, type: 'mention' }), draws).content[0].content
@@ -95,7 +95,7 @@ test('keeps a uuid in capitals in capitals', () => {
   assert.ok(/[A-F]/.test(mention.attrs.localId))
 })
 
-test('keeps a uuid inside a longer value a uuid of its version', () => {
+test('keeps the version of a uuid inside a longer value', () => {
   const inside = '[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}'
   const accountId = '712020:abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
   const href = 'https://wiki.example.org/page?id=abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
@@ -111,7 +111,7 @@ test('reports a uuid whose version changed', () => {
   assert.deepEqual(shapeDifferences(before, after), ['/content/0/content/0/attrs/id'])
 })
 
-test('keeps a date timestamp a number of the same length', () => {
+test('replaces a date timestamp with a number of the same length', () => {
   const [date] = anonymize(paragraph({ attrs: { timestamp: '1700000000000' }, type: 'date' }), draws).content[0].content
   assert.match(date.attrs.timestamp, /^[1-9]\d{12}$/)
   assert.notEqual(date.attrs.timestamp, '1700000000000')
@@ -160,9 +160,104 @@ test('never draws a uuid holding a part of an original one', () => {
   assert.equal(mention.attrs.localId, draws.uuids[0])
 })
 
-test('never runs out of draws, however many words a document holds', () => {
-  const text = Array.from({ length: 500 }, (_, index) => `word${'x'.repeat(index % 9)}`).join(' ')
-  assert.equal(textOf(anonymize(paragraph({ text, type: 'text' }), draws)).split(' ').length, 500)
+test('never runs out of draws when a document holds every word of the pool', () => {
+  const document = paragraph({ text: draws.words.join(' '), type: 'text' })
+  assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
+})
+
+test('ends and reports the leak when every draw holds the original', () => {
+  const document = paragraph({ text: '123', type: 'text' })
+  assert.deepEqual(leakedTokens(document, anonymize(document, { ...draws, digits: ['123'] })), ['123'])
+})
+
+test('replaces a one-word mention with one word', () => {
+  const [mention] = anonymize(paragraph({ attrs: { id: 'x', text: '@Cher' }, type: 'mention' }), draws).content[0].content
+  assert.match(mention.attrs.text, /^@[A-Z][a-z]+$/)
+})
+
+test('spells a name the same way in a mention and in text', () => {
+  const [mention, text] = anonymize(paragraph({ attrs: { text: '@Bob' }, type: 'mention' }, { text: 'Bob said', type: 'text' }), draws).content[0].content
+  assert.equal(mention.attrs.text.slice(1), text.text.split(' ')[0])
+})
+
+test('keeps every attribute naming a format, and the version', () => {
+  const attrs = Object.fromEntries(['__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode', 'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth', 'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType'].map((key) => [key, 'zebra']))
+  const [written] = anonymize({ content: [{ attrs: { ...attrs, language: 'c/c++' }, type: 'someNode' }], type: 'doc', version: 1 }, draws).content
+  assert.deepEqual(written.attrs, { ...attrs, language: 'c/c++' })
+  assert.equal(anonymize(paragraph(), draws).version, 1)
+})
+
+test('spells a word with the nearest length the pool holds, in its case', () => {
+  const pool = { ...draws, words: ['amber', 'clever', 'lantern'] }
+  assert.equal(textOf(anonymize(paragraph({ text: 'Alpha', type: 'text' }), pool)), 'Amber')
+  assert.equal(textOf(anonymize(paragraph({ text: 'channel', type: 'text' }), pool)), 'lantern')
+  assert.equal(textOf(anonymize(paragraph({ text: 'NINELONGS', type: 'text' }), pool)), 'LANTERN')
+  assert.equal(textOf(anonymize(paragraph({ text: 'tiny', type: 'text' }), pool)), 'amber')
+  assert.equal(textOf(anonymize(paragraph({ text: 'Bob', type: 'text' }), pool)), 'Amber')
+})
+
+test('spells a short word from the letters, keeping its case', () => {
+  assert.equal(textOf(anonymize(paragraph({ text: 'Al', type: 'text' }), draws)), 'Qk')
+})
+
+test('reads a word with an accent as a word', () => {
+  assert.equal(textOf(anonymize(paragraph({ text: 'caf\u00e9', type: 'text' }), { ...draws, words: ['amber'] })), 'amber')
+})
+
+test('spells a word of a caseless script in small letters', () => {
+  assert.match(textOf(anonymize(paragraph({ text: '\u6f22\u5b57 \u6f22\u5b57\u6f22', type: 'text' }), draws)), /^[a-z]+ [a-z]+$/)
+})
+
+test('keeps letters and digits in their places in a mixed token', () => {
+  assert.match(textOf(anonymize(paragraph({ text: 'beta2 2beta a1x xa1', type: 'text' }), draws)), /^[a-z]{4}\d \d[a-z]{4} [a-z]\d[a-z] [a-z]{2}\d$/)
+})
+
+test('spells a hex token from the hex draws', () => {
+  assert.equal(textOf(anonymize(paragraph({ text: '3f2a', type: 'text' }), draws)), '9f3a')
+})
+
+test('keeps a combining mark in a mixed token', () => {
+  assert.match(textOf(anonymize(paragraph({ text: 'a1e\u0301', type: 'text' }), draws)), /^[a-z]\d[a-z]\u0301$/)
+})
+
+test('keeps a scheme only before its colon', () => {
+  assert.match(textOf(anonymize(paragraph({ text: 'file:///x ftp://x http://x https://x mailto:x tel:x', type: 'text' }), draws)), /^file:\/\/\/[a-z] ftp:\/\/[a-z] http:\/\/[a-z] https:\/\/[a-z] mailto:[a-z] tel:[a-z]$/)
+  assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'secret: value', type: 'text' }), draws)), /secret/)
+  assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'file https', type: 'text' }), draws)), /file|https/)
+})
+
+test('does not call a kept scheme a leak', () => {
+  const document = paragraph({ marks: [{ attrs: { href: 'https://example.org' }, type: 'link' }], text: 'x', type: 'text' })
+  assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
+})
+
+test('keeps a boolean and a null, and replaces the strings beside them', () => {
+  const [node] = anonymize(paragraph({ attrs: { flag: true, list: [false, 'secret'], none: null }, type: 'someNode' }), draws).content[0].content
+  assert.equal(node.attrs.flag, true)
+  assert.equal(node.attrs.list[0], false)
+  assert.notEqual(node.attrs.list[1], 'secret')
+  assert.equal(node.attrs.none, null)
+})
+
+test('replaces a string among a content array, and keeps a null there', () => {
+  const [held] = anonymize({ content: [{ content: ['secret', null], type: 'someNode' }], type: 'doc', version: 1 }, draws).content
+  assert.notEqual(held.content[0], 'secret')
+  assert.equal(held.content[1], null)
+})
+
+test('replaces a value whose key holds a slash, and the value beside it', () => {
+  const [node] = anonymize(paragraph({ attrs: { a: { b: 4242 }, 'a/b': 4243 }, type: 'someNode' }), draws).content[0].content
+  assert.notEqual(node.attrs['a/b'], 4243)
+  assert.notEqual(node.attrs.a.b, 4242)
+  assert.deepEqual(Object.keys(node.attrs), ['a', 'a/b'])
+})
+
+test('gives a number no leading zero, so it keeps its length', () => {
+  assert.equal(textOf(anonymize(paragraph({ text: '2024 0123', type: 'text' }), { ...draws, digits: ['00000000'] })), '1000 0000')
+})
+
+test('moves on through the digits between numbers', () => {
+  assert.equal(textOf(anonymize(paragraph({ text: '1234 5678', type: 'text' }), draws)), '7391 2846')
 })
 
 test('finds no difference between a document and its anonymized form', () => {
@@ -171,6 +266,17 @@ test('finds no difference between a document and its anonymized form', () => {
   assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
 })
 
+test('finds no difference in a kept array or a replaced number', () => {
+  const document = paragraph({ attrs: { colwidth: [100, 200], pageId: 42871 }, type: 'tableCell' })
+  assert.deepEqual(shapeDifferences(document, anonymize(document, draws)), [])
+})
+
+test('reports a number turned into a string, an added key and a removed key', () => {
+  const before = paragraph({ attrs: { kept: 12, removed: 1 }, type: 'someNode' })
+  const after = paragraph({ attrs: { added: 1, kept: '12' }, type: 'someNode' })
+  assert.deepEqual(shapeDifferences(before, after).sort(), ['/content/0/content/0/attrs/added', '/content/0/content/0/attrs/kept', '/content/0/content/0/attrs/removed'])
+})
+
 test('reports a changed structure with its path', () => {
   const before = paragraph({ marks: [{ type: 'em' }], text: 'a b', type: 'text' })
   const after = paragraph({ marks: [{ type: 'strong' }], text: 'c-d', type: 'text' })
@@ -183,6 +289,21 @@ test('reports a word the output still holds', () => {
   assert.deepEqual(leakedTokens(before, after), ['private'])
 })
 
+test('counts a token of three letters as a leak, and a shorter one not', () => {
+  const document = paragraph({ text: 'Bob Al', type: 'text' })
+  assert.deepEqual(leakedTokens(document, document), ['bob'])
+})
+
+test('gives the anonymized document when it holds the shape and no original token', () => {
+  const document = paragraph({ text: 'private word', type: 'text' })
+  assert.deepEqual(anonymizeChecked(document, draws), { ok: true, written: anonymize(document, draws) })
+})
+
+test('keeps out a document whose output still holds an original token', () => {
+  const document = paragraph({ text: '123', type: 'text' })
+  assert.deepEqual(anonymizeChecked(document, { ...draws, digits: ['123'] }), { differences: [], leaks: ['123'], ok: false })
+})
+
 test('does not call a kept attribute value a leak', () => {
   const document = { attrs: {}, content: [{ attrs: { language: 'python' }, content: [{ text: 'python', type: 'text' }], type: 'codeBlock' }], type: 'doc', version: 1 }
   assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])

corpus-gathering/fetch.mjs

@@ -1,22 +1,28 @@
-// Usage: node fetch.mjs <sites.json> <raw folder>. The folder stays outside the repo: its documents hold the sites' real text.
-import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
 import { setTimeout as delay } from 'node:timers/promises'
 
-// Robots.txt on these sites asks for one second between requests.
-const crawlDelay = 1000
-const pagesPerSource = 20
-const [sitesPath, rawFolder] = process.argv.slice(2)
-const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
-const documentsFolder = join(rawFolder, 'documents')
-mkdirSync(documentsFolder, { recursive: true })
+import { jiraUrl, wikiUrl } from './source-urls.mjs'
 
-async function getJson(url) {
+// Robots.txt on these sites allows these paths and asks for one second between requests.
+const crawlDelay = 1000
+const defaultRetryAfter = 60
+const [sitesPath, rawFolder] = process.argv.slice(2)
+// Keeps a fetch to minutes.
+const pagesPerListing = 20
+const retries = 5
+const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
+const partialFolder = join(rawFolder, 'documents.partial')
+rmSync(partialFolder, { force: true, recursive: true })
+mkdirSync(partialFolder, { recursive: true })
+
+async function getJson(url, retriesLeft = retries) {
   await delay(crawlDelay)
   const response = await fetch(url, { headers: { Accept: 'application/json', 'User-Agent': 'adf-codec-corpus-gathering (https://github.com/larvit/adf-codec)' } })
-  if (response.status === 429) {
-    await delay(Number(response.headers.get('retry-after') ?? 60) * 1000)
-    return getJson(url)
+  if (response.status === 429 && retriesLeft > 0) {
+    const asked = Number(response.headers.get('retry-after'))
+    await delay((asked > 0 ? asked : defaultRetryAfter) * 1000)
+    return getJson(url, retriesLeft - 1)
   }
   if (!response.ok) throw new Error(`${url} answered ${response.status}`)
   return response.json()
@@ -26,18 +32,18 @@ let written = 0
 
 function keep(name, source, document) {
   if (document?.type !== 'doc' || !Array.isArray(document.content) || document.content.length === 0) return
-  writeFileSync(join(documentsFolder, `${name}.json`), JSON.stringify({ document, source }))
+  writeFileSync(join(partialFolder, `${name}.json`), JSON.stringify({ document, retrieved: new Date().toISOString(), source }))
   written += 1
 }
 
 async function fetchConfluence(site, collection) {
   const base = `https://${site}.atlassian.net`
   let next = `/wiki/api/v2/${collection}?limit=100&body-format=atlas_doc_format`
-  for (let page = 0; next !== undefined && page < pagesPerSource; page += 1) {
+  for (let page = 0; next !== undefined && page < pagesPerListing; page += 1) {
     const answer = await getJson(`${base}${next}`)
     for (const result of answer.results) {
       const body = result.body?.atlas_doc_format?.value
-      if (body) keep(`${site}-wiki-${collection}-${result.id}`, `${base}/wiki/api/v2/${collection}/${result.id}`, JSON.parse(body))
+      if (body) keep(`${site}-wiki-${collection}-${result.id}`, wikiUrl(base, result._links.webui), JSON.parse(body))
     }
     next = answer._links?.next
   }
@@ -46,18 +52,19 @@ async function fetchConfluence(site, collection) {
 async function fetchJira(site) {
   const base = `https://${site}.atlassian.net`
   let token
-  for (let page = 0; page < pagesPerSource; page += 1) {
+  for (let page = 0; page < pagesPerListing; page += 1) {
     const query = new URLSearchParams({ fields: 'comment,description', jql: 'created >= "2015-01-01" ORDER BY created DESC', maxResults: '100', ...(token ? { nextPageToken: token } : {}) })
     const answer = await getJson(`${base}/rest/api/3/search/jql?${query}`)
     for (const issue of answer.issues) {
-      keep(`${site}-jira-${issue.key}-description`, `${base}/rest/api/3/issue/${issue.key}?fields=description`, issue.fields.description)
-      for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, `${base}/rest/api/3/issue/${issue.key}/comment/${comment.id}`, comment.body)
+      keep(`${site}-jira-${issue.key}-description`, jiraUrl(base, issue.key), issue.fields.description)
+      for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, jiraUrl(base, issue.key, comment.id), comment.body)
     }
     token = answer.nextPageToken
     if (token === undefined) break
   }
 }
 
+const failures = []
 for (const { products, site } of sites) {
   for (const product of Object.keys(products)) {
     try {
@@ -69,7 +76,14 @@ for (const { products, site } of sites) {
       }
     } catch (error) {
       console.error(`${site} ${product}: ${error.message}`)
+      failures.push(`${site} ${product}`)
     }
     console.error(`${site} ${product}: ${written} documents so far`)
   }
 }
+
+if (failures.length > 0) {
+  console.error(`No corpus source is complete without: ${failures.join(', ')}. Fetch again, or take the site out of sites.json.`)
+  process.exit(1)
+}
+renameSync(partialFolder, join(rawFolder, 'documents'))

corpus-gathering/run.sh

@@ -1,12 +1,12 @@
 #!/bin/sh
-# Usage: corpus-gathering/run.sh <raw folder>. Keep the raw folder outside the repo: its documents hold the sites' real text.
 set -eu
-here=$(cd "$(dirname "$0")" && pwd)
-raw=$(mkdir -p "$1" && cd "$1" && pwd)
+[ $# -eq 1 ] || { echo 'Usage: corpus-gathering/run.sh <raw folder>' >&2; exit 2; }
+here=$(cd "$(dirname "$0")" && pwd -P)
+raw=$(mkdir -p "$1" && cd "$1" && pwd -P)
+case "$raw/" in "$(cd "$here/.." && pwd -P)"/*) echo "run.sh: keep the raw folder outside the repo: its documents hold the sites' real text" >&2; exit 2 ;; esac
 fejkdata_image=larvit/fejkdata:0.2.0
 node_image=node:26.13.0-alpine3.24
 
-# Takes how to mount the repo, then the command.
 in_node() {
   repo_mount=$1
   shift
@@ -25,7 +25,6 @@ mkdir -p "$raw/draws"
 draw digits 5000 '{digits(32)}'
 draw hex 5000 '{hex(32)}'
 draw letters 5000 '{lower(32)}'
-draw persons 2000 '{/en_US.person}'
 draw uuids 20000 '{/misc.uuid}'
 draw words 20000 '{/en_US.word}'
 

corpus-gathering/run.test.mjs

@@ -0,0 +1,25 @@
+import assert from 'node:assert/strict'
+import { spawnSync } from 'node:child_process'
+import { test } from 'node:test'
+
+const repo = new URL('..', import.meta.url).pathname
+
+function run(...args) {
+  return spawnSync('sh', ['corpus-gathering/run.sh', ...args], { cwd: repo, encoding: 'utf8' })
+}
+
+test('refuses a raw folder inside the repo', () => {
+  const { status, stderr } = run('corpus')
+  assert.equal(status, 2)
+  assert.match(stderr, /outside the repo/)
+})
+
+test('refuses the repo itself as the raw folder', () => {
+  assert.equal(run('.').status, 2)
+})
+
+test('names its argument when it is missing', () => {
+  const { status, stderr } = run()
+  assert.equal(status, 2)
+  assert.match(stderr, /Usage: corpus-gathering\/run\.sh <raw folder>/)
+})

corpus-gathering/select.mjs

@@ -23,7 +23,7 @@ function collect(node, parentType, found) {
   })
 }
 
-// What a document shows the corpus: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
+// A document's features: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
 export function features(document) {
   const found = new Set()
   collect(document, undefined, found)
@@ -47,7 +47,7 @@ function bestCandidate(candidates, covered) {
       showsType ||= isType(feature)
     }
     const score = added / candidate.size
-    if (added > 0 && (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore)) {
+    if (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore) {
       best = candidate
       bestScore = score
       bestShowsType = showsType
@@ -60,13 +60,11 @@ function bestCandidate(candidates, covered) {
 export function selectDocuments(candidates, limit) {
   const covered = new Set()
   const chosen = []
-  let remaining = candidates
   while (chosen.length < limit) {
-    const best = bestCandidate(remaining, covered)
+    const best = bestCandidate(candidates, covered)
     if (best === undefined) break
     chosen.push(best)
     for (const feature of best.features) covered.add(feature)
-    remaining = remaining.filter((candidate) => candidate !== best)
   }
   return chosen
 }

corpus-gathering/select.test.mjs

@@ -1,5 +1,6 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
+
 import { features, selectDocuments } from './select.mjs'
 
 function paragraph(...content) {
@@ -11,10 +12,23 @@ test('names node and mark types, attribute keys and nesting', () => {
   for (const feature of ['node:doc', 'node:paragraph', 'node:text', 'mark:link', 'attr:link.href', 'child:paragraph>text']) assert.ok(found.has(feature), feature)
 })
 
+test('names a node attribute', () => {
+  assert.ok(features(paragraph({ attrs: { level: 2 }, type: 'heading' })).has('attr:heading.level'))
+})
+
+test('names no mark for a node without marks, and no parent for the document', () => {
+  assert.deepEqual([...features({ type: 'doc' })], ['node:doc'])
+})
+
 test('names marks held together', () => {
   assert.ok(features(paragraph({ marks: [{ type: 'strong' }, { type: 'em' }], text: 'a', type: 'text' })).has('marks:em+strong'))
 })
 
+test('names no marks held together for one mark, or for marks on a node other than text', () => {
+  const found = [...features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'strong' }, { type: 'em' }], type: 'mediaInline' }))]
+  assert.deepEqual(found.filter((feature) => feature.startsWith('marks:')), [])
+})
+
 test('names bold beside a code span and italics beside a hard break', () => {
   const found = features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'code' }], text: 'b', type: 'text' }, { type: 'hardBreak' }, { marks: [{ type: 'em' }], text: 'c', type: 'text' }))
   assert.ok(found.has('beside:text[strong]|text[code]'))
@@ -34,6 +48,18 @@ test('takes a candidate showing a new node or mark type ahead of any other', ()
   assert.deepEqual(selectDocuments([cheap, rare], 1), [rare])
 })
 
+test('takes a candidate showing a new type wherever it stands among the candidates', () => {
+  const rare = { features: new Set(['mark:breakout']), size: 1000 }
+  const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
+  assert.deepEqual(selectDocuments([rare, cheap], 1), [rare])
+})
+
+test('counts a feature naming a type only by its opening', () => {
+  const rare = { features: new Set(['beside:node:x']), size: 1000 }
+  const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
+  assert.deepEqual(selectDocuments([rare, cheap], 1), [cheap])
+})
+
 test('stops at the limit', () => {
   const candidates = [{ features: new Set(['a']), size: 1 }, { features: new Set(['b']), size: 1 }]
   assert.equal(selectDocuments(candidates, 1).length, 1)

corpus-gathering/sites.json

@@ -1,30 +1,4 @@
 [
-  {
-    "products": {
-      "jira": {
-        "licence": "CC-BY-3.0",
-        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
-      },
-      "wiki": {
-        "licence": "CC-BY-SA-4.0",
-        "licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
-      }
-    },
-    "site": "lf-onap"
-  },
-  {
-    "products": {
-      "jira": {
-        "licence": "CC-BY-3.0",
-        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
-      },
-      "wiki": {
-        "licence": "CC-BY-3.0",
-        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
-      }
-    },
-    "site": "lf-opendaylight"
-  },
   {
     "products": {
       "jira": {
@@ -58,7 +32,7 @@
         "licenceSource": "https://www.linuxfoundation.org/legal/terms"
       }
     },
-    "site": "lf-hyperledger"
+    "site": "lf-edgexfoundry"
   },
   {
     "products": {
@@ -67,6 +41,32 @@
         "licenceSource": "https://www.linuxfoundation.org/legal/terms"
       }
     },
-    "site": "lf-edgexfoundry"
+    "site": "lf-hyperledger"
+  },
+  {
+    "products": {
+      "jira": {
+        "licence": "CC-BY-3.0",
+        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
+      },
+      "wiki": {
+        "licence": "CC-BY-SA-4.0",
+        "licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
+      }
+    },
+    "site": "lf-onap"
+  },
+  {
+    "products": {
+      "jira": {
+        "licence": "CC-BY-3.0",
+        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
+      },
+      "wiki": {
+        "licence": "CC-BY-3.0",
+        "licenceSource": "https://www.linuxfoundation.org/legal/terms"
+      }
+    },
+    "site": "lf-opendaylight"
   }
 ]

corpus-gathering/source-urls.mjs

@@ -0,0 +1,10 @@
+// The pages a person reads, which name each document's creator. A wiki's web link also holds the page's title, and can hold an attachment's name, which the repo must not.
+export function jiraUrl(base, key, commentId) {
+  return `${base}/browse/${key}${commentId === undefined ? '' : `?focusedCommentId=${commentId}`}`
+}
+
+export function wikiUrl(base, webLink) {
+  const [path, query] = webLink.split('?')
+  const commentId = new URLSearchParams(query).get('focusedCommentId')
+  return `${base}/wiki${path.replace(/(\/\d+)\/[^/]+$/, '$1')}${commentId === null ? '' : `?focusedCommentId=${commentId}`}`
+}

corpus-gathering/source-urls.test.mjs

@@ -0,0 +1,32 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { jiraUrl, wikiUrl } from './source-urls.mjs'
+
+const base = 'https://site.atlassian.net'
+
+test('links a Jira issue, and a comment on it', () => {
+  assert.equal(jiraUrl(base, 'KEY-1'), 'https://site.atlassian.net/browse/KEY-1')
+  assert.equal(jiraUrl(base, 'KEY-1', '42'), 'https://site.atlassian.net/browse/KEY-1?focusedCommentId=42')
+})
+
+test('links a wiki page without the title its web link holds', () => {
+  assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13')
+})
+
+test('links a wiki comment without the title of its page', () => {
+  assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13?focusedCommentId=42')
+})
+
+test('links a comment on a blog post without its title', () => {
+  assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/blog/2020/01/02/13?focusedCommentId=42')
+})
+
+test('wikiUrl drops a preview, which names an attached file', () => {
+  assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?preview=%2F13%2F14%2FSecret.jpeg'), `${base}/wiki/spaces/AK/blog/2020/01/02/13`)
+  assert.equal(wikiUrl(base, '/spaces/AK/overview?preview=%2F13%2F14%2FSecret.jpeg&focusedCommentId=42'), `${base}/wiki/spaces/AK/overview?focusedCommentId=42`)
+})
+
+test('wikiUrl keeps a space overview, which holds no title', () => {
+  assert.equal(wikiUrl(base, '/spaces/AK/overview'), `${base}/wiki/spaces/AK/overview`)
+})

corpus-gathering/write-corpus.mjs

@@ -1,9 +1,10 @@
-// Usage: node write-corpus.mjs <sites.json> <raw folder> <documents folder> <sources file>. The raw folder holds fetch.mjs's documents and run.sh's draws.
-import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
-import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
+import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+
+import { anonymizeChecked } from './anonymize.mjs'
 import { features, selectDocuments } from './select.mjs'
 
+// Selection prefers small documents, which keeps the corpus near 660 KB.
 const documentLimit = 150
 const largestDocument = 64 * 1024
 const [sitesPath, rawFolder, corpusFolder, sourcesPath] = process.argv.slice(2)
@@ -23,22 +24,19 @@ function canonical(value) {
 }
 
 const licences = new Map(JSON.parse(readFileSync(sitesPath, 'utf8')).flatMap(({ products, site }) => Object.entries(products).map(([product, licence]) => [`${site}-${product}`, licence])))
-const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), persons: lines('persons'), uuids: lines('uuids'), words: lines('words') }
+const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), uuids: lines('uuids'), words: lines('words') }
 const documentsFolder = join(rawFolder, 'documents')
 const candidates = readdirSync(documentsFolder).flatMap((file) => {
-  const path = join(documentsFolder, file)
-  const text = readFileSync(path, 'utf8')
+  const text = readFileSync(join(documentsFolder, file), 'utf8')
   if (text.length > largestDocument) return []
-  const { document, source } = JSON.parse(text)
+  const { document, retrieved, source } = JSON.parse(text)
   const name = file.slice(0, -'.json'.length)
-  const written = anonymize(document, draws)
-  const differences = shapeDifferences(document, written)
-  const leaks = leakedTokens(document, written)
-  if (differences.length > 0 || leaks.length > 0) {
-    console.error(`${name} left out: ${differences.length} shape differences, ${leaks.length} tokens still held`)
+  const checked = anonymizeChecked(document, draws)
+  if (!checked.ok) {
+    console.error(`${name} left out: ${checked.differences.length} shape differences, ${checked.leaks.length} tokens still held`)
     return []
   }
-  return [{ features: features(document), name, retrieved: statSync(path).mtime.toISOString().slice(0, 10), size: text.length, source, written }]
+  return [{ features: features(document), name, retrieved, size: text.length, source, written: checked.written }]
 })
 
 rmSync(corpusFolder, { force: true, recursive: true })

corpus/README.md

@@ -13,10 +13,13 @@ One directory per contract kind:
 - `real-payloads/` — `<name>.json`: ADF Atlassian's editor wrote, round-tripped ADF→markdown→ADF
   and ADF→HTML→ADF. No expected markdown or HTML.
 - `real-world/` — `<name>.json`: anonymized documents from public Jira and Confluence sites,
-  round-tripped as `real-payloads/` is. `real-world-sources.json` names each one's source, licence
-  and retrieval date. `corpus-gathering/run.sh <raw folder>` writes both, and the raw folder stays
-  outside the repo because its documents hold the sites' real text
+  round-tripped as `real-payloads/` is. `real-world-sources.json` names each one's original page,
+  licence and retrieval time. `corpus-gathering/run.sh <folder outside the repo>` fetches the
+  documents into that folder, then replaces the whole set and that file. It needs Docker and
+  network, and takes minutes
   (`docs/decisions.md` §The real-world corpus holds anonymized documents from sites whose licence it records).
+  `typesNoFetchedDocumentHolds` in `src/conformance/corpus.test.ts` lists the node types these
+  documents don't show.
 - `commonmark-spec/` — the CommonMark suite run against `commonMarkToAdf` by three checks.
   `spec.json` is the suite; `refusals.json` pins each refusing example to its error `code`;
   `exceptions.json` pins each known divergence by `check`, `example`, `kind` and the exact
@@ -30,3 +33,12 @@ suite, byte-exact from [spec.commonmark.org](https://spec.commonmark.org/0.31.2/
 (CommonMark 0.31.2, © John MacFarlane,
 [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/)), and is not re-serialized by the
 corpus gate.
+
+`real-world/` isn't under this repository's MIT licence. Each document adapts the work its entry's
+`url` names, replacing its text, names, ids and URLs, and that page credits its creators, whom the
+documents don't name. It stays under the licence its entry names,
+[CC-BY-3.0](https://creativecommons.org/licenses/by/3.0/) or
+[CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/), without warranty, and an adaptation
+of a CC-BY-SA-4.0 document is shared under that licence. Only the ONAP wiki states its own licence.
+The others rest on the Linux Foundation's terms of use, which don't name these sites, so their
+licence is our reading of those terms.

corpus/real-world/lf-akraino-wiki-footer-comments-13690356.json

@@ -5,7 +5,7 @@
         {
           "attrs": {
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Lavender Rosemary"
           },
           "type": "mention"
         },
@@ -80,7 +80,7 @@
     {
       "content": [
         {
-          "text": "E lichen spire ivy ash ulcn9, wn gorge qu trellis ash moss morning. Nc sage i wren dusk, um ash lavender olive ivy cinder modern? Rp ash olive ivy rosemary modern, E beach ivy emerald pillar ash cedar ivy ivy 9 brook ash sycamore. E ash dapple ivy able wd ash wren 9 brook.",
+          "text": "E lichen spire ivy ash ulcn9, wn gorge qu trellis ash moss morning. Nc sage i wren dusk, um ash sycamore olive ivy cinder modern? Rp ash olive ivy rosemary modern, E beach ivy emerald pillar ash cedar ivy ivy 9 brook ash mountain. E ash dapple ivy able wd ash wren 9 brook.",
           "type": "text"
         }
       ],

corpus/real-world/lf-akraino-wiki-footer-comments-13699635.json

@@ -5,12 +5,12 @@
         {
           "attrs": {
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@wren HI"
           },
           "type": "mention"
         },
         {
-          "text": "  Hi glade Wren Trellis ivy morning pine.",
+          "text": "  Rz glade Pine Trellis ivy morning dawn.",
           "type": "text"
         }
       ],
@@ -22,7 +22,7 @@
       },
       "content": [
         {
-          "text": "3. Willow lavender dawn",
+          "text": "3. Willow lavender moss",
           "type": "text"
         }
       ],
@@ -31,7 +31,7 @@
     {
       "content": [
         {
-          "text": "Ash lavender rz moss yz rosemary ash sycamore ivy ash",
+          "text": "Ash lavender yz sage eu rosemary ash sycamore ivy ash",
           "type": "text"
         },
         {
@@ -42,12 +42,12 @@
           "marks": [
             {
               "attrs": {
-                "href": "https://eu-emerald.mountain.ash/wren/shimmer/LC/SAGE+Fountain+Gorge"
+                "href": "https://lc-emerald.mountain.ash/dusk/shimmer/NW/WREN+Fountain+Gorge"
               },
               "type": "link"
             }
           ],
-          "text": "SAGE rosemary spire",
+          "text": "WREN rosemary spire",
           "type": "text"
         },
         {
@@ -80,7 +80,7 @@
     {
       "content": [
         {
-          "text": "Ash lichen dusk ash ash sterling able ash lantern IVY ash sapling ash Pine. Dawn nw glimmer:",
+          "text": "Ash lichen able ash ash sterling pine ash lantern IVY ash sapling ash Dawn. Reef nq glimmer:",
           "type": "text"
         }
       ],
@@ -91,7 +91,7 @@
         {
           "content": [
             {
-              "text": "reef -N /ivy/onyx \"",
+              "text": "onyx -U /ivy/fern \"",
               "type": "text"
             },
             {
@@ -130,7 +130,7 @@
               "type": "textColor"
             }
           ],
-          "text": "Wren Trellis",
+          "text": "Pine Trellis",
           "type": "text"
         }
       ],

corpus/real-world/lf-akraino-wiki-footer-comments-13700648.json

@@ -1,58 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": "“",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "attrs": {
-        "level": 2
-      },
-      "content": [
-        {
-          "text": "Lavender wren",
-          "type": "text"
-        }
-      ],
-      "type": "heading"
-    },
-    {
-      "content": [
-        {
-          "attrs": {
-            "extensionKey": "inline-external-image",
-            "extensionType": "com.atlassian.confluence.migration",
-            "parameters": {
-              "alt": "ash pine",
-              "height": "",
-              "src": "http://rosemary:3369/trellis/sycamore/rosemary/rosemary_wren.ivy",
-              "width": ""
-            }
-          },
-          "type": "inlineExtension"
-        },
-        {
-          "text": "”",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "content": [
-        {
-          "text": "Ash dawn hi moss",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-akraino-wiki-footer-comments-13701432.json

@@ -1,74 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": "H ",
-          "type": "text"
-        },
-        {
-          "text": "ir willow zy trellis ivy lavender ze rosemary ivy sycamore.  Morning glade emerald ivy ",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "attrs": {
-                "color": "#172b4d"
-              },
-              "type": "textColor"
-            }
-          ],
-          "text": " ",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "attrs": {
-                "href": "https://beacon.shimmer.ash/u/spire/gorge/3l-ash-olive-lichen"
-              },
-              "type": "link"
-            }
-          ],
-          "text": "https://beacon.shimmer.ash/u/spire/gorge/3l-ash-olive-lichen",
-          "type": "text"
-        },
-        {
-          "text": "  rosemary, H cn ivy ash w wren beach ",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "type": "strong"
-            },
-            {
-              "attrs": {
-                "color": "#172b4d"
-              },
-              "type": "textColor"
-            }
-          ],
-          "text": "cinder_nq.un.",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "attrs": {
-                "color": "#172b4d"
-              },
-              "type": "textColor"
-            }
-          ],
-          "text": "  Cedar ash H pine dawn wren? Modern.",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-akraino-wiki-pages-13670209.json

@@ -0,0 +1,168 @@
+{
+  "content": [
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Lavender",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "content": [
+        {
+          "content": [
+            {
+              "content": [
+                {
+                  "attrs": {
+                    "id": "3a41c5441d641d8d074ef44f"
+                  },
+                  "type": "mention"
+                }
+              ],
+              "type": "paragraph"
+            }
+          ],
+          "type": "listItem"
+        },
+        {
+          "content": [
+            {
+              "content": [
+                {
+                  "attrs": {
+                    "id": "7f31bac87cc35769000bf04a"
+                  },
+                  "type": "mention"
+                }
+              ],
+              "type": "paragraph"
+            }
+          ],
+          "type": "listItem"
+        },
+        {
+          "content": [
+            {
+              "content": [
+                {
+                  "attrs": {
+                    "id": "336999:a41c5441-d641-4d8d-b4ef-44f7f31bac87"
+                  },
+                  "type": "mention"
+                }
+              ],
+              "type": "paragraph"
+            }
+          ],
+          "type": "listItem"
+        }
+      ],
+      "type": "bulletList"
+    },
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Willow",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "content": [
+        {
+          "content": [
+            {
+              "content": [
+                {
+                  "text": "Trellis wren ivy pine ash beacon.",
+                  "type": "text"
+                }
+              ],
+              "type": "paragraph"
+            }
+          ],
+          "type": "listItem"
+        }
+      ],
+      "type": "bulletList"
+    },
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Morning",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "attrs": {
+        "localId": ""
+      },
+      "content": [
+        {
+          "attrs": {
+            "localId": "1",
+            "state": "TODO"
+          },
+          "content": [
+            {
+              "text": "Glade rosemary ash emerald pine ash beacon. Hi dawn pine ash, r shimmer lantern sycamore moss zy rosemary sapling mountain. Ivy lichen ash fountain mountain, ash glimmer Cinder'z sterling eu spire splendid sage MODERN Pillar wren.",
+              "type": "text"
+            }
+          ],
+          "type": "taskItem"
+        },
+        {
+          "attrs": {
+            "localId": "74",
+            "state": "TODO"
+          },
+          "content": [
+            {
+              "text": "Dusk dapple gorge lc olive ash juniper beacon nw pine ash",
+              "type": "text"
+            }
+          ],
+          "type": "taskItem"
+        }
+      ],
+      "type": "taskList"
+    },
+    {
+      "attrs": {
+        "layout": "align-start",
+        "width": 431,
+        "widthType": "pixel"
+      },
+      "content": [
+        {
+          "attrs": {
+            "collection": "fountain-15856850",
+            "height": 672,
+            "id": "c3576900-0bf0-44ac-9065-5a98847af7ed",
+            "type": "file",
+            "width": 1161
+          },
+          "type": "media"
+        }
+      ],
+      "type": "mediaSingle"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-anuket-jira-BAROMETER-98-description.json

@@ -0,0 +1,93 @@
+{
+  "content": [
+    {
+      "content": [
+        {
+          "text": " ",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "content": [
+        {
+          "text": " ",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "isNumberColumnEnabled": false,
+        "layout": "default"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "attrs": {},
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "wren.willow.lavender",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "tableCell"
+            }
+          ],
+          "type": "tableRow"
+        },
+        {
+          "content": [
+            {
+              "attrs": {},
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "wren.willow.rosemary",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "tableCell"
+            }
+          ],
+          "type": "tableRow"
+        },
+        {
+          "content": [
+            {
+              "attrs": {},
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "wren.willow.glade",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "tableCell"
+            }
+          ],
+          "type": "tableRow"
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-anuket-jira-VINEPERF-349-comment-10743.json

@@ -10,7 +10,7 @@
           "attrs": {
             "accessLevel": "",
             "id": "33699:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Lavender Rosemary"
           },
           "type": "mention"
         },
@@ -25,16 +25,16 @@
           "marks": [
             {
               "attrs": {
-                "href": "http://lavender.spire.ash/rosemary/pine/sycamore/gorge.dawn"
+                "href": "http://sycamore.spire.ash/rosemary/pine/mountain/gorge.dawn"
               },
               "type": "link"
             }
           ],
-          "text": "http://lavender.spire.ash/rosemary/pine/sycamore/gorge.dawn",
+          "text": "http://sycamore.spire.ash/rosemary/pine/mountain/gorge.dawn",
           "type": "text"
         },
         {
-          "text": " olive moss lichen trellis ivy cinder rosemary",
+          "text": " olive moss lichen trellis ivy cinder fountain",
           "type": "text"
         }
       ],

corpus/real-world/lf-anuket-jira-VINEPERF-504-comment-10494.json

@@ -1,48 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": " ",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "content": [
-        {
-          "text": " ",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "attrs": {
-        "language": "java"
-      },
-      "type": "codeBlock"
-    },
-    {
-      "content": [
-        {
-          "text": " ",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "content": [
-        {
-          "text": " ",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-anuket-wiki-pages-158728216.json

@@ -14,7 +14,7 @@
           "type": "emoji"
         },
         {
-          "text": " EU",
+          "text": " eu",
           "type": "text"
         }
       ],
@@ -24,7 +24,7 @@
       "content": [
         {
           "attrs": {
-            "text": "LAVENDER。LC“//”TRELLIS，LC“/NW”ROSEMARY。"
+            "text": "lavender。lc“//”trellis，lc“/nw”rosemary。"
           },
           "type": "placeholder"
         }
@@ -55,14 +55,14 @@
     },
     {
       "attrs": {
-        "title": "NQ WREN"
+        "title": "nq wren"
       },
       "content": [
         {
           "content": [
             {
               "attrs": {
-                "text": "SYCAMORE。"
+                "text": "sycamore。"
               },
               "type": "placeholder"
             }
@@ -89,7 +89,7 @@
           "type": "emoji"
         },
         {
-          "text": " WILLOW",
+          "text": " willow",
           "type": "text"
         }
       ],
@@ -119,14 +119,14 @@
     },
     {
       "attrs": {
-        "title": "NQ WREN"
+        "title": "nq wren"
       },
       "content": [
         {
           "content": [
             {
               "attrs": {
-                "text": "MOUNTAIN。"
+                "text": "mountain。"
               },
               "type": "placeholder"
             }
@@ -153,7 +153,7 @@
           "type": "emoji"
         },
         {
-          "text": " MORNING",
+          "text": " morning",
           "type": "text"
         }
       ],
@@ -165,7 +165,7 @@
       },
       "content": [
         {
-          "text": "UN 3",
+          "text": "un 3",
           "type": "text"
         }
       ],
@@ -195,7 +195,7 @@
       },
       "content": [
         {
-          "text": "UN 6",
+          "text": "un 6",
           "type": "text"
         }
       ],

corpus/real-world/lf-hyperledger-wiki-footer-comments-17433582.json

@@ -9,14 +9,14 @@
         {
           "attrs": {
             "id": "336999:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Glade Willow"
           },
           "type": "mention"
         },
         {
           "attrs": {
             "id": "336999:c3576900-0bf0-44ac-9065-5a98847af7ed",
-            "text": "@Eugene McCarthy"
+            "text": "@Trellis Beacon"
           },
           "type": "mention"
         }
@@ -26,7 +26,7 @@
     {
       "content": [
         {
-          "text": "R wren zy glade zy willow ivy pine lavender dawn R moss ze sage zy wren beacon",
+          "text": "R wren zy spire zy lichen ivy pine lavender dawn R moss ze sage zy wren cinder",
           "type": "text"
         }
       ],
@@ -35,7 +35,7 @@
     {
       "content": [
         {
-          "text": "R dusk u trellis dawn pine lichen moss dawn ash reef rosemary, ash cinder ivy glade lc onyx nw R moss ze fern zy spire",
+          "text": "R dusk u morning dawn pine modern moss dawn ash reef rosemary, ash pillar ivy spire lc onyx nw R moss ze fern zy gorge",
           "type": "text"
         }
       ],
@@ -44,7 +44,7 @@
     {
       "content": [
         {
-          "text": "R moss wren fern nq R un tarn vine trellis",
+          "text": "R moss wren fern nq R un tarn vine morning",
           "type": "text"
         }
       ],

corpus/real-world/lf-hyperledger-wiki-footer-comments-20563002.json

@@ -21,7 +21,7 @@
         {
           "attrs": {
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Gorge Beacon"
           },
           "type": "mention"
         },

corpus/real-world/lf-hyperledger-wiki-footer-comments-20873387.json

@@ -7,12 +7,12 @@
             "extensionKey": "legacy-content",
             "extensionType": "com.atlassian.confluence.migration",
             "parameters": {
-              "cxhtml": "<h3>Lavender/Glade Rosemary 3:</h3>",
+              "cxhtml": "<h9>Lavender/Glade Rosemary 3:</h9>",
               "nestedContent": {
                 "content": [
                   {
                     "attrs": {
-                      "level": 6
+                      "level": 3
                     },
                     "content": [
                       {
@@ -24,7 +24,7 @@
                   }
                 ],
                 "type": "ivy",
-                "version": 9
+                "version": 6
               }
             },
             "text": ""

corpus/real-world/lf-hyperledger-wiki-footer-comments-21438452.json

@@ -5,12 +5,12 @@
         {
           "attrs": {
             "id": "336999:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Hi Glade"
           },
           "type": "mention"
         },
         {
-          "text": " : Hi glade rz wren pine dawn? Ivy ash ash yz moss ivy E ul lavender wren spire?",
+          "text": " : Rz spire yz wren pine dawn? Ivy ash ash eu moss ivy L cn lavender wren gorge?",
           "type": "text"
         }
       ],
@@ -48,12 +48,12 @@
               "marks": [
                 {
                   "attrs": {
-                    "href": "https://cn-rosemary.mountain.ash/sage/emerald/IVY/Gorge+Willow+olive+sycamore+ash+Beach+Cedar+Morning"
+                    "href": "https://wn-rosemary.mountain.ash/sage/emerald/IVY/Olive+Willow+beach+sycamore+ash+Cedar+Brook+Morning"
                   },
                   "type": "link"
                 }
               ],
-              "text": "Gorge Willow olive sycamore ash Beach Cedar Morning",
+              "text": "Olive Willow beach sycamore ash Cedar Brook Morning",
               "type": "text"
             }
           ],
@@ -71,7 +71,7 @@
                 "extensionKey": "legacy-content",
                 "extensionType": "com.atlassian.confluence.migration",
                 "parameters": {
-                  "cxhtml": "<fountain>Beacon lichen sycamore ash brook stork shimmer?</fountain>",
+                  "cxhtml": "<fountain>Beacon lichen sycamore ash stork sleet shimmer?</fountain>",
                   "nestedContent": {
                     "content": [
                       {
@@ -79,7 +79,7 @@
                           {
                             "content": [
                               {
-                                "text": "Beacon lichen sycamore ash brook stork shimmer?",
+                                "text": "Beacon lichen sycamore ash stork sleet shimmer?",
                                 "type": "wren"
                               }
                             ],
@@ -107,7 +107,7 @@
                 "extensionKey": "legacy-content",
                 "extensionType": "com.atlassian.confluence.migration",
                 "parameters": {
-                  "cxhtml": "<fountain>Dusk: 7 ivy, 7 lantern, 4 wn. STERLING.</fountain>",
+                  "cxhtml": "<fountain>Dusk: 7 ivy, 7 lantern, 4 qu. STERLING.</fountain>",
                   "nestedContent": {
                     "content": [
                       {
@@ -115,7 +115,7 @@
                           {
                             "content": [
                               {
-                                "text": "Dusk: 7 ivy, 7 lantern, 4 wn. STERLING.",
+                                "text": "Dusk: 7 ivy, 7 lantern, 4 qu. STERLING.",
                                 "type": "wren"
                               }
                             ],

corpus/real-world/lf-hyperledger-wiki-footer-comments-21791461.json

@@ -23,12 +23,12 @@
         {
           "attrs": {
             "id": "336999:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Ash Coral"
           },
           "type": "mention"
         },
         {
-          "text": " sage eu modern sycamore sapling ash iu fountain glimmer mr coral mountain sterling pw c splendid juniper - fountain dp ivy mountain slope hazel jasmine rosemary fountain mr ivy harvest pillar.",
+          "text": " sage eu modern sycamore sapling ash iu fountain glimmer mr slope mountain sterling pw c splendid juniper - fountain dp ivy mountain hazel sleet jasmine rosemary fountain mr ivy harvest pillar.",
           "type": "text"
         }
       ],
@@ -37,7 +37,7 @@
     {
       "content": [
         {
-          "text": "Sleet, ash raven mr glacier wren lavender - swallow cottage mr drizzle sycamore eu dusk",
+          "text": "Raven, ash fable mr glacier wren lavender - swallow cottage mr drizzle sycamore eu dusk",
           "type": "text"
         }
       ],
@@ -51,11 +51,11 @@
               "type": "strong"
             }
           ],
-          "text": "(fable 1740.58.56)",
+          "text": "(mossy 1740.58.56)",
           "type": "text"
         },
         {
-          "text": " Able ivy ash rosemary lavender & rosemary granite lavender, vibrant rosemary hazel nw mr dapple c dewdrop 'ivy & mossy' ash qm vivid ivy ivy pine mountain, prairie mr vivid splendid slope. ",
+          "text": " Able ivy ivy rosemary lavender & rosemary granite lavender, vibrant rosemary sleet nw mr dapple c dewdrop 'ivy & vivid' ash qm vivid ivy ivy pine mountain, prairie mr wharf splendid hazel. ",
           "type": "text"
         }
       ],
@@ -85,7 +85,7 @@
             {
               "content": [
                 {
-                  "text": "Wharf nw c window sage mr sleet eu dawn'f granite tranquil ash dawn qm ivy vivid, timber reef ribbon vivid onyx ivy window lc twilight?  ",
+                  "text": "Coral nw c window sage mr sleet eu dawn'f granite tranquil ash dawn qm ivy wharf, timber reef ribbon wharf onyx ash window lc twilight?  ",
                   "type": "text"
                 }
               ],
@@ -99,7 +99,7 @@
             {
               "content": [
                 {
-                  "text": "'Ash fern' sterling onyx ash nw pasture mr sapling nz ivy tranquil qm coral beacon / pollen mr lc thicket woodland",
+                  "text": "'Ash fern' sterling onyx ivy nw pasture mr sapling nz ivy tranquil qm gleam beacon / pollen mr lc thicket woodland",
                   "type": "text"
                 }
               ],
@@ -113,7 +113,7 @@
             {
               "content": [
                 {
-                  "text": "Wharf fern sapling mr 'tarn kettle nz gleam Lavender rosemary lavender ledger vine beacon / ash cliff & rosemary glacier gull fountain",
+                  "text": "Coral fern sapling mr 'tarn kettle nz cliff Lavender rosemary lavender ledger vine beacon / ivy dream & rosemary glacier gull fountain",
                   "type": "text"
                 }
               ],
@@ -133,11 +133,11 @@
               "type": "strong"
             }
           ],
-          "text": "(fable 1740.58.56)",
+          "text": "(mossy 1740.58.56)",
           "type": "text"
         },
         {
-          "text": " Mountain vivid tarn 8-dream cove ivy",
+          "text": " Mountain wharf tarn 8-gable cove ivy",
           "type": "text"
         }
       ],
@@ -151,7 +151,7 @@
               "type": "strong"
             }
           ],
-          "text": "GABLE 5",
+          "text": "EAGLE 5",
           "type": "text"
         }
       ],
@@ -178,7 +178,7 @@
             {
               "content": [
                 {
-                  "text": "Silver 5-1 slope",
+                  "text": "Silver 5-1 hazel",
                   "type": "text"
                 }
               ],
@@ -216,7 +216,7 @@
     {
       "content": [
         {
-          "text": "Able Eagle = Ivy Clover",
+          "text": "Able Arrow = Ivy Clover",
           "type": "text"
         }
       ],
@@ -243,7 +243,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "B dewdrop lantern arrow wicker, ivy gleam sterling splendid ash nw parlor ash ash feather, ivy morning ivy arrow mr crater splendid mf c harvest sterling",
+                  "text": "B dewdrop lantern arrow wicker, ash cliff sterling splendid ivy nw parlor ash ivy feather, ash morning ivy flint mr crater splendid mf c harvest sterling",
                   "type": "text"
                 }
               ],
@@ -274,7 +274,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": " lily harvest, pw: ash zephyr, fountain, fountain qm flint - ",
+                  "text": " lily harvest, pw: ivy zephyr, fountain, fountain qm aspen - ",
                   "type": "text"
                 },
                 {
@@ -286,7 +286,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "ivy lily nq",
+                  "text": "ash lily nq",
                   "type": "text"
                 },
                 {
@@ -295,7 +295,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": " lavender mf c arrow onyx sycamore sapling ash ginger lavender & lavender quaint",
+                  "text": " lavender mf c flint onyx sycamore sapling ivy ginger lavender & lavender quaint",
                   "type": "text"
                 }
               ],
@@ -314,7 +314,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "dewdrop aspen ash vine lunar ash able nw parlor lc coral fountain lattice (dp lavender)",
+                  "text": "dewdrop lunar ash vine apple ivy able nw parlor lc slope fountain lattice (dp lavender)",
                   "type": "text"
                 }
               ],
@@ -332,7 +332,7 @@
                               "type": "strong"
                             }
                           ],
-                          "text": "(fable 1740.58.56)",
+                          "text": "(mossy 1740.58.56)",
                           "type": "text"
                         },
                         {
@@ -345,7 +345,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": "Ivy ",
+                          "text": "Ash ",
                           "type": "text"
                         },
                         {
@@ -369,7 +369,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": " ash ash dusk emerald?",
+                          "text": " ivy ash dusk emerald?",
                           "type": "text"
                         },
                         {
@@ -382,7 +382,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": "Apple: tarn mountain mf c sterling; sapling ash wisp parlor vivid ivy flint qm twilight",
+                          "text": "Dream: tarn mountain mf c sterling; sapling ivy wisp parlor wharf ivy aspen qm twilight",
                           "type": "text"
                         }
                       ],
@@ -442,7 +442,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "Sage nz ivy 'cove' dream ",
+                  "text": "Sage nz ivy 'cove' wharf ",
                   "type": "text"
                 },
                 {
@@ -463,7 +463,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": " qm ivy pine mossy, sycamore, island, lattice, ash. mr nw tranquil bs fountain fountain ash onyx cove.",
+                  "text": " qm ivy pine vivid, sycamore, island, lattice, ash. mr nw tranquil bs fountain fountain ash onyx cove.",
                   "type": "text"
                 },
                 {
@@ -485,7 +485,7 @@
                               "type": "strong"
                             }
                           ],
-                          "text": "(fable 1740.58.56)",
+                          "text": "(mossy 1740.58.56)",
                           "type": "text"
                         },
                         {
@@ -510,7 +510,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": "wharf beacon vivid",
+                          "text": "after beacon vivid",
                           "type": "text"
                         },
                         {
@@ -562,7 +562,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "morning c 'journey' crystal arrow fountain ash glacier, fern, twilight, ash.",
+                  "text": "morning c 'journey' crystal arrow fountain ivy glacier, fern, twilight, ash.",
                   "type": "text"
                 }
               ],
@@ -580,7 +580,7 @@
                               "type": "strong"
                             }
                           ],
-                          "text": "(fable 1740.58.56) ",
+                          "text": "(mossy 1740.58.56) ",
                           "type": "text"
                         },
                         {
@@ -642,7 +642,7 @@
               "type": "strong"
             }
           ],
-          "text": "GABLE 8",
+          "text": "EAGLE 8",
           "type": "text"
         }
       ],
@@ -669,7 +669,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "Ol teak arrow sycamore sapling ash 5) warm coral Lavender-feather rosemary, ivy 8) lily ash rosemary able ivy nw twilight lc twilight lc / mountain wren",
+                  "text": "Ol teak arrow sycamore sapling ivy 5) warm slope Lavender-feather rosemary, ash 8) lily ash rosemary able ash nw twilight lc twilight lc / mountain wren",
                   "type": "text"
                 },
                 {
@@ -692,7 +692,7 @@
                       "type": "strong"
                     }
                   ],
-                  "text": "(fable 1740.58.56)",
+                  "text": "(mossy 1740.58.56)",
                   "type": "text"
                 },
                 {
@@ -705,7 +705,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "- pw: ash heather mr wharf beacon sterling stellar",
+                  "text": "- pw: ivy heather mr after beacon sterling stellar",
                   "type": "text"
                 }
               ],
@@ -723,7 +723,7 @@
                               "type": "strong"
                             }
                           ],
-                          "text": "(fable 1740.58.56)",
+                          "text": "(mossy 1740.58.56)",
                           "type": "text"
                         },
                         {
@@ -732,14 +732,14 @@
                               "type": "em"
                             }
                           ],
-                          "text": " Ivy ",
+                          "text": " Ash ",
                           "type": "text"
                         },
                         {
                           "marks": [
                             {
                               "attrs": {
-                                "href": "https://tranquil.lavender.ash/vine/8642593/sycamore-ash-sequoia-ivy-mountain?saffron=ledger&ash=rosemary"
+                                "href": "https://tranquil.lavender.ash/vine/8642593/sycamore-ash-sequoia-ash-mountain?saffron=ledger&ash=rosemary"
                               },
                               "type": "link"
                             },
@@ -747,7 +747,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": "Lavender (Ivy, Glacier, ivy Splendid)",
+                          "text": "Lavender (Ash, Glacier, ash Splendid)",
                           "type": "text"
                         },
                         {
@@ -756,7 +756,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": " ash ash dusk emerald?",
+                          "text": " ivy ash dusk emerald?",
                           "type": "text"
                         }
                       ],
@@ -795,7 +795,7 @@
                       "type": "em"
                     }
                   ],
-                  "text": "Ol teak arrow glade sapling ash warm rosemary eu ash mr boulder saffron, pw: pine, lunar island, ash.",
+                  "text": "Ol teak arrow glade sapling ivy warm rosemary eu ivy mr boulder saffron, pw: pine, apple island, ash.",
                   "type": "text"
                 }
               ],
@@ -813,7 +813,7 @@
                               "type": "strong"
                             }
                           ],
-                          "text": "(fable 1740.58.56)",
+                          "text": "(mossy 1740.58.56)",
                           "type": "text"
                         },
                         {
@@ -822,14 +822,14 @@
                               "type": "em"
                             }
                           ],
-                          "text": " Ivy ",
+                          "text": " Ash ",
                           "type": "text"
                         },
                         {
                           "marks": [
                             {
                               "attrs": {
-                                "href": "https://tranquil.lavender.ash/vine/8642593/sycamore-ash-sequoia-ivy-mountain?saffron=ledger&ash=rosemary"
+                                "href": "https://tranquil.lavender.ash/vine/8642593/sycamore-ash-sequoia-ash-mountain?saffron=ledger&ash=rosemary"
                               },
                               "type": "link"
                             },
@@ -837,7 +837,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": "Lavender (Ivy, Glacier, ivy Splendid)",
+                          "text": "Lavender (Ash, Glacier, ash Splendid)",
                           "type": "text"
                         },
                         {
@@ -846,7 +846,7 @@
                               "type": "em"
                             }
                           ],
-                          "text": " ash ash dusk emerald?",
+                          "text": " ivy ash dusk emerald?",
                           "type": "text"
                         }
                       ],

corpus/real-world/lf-hyperledger-wiki-footer-comments-22154564.json

@@ -1,28 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": "Hi wren lavender rz ",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "type": "em"
-            }
-          ],
-          "text": "pine",
-          "type": "text"
-        },
-        {
-          "text": " willow dawn trellis yz morning?",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-hyperledger-wiki-footer-comments-22315713.json

@@ -12,12 +12,12 @@
         {
           "attrs": {
             "id": "336999:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Spire Beacon"
           },
           "type": "mention"
         },
         {
-          "text": " dawn spire moss morning sage ash beacon wren dusk.",
+          "text": " dawn gorge moss morning sage ash lichen wren dusk.",
           "type": "text"
         }
       ],

corpus/real-world/lf-hyperledger-wiki-pages-18497343.json

@@ -441,12 +441,12 @@
                 {
                   "attrs": {
                     "id": "3a41c5441d641d8d074ef44f",
-                    "text": "@Heidi Finnegan"
+                    "text": "@Fountain Fountain (Lavender)"
                   },
                   "type": "mention"
                 },
                 {
-                  "text": " (Fountain fn Window) ",
+                  "text": " (Lavender fn Window) ",
                   "type": "text"
                 },
                 {
@@ -458,7 +458,7 @@
                       "type": "textColor"
                     }
                   ],
-                  "text": "<fountain.lavender@",
+                  "text": "<lavender.fountain@",
                   "type": "text"
                 },
                 {
@@ -496,7 +496,7 @@
             {
               "content": [
                 {
-                  "text": "@Pollen Bm-Kettle (Fountain fn Window) <",
+                  "text": "@Pollen Bm-Kettle (Lavender fn Window) <",
                   "type": "text"
                 },
                 {
@@ -546,7 +546,7 @@
       },
       "content": [
         {
-          "text": "Lavender",
+          "text": "Mountain",
           "type": "text"
         }
       ],
@@ -558,7 +558,7 @@
       },
       "content": [
         {
-          "text": "Lavender ash Fern Juniper",
+          "text": "Tranquil ash Fern Juniper",
           "type": "text"
         }
       ],
@@ -571,7 +571,7 @@
             {
               "content": [
                 {
-                  "text": "Tarn ash wren fountain (ASH) nw granite ash mountain eu ",
+                  "text": "Tarn ash wren lavender (ASH) nw granite ash rosemary eu ",
                   "type": "text"
                 },
                 {
@@ -587,7 +587,7 @@
                   "type": "text"
                 },
                 {
-                  "text": " gull vibrant. Pollen tarn tranquil nw wb ASH. Tarn cove sleet moss ash HI.",
+                  "text": " gull vibrant. Pollen tarn sterling nw wb ASH. Tarn cove sleet moss ash HI.",
                   "type": "text"
                 }
               ],
@@ -680,7 +680,7 @@
             {
               "content": [
                 {
-                  "text": "Prairie Lavender Rosemary Sterling - Teak",
+                  "text": "Prairie Twilight Fountain Lavender - Teak",
                   "type": "text"
                 }
               ],
@@ -694,7 +694,7 @@
             {
               "content": [
                 {
-                  "text": "Ivy wicker se twilight Prairie Lavender Rosemary?",
+                  "text": "Ivy wicker se woodland Prairie Twilight Fountain?",
                   "type": "text"
                 }
               ],

corpus/real-world/lf-onap-jira-CPS-2981-description.json

@@ -1,31 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": "Lavender Trellis : ",
-          "type": "text"
-        },
-        {
-          "type": "hardBreak"
-        },
-        {
-          "type": "hardBreak"
-        },
-        {
-          "attrs": {
-            "url": "https://hi-wren.rosemary.ivy/pine/willow/RZ/glade/336999174/ASH-1585+Sycamore+Rosemary+yz+Eu+Dawn+Mountain+Spire#Fountain-Mountain"
-          },
-          "type": "inlineCard"
-        },
-        {
-          "text": "  ",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-onap-jira-CPS-3088-comment-86784.json

@@ -2,31 +2,42 @@
   "content": [
     {
       "content": [
+        {
+          "text": "Hi ",
+          "type": "text"
+        },
         {
           "attrs": {
             "accessLevel": "",
             "id": "3a41c5441d641d8d074ef44f",
             "localId": "a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Glade Lavender"
           },
           "type": "mention"
         },
         {
-          "text": " willow trellis glade morning hi ",
+          "text": " ",
           "type": "text"
         },
         {
-          "marks": [
+          "type": "hardBreak"
+        },
         {
-              "type": "code"
-            }
-          ],
-          "text": "wren.rz",
+          "type": "hardBreak"
+        },
+        {
+          "text": "Rz ivy yz ASH spire gorge olive ash rosemary",
           "type": "text"
         },
         {
-          "text": " yz ivy.",
+          "type": "hardBreak"
+        },
+        {
+          "text": "Ivy ash eu willow",
           "type": "text"
+        },
+        {
+          "type": "hardBreak"
         }
       ],
       "type": "paragraph"

corpus/real-world/lf-onap-wiki-footer-comments-16236151.json

@@ -0,0 +1,50 @@
+{
+  "content": [
+    {
+      "content": [
+        {
+          "text": "Lavender rosemary hi ",
+          "type": "text"
+        },
+        {
+          "marks": [
+            {
+              "attrs": {
+                "__confluenceMetadata": {
+                  "contentTitle": "IVY 3369-99-17",
+                  "linkType": "wren",
+                  "spaceKey": "Sycamore"
+                },
+                "href": "https://rz-pine.rosemary.ash/dawn/willow/Sycamore/glade/40585685"
+              },
+              "type": "link"
+            }
+          ],
+          "text": "IVY 3369-99-17",
+          "type": "text"
+        },
+        {
+          "text": " (spire #13). ",
+          "type": "text"
+        },
+        {
+          "attrs": {
+            "__fileMimeType": "mountain/ash",
+            "__fileName": "MOSS-Mountain.ash",
+            "__fileSize": 846581,
+            "collection": "fountain-55864259",
+            "id": "a41c5441-d641-4d8d-b4ef-44f7f31bac87"
+          },
+          "type": "mediaInline"
+        },
+        {
+          "text": " yz sterling splendid.",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-onap-wiki-footer-comments-16242917.json

@@ -5,7 +5,7 @@
         {
           "attrs": {
             "id": "33699:a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Willow beacon"
           },
           "type": "mention"
         },
@@ -19,7 +19,7 @@
               "type": "em"
             }
           ],
-          "text": "\"Wren ash ivy Trellis pine rosemary willow (Glade-Dawn)?\" ",
+          "text": "\"Wren ash ivy Trellis pine rosemary lichen (Glade-Dawn)?\" ",
           "type": "text"
         },
         {

corpus/real-world/lf-onap-wiki-footer-comments-16244913.json

@@ -1,29 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "text": "Trellis, Willow ivy hi wren pine ash Lavender Rosemary morning rz sycamore dawn. Yz glade moss eu emerald ash spire gorge. Willow sage ash beacon ",
-          "type": "text"
-        },
-        {
-          "attrs": {
-            "__fileMimeType": "rosemary/ash",
-            "__fileName": "WREN_Fountain.ash",
-            "__fileSize": 336999,
-            "collection": "mountain-17405856",
-            "id": "a41c5441-d641-4d8d-b4ef-44f7f31bac87"
-          },
-          "type": "mediaInline"
-        },
-        {
-          "text": " . Mountain dusk sterling.",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-onap-wiki-footer-comments-16248177.json

@@ -12,7 +12,7 @@
         {
           "attrs": {
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Willow Trellis"
           },
           "type": "mention"
         },
@@ -23,7 +23,7 @@
         {
           "attrs": {
             "id": "7f31b",
-            "shortName": ":willow_glade:",
+            "shortName": ":beacon_glade:",
             "text": "🙂"
           },
           "type": "emoji"

corpus/real-world/lf-onap-wiki-footer-comments-16250665.json

@@ -1,34 +1,5 @@
 {
   "content": [
-    {
-      "content": [
-        {
-          "text": "IVY, wren ash hi r lavender zy ",
-          "type": "text"
-        },
-        {
-          "marks": [
-            {
-              "attrs": {
-                "__confluenceMetadata": {
-                  "contentTitle": "ASH Rosemary sycamore",
-                  "linkType": "pine"
-                },
-                "href": "https://ze-rosemary.mountain.ivy/dawn/willow/ASH/glade/33699917"
-              },
-              "type": "link"
-            }
-          ],
-          "text": "ASH Rosemary sycamore",
-          "type": "text"
-        },
-        {
-          "text": ", spire ul trellis.",
-          "type": "text"
-        }
-      ],
-      "type": "paragraph"
-    },
     {
       "attrs": {
         "__autoSize": true,
@@ -46,7 +17,25 @@
                 {
                   "content": [
                     {
-                      "text": "Fountain gorge hi beacon cn ash ASH moss ash mountain wn ash morning qu sage zy sterling wn emerald wren.",
+                      "text": "WREN",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "tableCell"
+            },
+            {
+              "attrs": {
+                "colspan": 1,
+                "rowspan": 1
+              },
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "Trellis lavender hi IVY rosemary.",
                       "type": "text"
                     }
                   ],
@@ -60,6 +49,21 @@
         }
       ],
       "type": "table"
+    },
+    {
+      "content": [
+        {
+          "text": "Pine dawn rz moss yz \"lavender hi IVY rosemary \"?  Sage wren moss dusk ASH-E ul glade cn sycamore ash IVY rosemary hi Able (Pine DAWN)  ivy mountain wren cn ash ash Spire-Gorge fountain morning? ",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "type": "paragraph"
     }
   ],
   "type": "doc",

corpus/real-world/lf-onap-wiki-pages-16230991.json

@@ -0,0 +1,34 @@
+{
+  "content": [
+    {
+      "content": [
+        {
+          "attrs": {
+            "extensionKey": "pagetree",
+            "extensionType": "com.atlassian.confluence.macro.core",
+            "parameters": {
+              "macroMetadata": {
+                "macroId": {
+                  "value": "a41c5441-d641-4d8d-b4ef-44f7f31bac87"
+                },
+                "schemaVersion": {
+                  "value": "3"
+                },
+                "title": "Pine Dawn"
+              },
+              "macroParams": {
+                "root": {
+                  "value": "@wren"
+                }
+              }
+            }
+          },
+          "type": "inlineExtension"
+        }
+      ],
+      "type": "paragraph"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-onap-wiki-pages-16231985.json

@@ -1,103 +0,0 @@
-{
-  "content": [
-    {
-      "content": [
-        {
-          "attrs": {
-            "extensionKey": "create-from-template",
-            "extensionType": "com.atlassian.confluence.macro.core",
-            "parameters": {
-              "macroMetadata": {
-                "macroId": {
-                  "value": "c3576900-0bf0-44ac-9065-5a98847af7ed"
-                },
-                "placeholder": [
-                  {
-                    "data": {
-                      "url": "https://hi-moss.lavender.ash/sage/fountain/mountain/ivy.lavender.rosemary.trellis.rosemary-beacon-morning-lichen/cinder/dawn-glade-sterling-36.ash"
-                    },
-                    "type": "dawn"
-                  }
-                ],
-                "schemaVersion": {
-                  "value": "3"
-                },
-                "title": "Willow wren Splendid"
-              },
-              "macroParams": {
-                "blueprintModuleCompleteKey": {
-                  "value": "ivy.lavender.rosemary.trellis.rosemary-sycamore-rosemary:wren-pine-mountain"
-                },
-                "contentBlueprintId": {
-                  "value": "a41c5441-d641-4d8d-b4ef-44f7f31bac87"
-                },
-                "createButtonLabel": {
-                  "value": "Willow wren pine"
-                }
-              }
-            }
-          },
-          "type": "inlineExtension"
-        }
-      ],
-      "marks": [
-        {
-          "attrs": {
-            "align": "end"
-          },
-          "type": "alignment"
-        }
-      ],
-      "type": "paragraph"
-    },
-    {
-      "type": "rule"
-    },
-    {
-      "attrs": {
-        "extensionKey": "content-report-table",
-        "extensionType": "com.atlassian.confluence.macro.core",
-        "parameters": {
-          "macroMetadata": {
-            "macroId": {
-              "value": "15037805-ba5b-44a0-95ed-42dc885440db"
-            },
-            "schemaVersion": {
-              "value": "3"
-            },
-            "title": "Shimmer Pillar Beach"
-          },
-          "macroParams": {
-            "analyticsKey": {
-              "value": "wren-pine"
-            },
-            "blankDescription": {
-              "value": "Modern, emerald ivy spire gorge dusk able pine."
-            },
-            "blankTitle": {
-              "value": "Dawn olive"
-            },
-            "blueprintModuleCompleteKey": {
-              "value": "ivy.lavender.rosemary.trellis.rosemary-sycamore-rosemary:wren-pine-mountain"
-            },
-            "contentBlueprintId": {
-              "value": "a41c5441-d641-4d8d-b4ef-44f7f31bac87"
-            },
-            "createButtonLabel": {
-              "value": "Willow wren pine"
-            },
-            "labels": {
-              "value": "wren-pine"
-            },
-            "spaces": {
-              "value": "RZ"
-            }
-          }
-        }
-      },
-      "type": "extension"
-    }
-  ],
-  "type": "doc",
-  "version": 1
-}

corpus/real-world/lf-onap-wiki-pages-16239857.json

@@ -0,0 +1,690 @@
+{
+  "content": [
+    {
+      "type": "paragraph"
+    },
+    {
+      "content": [
+        {
+          "marks": [
+            {
+              "attrs": {
+                "href": "https://hi-wren.rosemary.ash/pine/willow/RZ/glade/33699917"
+              },
+              "type": "link"
+            }
+          ],
+          "text": "IVY Lavender",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Sycamore",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "title": "ASH Rosemary Spire/Mountain"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "ASH trellis gorge/fountain mountain y sterling ivy ze beacon splendid, fountain ash mountain rosemary ul y ASH trellis. Ash ASH trellis cn morning wn olive ul splendid dawn sycamore ash rosemary ash emerald lavender splendid lavender.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "text": "Ash splendid dawn sycamore ash rosemary beacon ivy ash ash shimmer tranquil beach ul ivy ASH trellis, moss qu twilight sterling tranquil, nc-sage woodland, lavender ash rosemary ash fountain mountain.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "text": "Ash lavender splendid lavender beacon y ash ul lantern tranquil emerald iu y cedar ASH trellis ash ash iu woodland ash sterling brook. Wren dusk ivy mr splendid ash splendid able y cedar ASH trellis cn nc-sapling ash fountain pw ivy-dawn.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "text": "Ash ASH trellis gorge cn fountain stork ivy SLEET ASH lichen glimmer. Dp cn lavender, quiet pine ivy trellis lavender ash dawn, stork SLEET REEF coral.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "ASH ASH Slope"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "ASH ASH brook juniper ASH trellis Cinder lavender pine jasmine onyx brook ash modern fern IVY fountain mountain ze beacon ivy ASH trellis gorge ash harvest glacier. Ash tranquil brook ivy lavender...",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "ASH Swallow Cottage - drizzle y REEF fern hazel nc pillar rosemary iu ivy ASH trellis Cinder sterling",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "listItem"
+            },
+            {
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "ASH Swallow Twilight - fountain ivy glacier ul ivy ASH lavender ze dapple tarn woodland ivy vine sleet sycamore",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "listItem"
+            },
+            {
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "ASH Swallow Tranquil - fountain ASH trellis gorge ash lavender gull ivy REEF fern",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "listItem"
+            },
+            {
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "ASH Swallow Window - woodland ASH sterling fountain tarn y timber woodland iu COVE-N twilight",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "listItem"
+            },
+            {
+              "content": [
+                {
+                  "content": [
+                    {
+                      "text": "ASH Swallow Ash Ivy - fountain y \"ash ivy\" granite ze dapple tarn ivy harvest ivy mr twilight ribbon mountain",
+                      "type": "text"
+                    }
+                  ],
+                  "type": "paragraph"
+                }
+              ],
+              "type": "listItem"
+            }
+          ],
+          "type": "bulletList"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "ASH Splendid Raven"
+      },
+      "content": [
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "4) Woodland ",
+              "type": "text"
+            },
+            {
+              "marks": [
+                {
+                  "type": "strong"
+                },
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "window/vibrant",
+              "type": "text"
+            },
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": " gull Pollen",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "1) Teak rosemary-mr/splendid/granite, Early jasmine \"kettle-dewdrop wq -m\"",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "           1 Ledger twilight teak ivy:",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "           4) ASH Twilight",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "                  1) Mountain sterling.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": " ",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "5) Sage Prairie ash Able ",
+              "type": "text"
+            },
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#0563c1"
+                  },
+                  "type": "textColor"
+                },
+                {
+                  "type": "underline"
+                },
+                {
+                  "attrs": {
+                    "href": "#"
+                  },
+                  "type": "link"
+                },
+                {
+                  "type": "strong"
+                }
+              ],
+              "text": "http://{lily}:8568/pasture/rosemary",
+              "type": "text"
+            },
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                },
+                {
+                  "type": "strong"
+                }
+              ],
+              "text": "  (Lily Lavender)",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": " ",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "    Fable \"Beacon\" ze silver SLEET harvest",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "                 4) Splendid Splendid",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "                 1) Mountain Able ash Splendid Able ivy twilight splendid qu VINE thicket-4 ",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "                    fn ash glacier tarn.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        },
+        {
+          "content": [
+            {
+              "marks": [
+                {
+                  "attrs": {
+                    "color": "#000000"
+                  },
+                  "type": "textColor"
+                }
+              ],
+              "text": "                 5) Beacon, Wisp, Vesper ash Woodland SLEET lavender ivy lavender.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        },
+        {
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Lily Vesper",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "attrs": {
+        "title": "ASH Swallow Slope Lily Mossy"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "ASH Swallow Tranquil, mountain ASH trellis Cinder lavender pine y splendid reed ze beacon ivy ASH trellis gorge ash harvest glacier. Dp cn sage sycamore qu cove ul ivy ASH Splendid ASH brook.Ash harvest lavender vivid ash ul ivy ASH ASH jasmine onyx sycamore (Wren) ash modern-fern IVY fountain mountain wn vivid ze beacon ivy gorge ash ivy harvest glacier. Zj moss, bm cn sterling lavender ze ivy ASH ASH brook.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "Splendid Lily Mossy ash Tranquil"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "ASH",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "Splendid Lily Mossy ash ASH Lavender"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "ASH",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "level": 2
+      },
+      "content": [
+        {
+          "text": "Fountain",
+          "type": "text"
+        }
+      ],
+      "type": "heading"
+    },
+    {
+      "attrs": {
+        "title": "Ash-Fp"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "Jasmine iu ivy ASH ASH qu rosemary ash modern fern fountain mountain. Lantern ash ul ASH ASH rosemary gull splendid fern sycamore, wharf feather, lavender ash jasmine onyx sycamore. Sycamore iu ivy ASH Swallow Tranquil.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "Ash-Wb"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "Jasmine iu ivy ASH ASH qu rosemary ash modern fern fountain mountain. Lantern ash ul SE Splendid rosemary gull splendid fern sycamore, wharf feather, lavender ash jasmine onyx sycamore. Sycamore iu ivy ASH Swallow Tranquil ash COVE-N ASH ASH coral-morning harvest.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "Wb-Ash"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "Jasmine iu ivy SE Splendid qu y twilight IVY. Lantern ash ul SE lavender rosemary gull jasmine ash splendid fern sycamore. Sycamore iu ivy lavender zephyr.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "attrs": {
+        "title": "Wb-Dc"
+      },
+      "content": [
+        {
+          "content": [
+            {
+              "text": "Jasmine iu ivy SE Splendid qu y twilight IVY. Lantern ash ul ivy SE Splendid'z Ash reed tranquil ash twilight ul twilight ASH twilight. Sycamore iu ivy COVE-N mountain gleam.",
+              "type": "text"
+            }
+          ],
+          "type": "paragraph"
+        }
+      ],
+      "type": "expand"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "layout": "align-start"
+      },
+      "content": [
+        {
+          "attrs": {
+            "type": "external",
+            "url": "https://pine.onyx-o.ivy/rosemary/sycamore/5038465/lwdiq8.ash?dewdrop=4&mountain=1558642593042&ash=j5&lattice=ravine-lichen,teak-ravine,warm"
+          },
+          "type": "media"
+        }
+      ],
+      "type": "mediaSingle"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "type": "paragraph"
+    },
+    {
+      "type": "paragraph"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-opendaylight-jira-BGPCEP-1024-description.json

@@ -0,0 +1,49 @@
+{
+  "content": [
+    {
+      "content": [
+        {
+          "text": "Willow ivy lavender glade hi beacon wren rz trellis:",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    },
+    {
+      "attrs": {
+        "language": "java"
+      },
+      "content": [
+        {
+          "text": "/lavender/lichen -> /spire/pine/...?morning=lichen\n/lavender/rosemary -> /spire/pine/...?morning=sycamore\n/lavender/rosemary -> /spire/mountain",
+          "type": "text"
+        }
+      ],
+      "type": "codeBlock"
+    },
+    {
+      "content": [
+        {
+          "text": "Yz gorge eu emerald dawn shimmer lavender lc nw moss fountain. Ash ivy mountain olive beach ",
+          "type": "text"
+        },
+        {
+          "marks": [
+            {
+              "type": "code"
+            }
+          ],
+          "text": "wren",
+          "type": "text"
+        },
+        {
+          "text": ".",
+          "type": "text"
+        }
+      ],
+      "type": "paragraph"
+    }
+  ],
+  "type": "doc",
+  "version": 1
+}

corpus/real-world/lf-opendaylight-jira-BGPCEP-1047-comment-51588.json

@@ -6,12 +6,12 @@
           "attrs": {
             "accessLevel": "",
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Willow Glade"
           },
           "type": "mention"
         },
         {
-          "text": " Hi Willow, R wren glade zy ze-ivy pine dawn SPIRE ash gorge, ash dawn moss 3 369 lavender (trellis ul rosemary olive dawn 99 369) ivy cn ash ash sage wren ul morning",
+          "text": " Hi Willow, R wren spire zy ze-ivy pine dawn GORGE ash olive, ash dawn moss 3 369 lavender (trellis ul rosemary beach dawn 99 369) ivy cn ash ash sage wren ul morning",
           "type": "text"
         },
         {

corpus/real-world/lf-opendaylight-jira-NETCONF-1252-comment-35584.json

@@ -6,12 +6,12 @@
           "attrs": {
             "accessLevel": "",
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Willow Glade"
           },
           "type": "mention"
         },
         {
-          "text": "  hi willow ivy ash. Rz ash Y glade Y spire gorge olive beacon beach - ",
+          "text": "  hi beacon ivy ash. Rz ash Y spire Y gorge olive beach lichen cedar - ",
           "type": "text"
         }
       ],
@@ -32,7 +32,7 @@
     {
       "content": [
         {
-          "text": " ze mountain ivy lichen ul morning/fountain emerald ash cn ze wren olive pine mountain sterling. Dawn splendid ze ivy shimmer ",
+          "text": " ze mountain ivy cinder ul morning/fountain emerald ash cn ze wren beach pine mountain sterling. Dawn splendid ze ivy shimmer ",
           "type": "text"
         },
         {
@@ -66,7 +66,7 @@
           "type": "text"
         },
         {
-          "text": " moss sage cinder cedar mountain. Wn brook wren ul ash rosemary dusk sage qu modern ash sycamore fountain brook pillar. Stork sapling sleet ul nc glimmer ivy ash iu able rosemary mr ash dapple - Y glade pine sage juniper dawn Y lavender quiet jasmine Y ash ",
+          "text": " moss sage modern brook mountain. Wn stork wren ul ash rosemary dusk sage qu pillar ash sycamore fountain stork dapple. Sleet sapling quiet ul nc glimmer ivy ash iu able rosemary mr ash window - Y spire pine sage juniper dawn Y lavender coral jasmine Y ash ",
           "type": "text"
         },
         {
@@ -75,11 +75,11 @@
               "type": "strong"
             }
           ],
-          "text": "window()",
+          "text": "timber()",
           "type": "text"
         },
         {
-          "text": " splendid ivy reef timber sage ash ivy lavender pw ash ribbon. Harvest Y glade coral onyx rosemary mr ",
+          "text": " splendid ivy reef ribbon sage ash ivy lavender pw ash window. Harvest Y spire slope onyx rosemary mr ",
           "type": "text"
         }
       ],
@@ -91,7 +91,7 @@
       },
       "content": [
         {
-          "text": "ivy.lavender.trellis.window.Tranquil.twilight",
+          "text": "ivy.lavender.trellis.pollen.Tranquil.twilight",
           "type": "text"
         }
       ],
@@ -100,7 +100,7 @@
     {
       "content": [
         {
-          "text": " - cn fern ivy pollen sterling ash rosemary ivy splendid slope hazel dp fountain.",
+          "text": " - cn fern ivy kettle sterling ash rosemary ivy splendid hazel sleet dp fountain.",
           "type": "text"
         }
       ],

corpus/real-world/lf-opendaylight-jira-NETCONF-1438-comment-46069.json

@@ -6,12 +6,12 @@
           "attrs": {
             "accessLevel": "",
             "id": "3a41c5441d641d8d074ef44f",
-            "text": "@Heidi Finnegan"
+            "text": "@Wren Willow"
           },
           "type": "mention"
         },
         {
-          "text": " Ivy hi lavender trellis wren:",
+          "text": " Ivy hi lavender trellis pine:",
           "type": "text"
         }
       ],
@@ -20,7 +20,7 @@
     {
       "content": [
         {
-          "text": "{\n    \"willow\": {\n        \"glade\": [\n            {\n                \"glade-ash\": \"morning-emerald\",\n                \"glade-shimmer\": \"Lantern rosemary emerald 'spire-rz'\",\n                \"glade-pine\": \"sycamore\"\n            }\n        ]\n    }\n}",
+          "text": "{\n    \"beacon\": {\n        \"glade\": [\n            {\n                \"glade-ash\": \"morning-emerald\",\n                \"glade-shimmer\": \"Lantern rosemary emerald 'spire-rz'\",\n                \"glade-dawn\": \"sycamore\"\n            }\n        ]\n    }\n}",
           "type": "text"
         }
       ],

corpus/real-world/lf-opendaylight-jira-NETCONF-1474-comment-47986.json

@@ -7,7 +7,7 @@
             "accessLevel": "",
             "id": "3a41c5441d641d8d074ef44f",
             "localId": "a41c5441-d641-4d8d-b4ef-44f7f31bac87",
-            "text": "@Heidi Finnegan"
+            "text": "@Wren Willow"
           },
           "type": "mention"
         },
@@ -17,7 +17,7 @@
         },
         {
           "attrs": {
-            "url": "https://ivy.ash-willow.ash/ash/rzy3369.wren#trellis-9.9"
+            "url": "https://ivy.ash-beacon.ash/ash/rzy3369.pine#trellis-9.9"
           },
           "type": "inlineCard"
         },
@@ -41,7 +41,7 @@
                   "type": "code"
                 }
               ],
-              "text": "Pine dawn MOSS sage ze glade ivy wren MORNING rosemary, \"ash:dusk:beacon:emerald:wren:9.9\".  U dawn",
+              "text": "Dawn moss SAGE wren ze glade ivy dusk MORNING rosemary, \"ash:able:lichen:emerald:dusk:9.9\".  U moss",
               "type": "text"
             }
           ],
@@ -55,7 +55,7 @@
                   "type": "code"
                 }
               ],
-              "text": "ASH shimmer sycamore ivy rosemary MORNING mountain, hi fountain able lc mountain sterling splendid mountain.",
+              "text": "ASH shimmer sycamore ivy rosemary MORNING mountain, hi fountain pine lc mountain sterling splendid mountain.",
               "type": "text"
             }
           ],
@@ -74,12 +74,12 @@
             },
             {
               "attrs": {
-                "url": "https://ivy.ash-willow.ash/ash/rzy3369.wren#trellis-1.7"
+                "url": "https://ivy.ash-beacon.ash/ash/rzy3369.pine#trellis-1.7"
               },
               "type": "inlineCard"
             },
             {
-              "text": "  pine able ",
+              "text": "  dawn pine ",
               "type": "text"
             }
           ],
@@ -88,7 +88,7 @@
         {
           "content": [
             {
-              "text": "Dawn fountain ash lantern lichen ivy sapling ivy spire nw ivy MORNING\n   splendid.",
+              "text": "Reef fountain ash lantern cinder ivy sapling ivy spire nw ivy MORNING\n   splendid.",
               "type": "text"
             }
           ],
@@ -109,7 +109,7 @@
               "type": "code"
             }
           ],
-          "text": "dusk-emerald",
+          "text": "able-emerald",
           "type": "text"
         },
         {

docs/decisions.md

@@ -1185,29 +1185,29 @@ rather than Node, goes in the second table. A wrapper or fork of a listed packag
 
 ## The real-world corpus holds anonymized documents from sites whose licence it records
 
-2026-11-04, on the maintainer's answers of 2026-11-03. It serves goal 6: the project had no evidence
-of what real documents contain. Valid while no public source with a clearer licence serves ADF.
+2026-11-04, on the maintainer's answers of 2026-11-03. It serves goal 6: real documents show what
+ordinary input must convert without surprise. Valid while no public source with a clearer licence
+serves ADF.
 
-Each document keeps its exact format: every node, mark, attribute key and the shape of each value.
-Its text, names, ids and URLs are replaced with fejkdata's draws, and a document that fails the
-shape or leak check stays out. A uuid, whether a whole value or inside one, keeps its version digit
-and case: the 22,769 hold v1, v3 and v4 uuids. `formatKeys` in `corpus-gathering/anonymize.mjs` lists the attributes
-kept because they name a format or a layout. The fetched originals are never committed.
+Each document keeps every node, mark and attribute key. Its words, names, ids and URLs are replaced
+with values drawn from fejkdata, a fake-data tool run from its image. Replaced text keeps its
+punctuation and case, and a number keeps its digit count. A document stays out if its structure
+changed or its output still holds an original token. The fetched originals are never committed:
+`corpus-gathering/run.sh` refuses a raw folder inside the repo.
 
-A site joins only with a published licence. All six are hosted by the Linux Foundation:
+`corpus/real-world-sources.json` links each document to its original page, because the licences ask
+for credit and naming a creator in the document would undo the anonymizing.
 
-- The ONAP wiki's own pages disagree. A 2017 page says CC-BY-SA-4.0 and a 2019 page says CC-BY-4.0,
-  so its documents record CC-BY-SA-4.0, which satisfies both.
-- Every other document rests on the foundation's terms of use, which put wikis and other
-  non-code submissions under CC-BY-3.0. They don't name issue trackers or `*.atlassian.net`, so
-  their reach to these sites' Jira documents and wikis is an inference. A site's own stated licence
-  replaces it in `corpus-gathering/sites.json`.
+A site joins only with a published licence, its own or its host's. All six sites in
+`corpus-gathering/sites.json` are hosted by the Linux Foundation:
 
-The tool stops at 20 result pages of 100 per source, one request a second as the sites' robots.txt
-asks, and 150 documents of at most 64 KiB. Those numbers keep a fetch to minutes and the corpus
-under 1 MB. Selection takes new node and mark types first, then the most new features per byte.
+- The foundation's terms of use put wikis and other non-code submissions under CC-BY-3.0. Those
+  terms don't name issue trackers or `*.atlassian.net`, so applying them to these sites is an
+  inference, which `corpus/README.md` tells the reader. When a site states its own licence, that
+  licence replaces them in `sites.json`.
+- The ONAP wiki states its own, but its pages disagree: a 2017 page says CC-BY-SA-4.0 and a 2019
+  page says CC-BY-4.0. Its documents record CC-BY-SA-4.0, which satisfies both.
 
-22,769 fetched documents hold no `bodiedSyncBlock`, `decisionItem`, `decisionList`,
-`extensionFrame`, `fragment`, `multiBodiedExtension` or `syncBlock`, and neither did the sites'
-decision-titled pages. `corpus.test.ts` pins that list, so a regenerated corpus holding one of them
-has to shrink it.
+None of the 22,773 fetched documents holds a type in `typesNoFetchedDocumentHolds`
+(`src/conformance/corpus.test.ts`), and neither did up to 40 pages per site with "decision" in the
+title. A regenerated corpus holding one of those types fails that test until the type leaves the list.

renovate.json

@@ -24,7 +24,7 @@
       "customType": "regex",
       "datasourceTemplate": "docker",
       "depNameTemplate": "node",
-      "description": "Pin the node image the gate runs",
+      "description": "Pin the node image the gate and the tooling scripts run",
       "managerFilePatterns": ["converter-comparison/run.sh", "corpus-gathering/run.sh", "docker-runner.sh"],
       "matchStrings": ["node_image=node:(?<currentValue>[0-9][^\\s\"']*)"],
       "versioningTemplate": "docker"

src/adf/document.ts

@@ -240,7 +240,7 @@ function isAttributes(value: unknown): value is AdfAttributes {
   return isRecord(value) && isJsonValue(value)
 }
 
-function isRecord(value: unknown): value is Record<string, unknown> {
+export function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null && !Array.isArray(value)
 }
 

src/conformance/corpus.test.ts

@@ -6,10 +6,10 @@ import test from 'node:test'
 
 import { adfToHtml } from '../html/emit/adf-to-html.ts'
 import { adfToLosslessMarkdown } from '../markdown/emit/adf-to-markdown.ts'
-import { htmlToAdf } from '../html/parse/html-to-adf.ts'
 import { blockNodes } from '../adf/block-nodes.ts'
+import { htmlToAdf } from '../html/parse/html-to-adf.ts'
 import { inlineNodes } from '../adf/inline-nodes.ts'
-import { isAdfDocument } from '../adf/document.ts'
+import { isAdfDocument, isRecord } from '../adf/document.ts'
 import { isJsonValue } from '../json-value.ts'
 import { losslessMarkdownToAdf } from '../markdown/parse/markdown-to-adf.ts'
 import { markVocabularies } from '../adf/mark-attributes.ts'
@@ -22,8 +22,8 @@ const realPayloadsRoot = join(corpusRoot, 'real-payloads')
 const realWorldRoot = join(corpusRoot, 'real-world')
 const roundTripRoot = join(corpusRoot, 'round-trip')
 
-const typesNoFetchedDocumentHolds = ['bodiedSyncBlock', 'decisionItem', 'decisionList', 'extensionFrame', 'fragment', 'multiBodiedExtension', 'syncBlock']
 const roundTripDirectories = ['block-nodes', 'combinations', 'commonmark-subset', 'inline-nodes', 'opaque-carry']
+const typesNoFetchedDocumentHolds = ['bodiedSyncBlock', 'decisionItem', 'decisionList', 'extensionFrame', 'fragment', 'multiBodiedExtension', 'syncBlock']
 
 function directoryNames(root: string): string[] {
   return readdirSync(root, { withFileTypes: true })
@@ -140,9 +140,6 @@ for (const name of pairedNames(normalizationRoot, '.md', '.json')) {
   })
 }
 
-// The opening tag of each block type whose only attributes are `localId`, `level` and `order`, as the editor writes them.
-const blockElements: Readonly<Record<string, string>> = { blockquote: '<blockquote', bulletList: '<ul', heading: '<h', orderedList: '<ol', paragraph: '<p', rule: '<hr' }
-
 for (const root of [realPayloadsRoot, realWorldRoot]) {
   const kind = basename(root)
 
@@ -171,22 +168,25 @@ for (const root of [realPayloadsRoot, realWorldRoot]) {
   }
 }
 
-for (const name of names(realPayloadsRoot, '.json')) {
-  test(`real-payloads/${name} writes each block holding only its id as its element`, () => {
-    const payload: unknown = JSON.parse(readFileSync(join(realPayloadsRoot, `${name}.json`), 'utf8'))
+// The opening tag of each block type, for a block whose only attributes are `localId`, `level` and `order`, as Atlassian's editor writes it.
+const blockElements: Readonly<Record<string, string>> = { blockquote: '<blockquote', bulletList: '<ul', heading: '<h', orderedList: '<ol', paragraph: '<p', rule: '<hr' }
+
+for (const root of [realPayloadsRoot, realWorldRoot]) {
+  for (const name of names(root, '.json')) {
+    test(`${basename(root)}/${name} writes each block holding only its id as its element`, () => {
+      const payload: unknown = JSON.parse(readFileSync(join(root, `${name}.json`), 'utf8'))
       assert.ok(isAdfDocument(payload), `${name}.json is not an ADF document`)
       assert.ok(adfToHtml(payload).ok)
       for (const block of payload.content ?? []) {
         const opening = Object.hasOwn(blockElements, block.type) ? blockElements[block.type] : undefined
         if (opening === undefined || block.marks !== undefined || Object.keys(block.attrs ?? {}).some((key) => !['level', 'localId', 'order'].includes(key))) continue
+        // A heading without `level` has no element to write: it rides the carry.
+        if (block.type === 'heading' && block.attrs?.['level'] === undefined) continue
         const written = adfToHtml({ content: [block], type: 'doc', version: 1 })
         assert.ok(written.ok && written.value.startsWith(opening), `${block.type} wrote ${written.ok ? written.value.slice(0, 80) : written.error.code}`)
       }
     })
   }
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === 'object' && value !== null && !Array.isArray(value)
 }
 
 function typesHeld(value: unknown, found: Set<string>): void {
@@ -199,27 +199,36 @@ function typesHeld(value: unknown, found: Set<string>): void {
   }
 }
 
-test('real-world holds every type the flavour spells but the ones no fetched document holds', () => {
+test('real-world holds every type the flavour spells except the ones no fetched document holds', () => {
   const found = new Set<string>()
   for (const name of names(realWorldRoot, '.json')) typesHeld(JSON.parse(readFileSync(join(realWorldRoot, `${name}.json`), 'utf8')), found)
   const spelled = [...Object.keys(blockNodes), ...Object.keys(inlineNodes), ...markVocabularies.keys()]
   assert.deepEqual(spelled.filter((type) => !found.has(type)).sort(), [...typesNoFetchedDocumentHolds].sort())
 })
 
-test('real-world-sources.json records where each real-world document came from and its licence', () => {
-  const sources: unknown = JSON.parse(readFileSync(join(corpusRoot, 'real-world-sources.json'), 'utf8'))
-  assert.ok(isRecord(sources))
-  assert.deepEqual(Object.keys(sources).sort(), names(realWorldRoot, '.json'))
-  for (const [name, source] of Object.entries(sources)) {
-    assert.ok(isRecord(source), name)
-    assert.deepEqual(Object.keys(source).sort(), ['licence', 'licenceSource', 'retrieved', 'url'], name)
-    assert.match(String(source['licence']), /^CC-BY(-SA)?-\d\.\d$/, name)
-    assert.match(String(source['licenceSource']), /^https:\/\//, name)
-    assert.match(String(source['retrieved']), /^\d{4}-\d{2}-\d{2}$/, name)
-    assert.match(String(source['url']), /^https:\/\/[a-z-]+\.atlassian\.net\//, name)
-  }
+const realWorldSources: unknown = JSON.parse(readFileSync(join(corpusRoot, 'real-world-sources.json'), 'utf8'))
+const realWorldSites: unknown = JSON.parse(readFileSync(join(corpusRoot, '..', 'corpus-gathering', 'sites.json'), 'utf8'))
+
+test('real-world-sources.json has an entry for each real-world document', () => {
+  assert.ok(isRecord(realWorldSources))
+  assert.deepEqual(Object.keys(realWorldSources).sort(), names(realWorldRoot, '.json'))
 })
 
+for (const [name, source] of Object.entries(isRecord(realWorldSources) ? realWorldSources : {})) {
+  test(`real-world-sources.json records ${name}'s page, licence and retrieval`, () => {
+    assert.ok(isRecord(source))
+    const [, site = '', product = ''] = /^(.+)-(jira|wiki)-/.exec(name) ?? []
+    const entry: unknown = Array.isArray(realWorldSites) ? realWorldSites.find((held: unknown) => isRecord(held) && held['site'] === site) : undefined
+    assert.ok(isRecord(entry) && isRecord(entry['products']))
+    const { licence, licenceSource, retrieved, url } = source
+    assert.deepEqual({ licence, licenceSource }, entry['products'][product])
+    assert.deepEqual(Object.keys(source).sort(), ['licence', 'licenceSource', 'retrieved', 'url'])
+    assert.equal(new Date(String(retrieved)).toISOString(), retrieved)
+    // A page's title or an attachment's name in the link would put original text in the repo.
+    assert.match(String(url), new RegExp(`^https://${site}\\.atlassian\\.net/(browse/[A-Z0-9-]+|wiki/spaces/[^/?]+/(overview|pages/\\d+|blog/\\d+/\\d+/\\d+/\\d+))(\\?focusedCommentId=\\d+)?$`))
+  })
+}
+
 // The position the input itself gives an offset, recomputed rather than trusted from the parser.
 function lineStarting(markdown: string, offset: number): { line: number; offset: number } | undefined {
   const before = markdown.slice(0, offset)