3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-21 07:32:58 +01:00

Gather an anonymized corpus of real-world ADF documents with each one's source and licence

This commit is contained in:
2026-11-04 21:50:40 +01:00
committed by Lillem4n
parent dbaa6823aa
commit 59954f93b3
168 changed files with 25247 additions and 45 deletions
+223
View File
@@ -0,0 +1,223 @@
// A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
const formatKeys = new Set([
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
])
const schemes = new Set(['file', 'ftp', 'http', 'https', 'mailto', 'tel'])
const token = /[\p{L}\p{N}][\p{L}\p{M}\p{N}]*/gu
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
const shortestLeak = 3
const redraws = 64
function isScheme(text, match, offset) {
return schemes.has(match) && text[offset + match.length] === ':'
}
function tokensOf(text) {
return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
}
// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
function walk(value, path, visit) {
if (Array.isArray(value)) {
value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
} else if (typeof value === 'object' && value !== null) {
for (const [key, member] of Object.entries(value)) {
const memberPath = `${path}/${key}`
if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
else walkFree(member, memberPath, visit)
}
} else {
visit(value, path, 'free')
}
}
function walkAttributes(attributes, path, visit, nodeType) {
for (const [key, member] of Object.entries(attributes)) {
const memberPath = `${path}/${key}`
if (formatKeys.has(key)) visit(member, memberPath, 'kept')
else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
else walkFree(member, memberPath, visit)
}
}
function walkFree(value, path, visit) {
if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
else visit(value, path, 'free')
}
function replacedStrings(document) {
const strings = []
walk(document, '', (value, _path, kind) => {
if (kind !== 'kept' && typeof value === 'string') strings.push(value)
})
return strings
}
function fold(found) {
return found.toLowerCase()
}
function heldTokens(document) {
return new Set(replacedStrings(document).flatMap(tokensOf).map(fold).filter((found) => found.length >= shortestLeak))
}
function cursor(pool) {
let next = 0
return () => pool[next++ % pool.length]
}
function stream(pool) {
const nextPiece = cursor(pool)
let buffer = ''
return (length) => {
while (buffer.length < length) buffer += nextPiece()
const taken = buffer.slice(0, length)
buffer = buffer.slice(length)
return taken
}
}
function wordDrawer(words) {
const lengths = [...new Set(words.map((word) => word.length))]
const byLength = new Map(lengths.map((length) => [length, cursor(words.filter((word) => word.length === length))]))
return (length) => {
const nearest = lengths.reduce((best, candidate) => (Math.abs(candidate - length) < Math.abs(best - length) ? candidate : best))
return byLength.get(nearest)()
}
}
function withCaseOf(original, word) {
if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
return word
}
function replacer(document, draws) {
const held = heldTokens(document)
const digits = stream(draws.digits)
const hex = stream(draws.hex)
const letters = stream(draws.letters)
const nextPerson = cursor(draws.persons)
const nextUuid = cursor(draws.uuids)
const nextWord = wordDrawer(draws.words)
const spelled = new Map()
function fresh(draw) {
let drawn = draw()
for (let tries = 0; tries < redraws && tokensOf(drawn).some((found) => held.has(fold(found))); tries += 1) drawn = draw()
return drawn
}
function number(original) {
const drawn = fresh(() => digits(original.length))
return original[0] !== '0' && drawn[0] === '0' ? `1${drawn.slice(1)}` : drawn
}
function mixed(original) {
if (/^[0-9a-f]+$/.test(original)) return fresh(() => hex(original.length))
return fresh(() => [...original].map((character) => {
if (/\d/.test(character)) return digits(1)
if (!/\p{L}/u.test(character)) return character
return character === character.toLowerCase() ? letters(1) : letters(1).toUpperCase()
}).join(''))
}
function word(original) {
if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
let tries = 0
// A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
return withCaseOf(original, fresh(() => {
tries += 1
return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
}))
}
function spellToken(original) {
if (/^\d+$/.test(original)) return number(original)
if (/^\p{L}[\p{L}\p{M}]*$/u.test(original)) return word(original)
return mixed(original)
}
function memo(original, spell) {
if (!spelled.has(original)) spelled.set(original, spell(original))
return spelled.get(original)
}
function string(original) {
if (uuid.test(original)) return memo(original, () => fresh(nextUuid))
return original.replaceAll(token, (match, offset) => (isScheme(original, match, offset) ? match : memo(match, spellToken)))
}
function person(original) {
return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
}
function scalar(value) {
if (typeof value === 'string') return string(value)
if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
return value
}
return { person, scalar }
}
function setAt(target, path, value) {
const steps = path.split('/').slice(1)
const last = steps.pop()
let holder = target
for (const step of steps) holder = holder[step]
holder[last] = value
}
// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
export function anonymize(document, draws) {
const { person, scalar } = replacer(document, draws)
const written = structuredClone(document)
walk(document, '', (value, path, kind) => {
if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
else if (kind === 'free') setAt(written, path, scalar(value))
})
return written
}
function structure(document) {
const found = new Map()
walk(document, '', (value, path, kind) => {
found.set(path, { kind, value })
})
return found
}
function skeleton(text) {
return text.replaceAll(token, 'x')
}
function holdsSameShape(before, after) {
if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
if (typeof before.value !== typeof after.value) return false
return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
}
// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
export function shapeDifferences(before, after) {
const beforeValues = structure(before)
const afterValues = structure(after)
const paths = [...new Set([...beforeValues.keys(), ...afterValues.keys()])]
return paths.filter((path) => {
const first = beforeValues.get(path)
const second = afterValues.get(path)
return first === undefined || second === undefined || !holdsSameShape(first, second)
}).sort()
}
// The tokens of `before`'s replaced values that `after`'s replaced values still hold.
export function leakedTokens(before, after) {
const held = heldTokens(after)
return [...heldTokens(before)].filter((found) => held.has(found)).sort()
}
+140
View File
@@ -0,0 +1,140 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
const draws = {
digits: ['7391', '2846'],
hex: ['9f3ac07be1d4'],
letters: ['qkxwzj'],
persons: ['Pat Smith', 'Lo Berg'],
uuids: ['0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b', '0190a1b2-c3d4-7e5f-8a9b-1c1d2e3f4a5c'],
words: ['amber', 'clever', 'cobalt', 'merry', 'otter', 'cliff', 'tundra', 'pumice', 'lantern', 'harbour'],
}
function paragraph(...content) {
return { attrs: {}, content: [{ content, type: 'paragraph' }], type: 'doc', version: 1 }
}
function textOf(document) {
return document.content[0].content[0].text
}
test('replaces each word and keeps the whitespace and punctuation around it', () => {
const written = textOf(anonymize(paragraph({ text: ' Deploy *the* secret-name, now!\n', type: 'text' }), draws))
assert.match(written, /^ {2}[A-Z][a-z]+ \*[a-z]+\* [a-z]+-[a-z]+, [a-z]+!\n$/)
for (const word of ['Deploy', 'the', 'secret', 'name', 'now']) assert.ok(!written.includes(word))
})
test('keeps a word in capitals in capitals', () => {
assert.match(textOf(anonymize(paragraph({ text: 'NOTE', type: 'text' }), draws)), /^[A-Z]+$/)
})
test('keeps the length of a number and of a mixed token', () => {
assert.match(textOf(anonymize(paragraph({ text: '2024-12 v2Beta', type: 'text' }), draws)), /^[1-9]\d{3}-[1-9]\d [a-z]\d[A-Z][a-z]{3}$/)
})
test('spells a repeated word the same way everywhere in a document', () => {
const written = anonymize(paragraph({ text: 'alpha beta alpha', type: 'text' }, { text: 'alpha', type: 'text' }), draws).content[0].content
const [first, , third] = written[0].text.split(' ')
assert.equal(first, third)
assert.equal(written[1].text, first)
})
test('keeps a link text equal to its href, and the scheme of both', () => {
const href = 'https://wiki.example.org/display/page?id=12'
const [node] = anonymize(paragraph({ marks: [{ attrs: { href }, type: 'link' }], text: href, type: 'text' }), draws).content[0].content
assert.equal(node.text, node.marks[0].attrs.href)
assert.match(node.text, /^https:\/\/[a-z]+\.[a-z]+\.[a-z]+\/[a-z]+\/[a-z]+\?[a-z]{1,3}=[1-9]\d$/)
assert.ok(!node.text.includes('example'))
})
test('keeps node types, mark types and attribute keys', () => {
const document = paragraph({ marks: [{ type: 'strong' }, { attrs: { color: '#ff0000' }, type: 'textColor' }], text: 'word', type: 'text' }, { type: 'hardBreak' }, { attrs: { custom: 'value' }, type: 'someFutureNode' })
const [text, hardBreak, future] = anonymize(document, draws).content[0].content
assert.deepEqual(text.marks, [{ type: 'strong' }, { attrs: { color: '#ff0000' }, type: 'textColor' }])
assert.deepEqual(hardBreak, { type: 'hardBreak' })
assert.equal(future.type, 'someFutureNode')
assert.deepEqual(Object.keys(future.attrs), ['custom'])
assert.notEqual(future.attrs.custom, 'value')
})
test('keeps an attribute naming a format, and a layout number', () => {
const panel = { attrs: { panelType: 'info' }, content: [], type: 'panel' }
const code = { attrs: { language: 'python' }, content: [], type: 'codeBlock' }
const table = { attrs: { isNumberColumnEnabled: false, layout: 'center', width: 760 }, content: [], type: 'table' }
const [kept, keptCode, keptTable] = anonymize({ content: [panel, code, table], type: 'doc', version: 1 }, draws).content
assert.deepEqual(kept.attrs, { panelType: 'info' })
assert.deepEqual(keptCode.attrs, { language: 'python' })
assert.deepEqual(keptTable.attrs, { isNumberColumnEnabled: false, layout: 'center', width: 760 })
})
test('replaces a person, an account id and a uuid, keeping each shape', () => {
const uuid = '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e'
const accountId = '60f1c2d3e4a5b6c7d8e9f012'
const [mention] = anonymize(paragraph({ attrs: { accessLevel: 'CONTAINER', id: accountId, localId: uuid, text: '@Real Person' }, type: 'mention' }), draws).content[0].content
assert.equal(mention.attrs.accessLevel, 'CONTAINER')
assert.match(mention.attrs.id, /^[0-9a-f]{24}$/)
assert.notEqual(mention.attrs.id, accountId)
assert.match(mention.attrs.localId, /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)
assert.notEqual(mention.attrs.localId, uuid)
assert.match(mention.attrs.text, /^@[A-Z][a-z]+ [A-Z][a-z]+$/)
assert.ok(!mention.attrs.text.includes('Person'))
})
test('keeps a date timestamp a number of the same length', () => {
const [date] = anonymize(paragraph({ attrs: { timestamp: '1700000000000' }, type: 'date' }), draws).content[0].content
assert.match(date.attrs.timestamp, /^[1-9]\d{12}$/)
assert.notEqual(date.attrs.timestamp, '1700000000000')
})
test('replaces everything inside a free-form attribute, whatever its keys', () => {
const extension = { attrs: { extensionKey: 'toc', extensionType: 'com.example.macro', parameters: { layout: 'secret', pageId: 42871, type: 'hidden' } }, type: 'extension' }
const [written] = anonymize({ content: [extension], type: 'doc', version: 1 }, draws).content
assert.equal(written.attrs.extensionKey, 'toc')
assert.equal(written.attrs.extensionType, 'com.example.macro')
assert.deepEqual(Object.keys(written.attrs.parameters), ['layout', 'pageId', 'type'])
assert.notEqual(written.attrs.parameters.layout, 'secret')
assert.notEqual(written.attrs.parameters.type, 'hidden')
assert.notEqual(written.attrs.parameters.pageId, 42871)
assert.match(String(written.attrs.parameters.pageId), /^[1-9]\d{4}$/)
})
test('never draws a word the document already holds', () => {
const written = textOf(anonymize(paragraph({ text: 'amber clever', type: 'text' }), draws))
assert.ok(!written.includes('amber'))
assert.ok(!written.includes('clever'))
})
test('never draws a uuid holding a part of an original one', () => {
const original = '1c5f0e3a-7958-4eb4-b3d1-2d9b6e417c80'
const [mention] = anonymize(paragraph({ attrs: { localId: original }, type: 'mention' }), { ...draws, uuids: ['7baa5903-cd2a-7958-ac4d-6f3ee1010175', draws.uuids[0]] }).content[0].content
assert.equal(mention.attrs.localId, draws.uuids[0])
})
test('never runs out of draws, however many words a document holds', () => {
const text = Array.from({ length: 500 }, (_, index) => `word${'x'.repeat(index % 9)}`).join(' ')
assert.equal(textOf(anonymize(paragraph({ text, type: 'text' }), draws)).split(' ').length, 500)
})
test('finds no difference between a document and its anonymized form', () => {
const document = paragraph({ marks: [{ type: 'em' }], text: 'Some text, 12', type: 'text' }, { attrs: { width: 3 }, type: 'media' })
assert.deepEqual(shapeDifferences(document, anonymize(document, draws)), [])
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
})
test('reports a changed structure with its path', () => {
const before = paragraph({ marks: [{ type: 'em' }], text: 'a b', type: 'text' })
const after = paragraph({ marks: [{ type: 'strong' }], text: 'c-d', type: 'text' })
assert.deepEqual(shapeDifferences(before, after), ['/content/0/content/0/marks/0/type', '/content/0/content/0/text'])
})
test('reports a word the output still holds', () => {
const before = paragraph({ text: 'private word', type: 'text' })
const after = paragraph({ text: 'private other', type: 'text' })
assert.deepEqual(leakedTokens(before, after), ['private'])
})
test('does not call a kept attribute value a leak', () => {
const document = { attrs: {}, content: [{ attrs: { language: 'python' }, content: [{ text: 'python', type: 'text' }], type: 'codeBlock' }], type: 'doc', version: 1 }
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
})
+75
View File
@@ -0,0 +1,75 @@
// Usage: node fetch.mjs <sites.json> <raw folder>. The folder stays outside the repo: its documents hold the sites' real text.
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { setTimeout as delay } from 'node:timers/promises'
// Robots.txt on these sites asks for one second between requests.
const crawlDelay = 1000
const pagesPerSource = 20
const [sitesPath, rawFolder] = process.argv.slice(2)
const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
const documentsFolder = join(rawFolder, 'documents')
mkdirSync(documentsFolder, { recursive: true })
async function getJson(url) {
await delay(crawlDelay)
const response = await fetch(url, { headers: { Accept: 'application/json', 'User-Agent': 'adf-codec-corpus-gathering (https://github.com/larvit/adf-codec)' } })
if (response.status === 429) {
await delay(Number(response.headers.get('retry-after') ?? 60) * 1000)
return getJson(url)
}
if (!response.ok) throw new Error(`${url} answered ${response.status}`)
return response.json()
}
let written = 0
function keep(name, source, document) {
if (document?.type !== 'doc' || !Array.isArray(document.content) || document.content.length === 0) return
writeFileSync(join(documentsFolder, `${name}.json`), JSON.stringify({ document, source }))
written += 1
}
async function fetchConfluence(site, collection) {
const base = `https://${site}.atlassian.net`
let next = `/wiki/api/v2/${collection}?limit=100&body-format=atlas_doc_format`
for (let page = 0; next !== undefined && page < pagesPerSource; page += 1) {
const answer = await getJson(`${base}${next}`)
for (const result of answer.results) {
const body = result.body?.atlas_doc_format?.value
if (body) keep(`${site}-wiki-${collection}-${result.id}`, `${base}/wiki/api/v2/${collection}/${result.id}`, JSON.parse(body))
}
next = answer._links?.next
}
}
async function fetchJira(site) {
const base = `https://${site}.atlassian.net`
let token
for (let page = 0; page < pagesPerSource; page += 1) {
const query = new URLSearchParams({ fields: 'comment,description', jql: 'created >= "2015-01-01" ORDER BY created DESC', maxResults: '100', ...(token ? { nextPageToken: token } : {}) })
const answer = await getJson(`${base}/rest/api/3/search/jql?${query}`)
for (const issue of answer.issues) {
keep(`${site}-jira-${issue.key}-description`, `${base}/rest/api/3/issue/${issue.key}?fields=description`, issue.fields.description)
for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, `${base}/rest/api/3/issue/${issue.key}/comment/${comment.id}`, comment.body)
}
token = answer.nextPageToken
if (token === undefined) break
}
}
for (const { products, site } of sites) {
for (const product of Object.keys(products)) {
try {
if (product === 'wiki') {
await fetchConfluence(site, 'pages')
await fetchConfluence(site, 'footer-comments')
} else {
await fetchJira(site)
}
} catch (error) {
console.error(`${site} ${product}: ${error.message}`)
}
console.error(`${site} ${product}: ${written} documents so far`)
}
}
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# Usage: corpus-gathering/run.sh <raw folder>. Keep the raw folder outside the repo: its documents hold the sites' real text.
set -eu
here=$(cd "$(dirname "$0")" && pwd)
raw=$(mkdir -p "$1" && cd "$1" && pwd)
fejkdata_image=larvit/fejkdata:0.2.0
node_image=node:26.13.0-alpine3.24
in_node() {
docker run --rm --memory 2g --memory-swap 2g --ulimit core=0 -u "$(id -u):$(id -g)" -e HOME=/tmp -v "$here/..:/repo" -v "$raw:/raw" -w /repo "$node_image" "$@"
}
# Takes a draws file's name, the number of lines and the template of a line.
draw() {
printf '%s\n' "$3" | docker run --rm -i --memory 256m --memory-swap 256m "$fejkdata_image" --seed 163 --repeat "$2" > "$raw/draws/$1.txt"
}
[ -d "$raw/documents" ] || in_node node corpus-gathering/fetch.mjs corpus-gathering/sites.json /raw
mkdir -p "$raw/draws"
draw digits 5000 '{digits(32)}'
draw hex 5000 '{hex(32)}'
draw letters 5000 '{lower(32)}'
draw persons 2000 '{/en_US.person}'
draw uuids 20000 '{uuid()}'
draw words 20000 '{/en_US.word}'
in_node node corpus-gathering/write-corpus.mjs corpus-gathering/sites.json /raw corpus/real-world corpus/real-world-sources.json
+72
View File
@@ -0,0 +1,72 @@
function markSet(node) {
return (node.marks ?? []).map((mark) => mark.type).sort().join('+')
}
function inlineLabel(node) {
return node.type === 'text' ? `text[${markSet(node)}]` : node.type
}
function collect(node, parentType, found) {
found.add(`node:${node.type}`)
if (parentType !== undefined) found.add(`child:${parentType}>${node.type}`)
for (const key of Object.keys(node.attrs ?? {})) found.add(`attr:${node.type}.${key}`)
for (const mark of node.marks ?? []) {
found.add(`mark:${mark.type}`)
for (const key of Object.keys(mark.attrs ?? {})) found.add(`attr:${mark.type}.${key}`)
}
if (node.type === 'text' && (node.marks ?? []).length > 1) found.add(`marks:${markSet(node)}`)
const children = node.content ?? []
children.forEach((child, index) => {
collect(child, node.type, found)
const next = children[index + 1]
if (next !== undefined) found.add(`beside:${inlineLabel(child)}|${inlineLabel(next)}`)
})
}
// What a document shows the corpus: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
export function features(document) {
const found = new Set()
collect(document, undefined, found)
return found
}
function isType(feature) {
return feature.startsWith('node:') || feature.startsWith('mark:')
}
function bestCandidate(candidates, covered) {
let best
let bestScore = 0
let bestShowsType = false
for (const candidate of candidates) {
let added = 0
let showsType = false
for (const feature of candidate.features) {
if (covered.has(feature)) continue
added += 1
showsType ||= isType(feature)
}
const score = added / candidate.size
if (added > 0 && (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore)) {
best = candidate
bestScore = score
bestShowsType = showsType
}
}
return best
}
// Takes the candidate showing the most new features per byte, new node and mark types first, until `limit` or until none shows a new one.
export function selectDocuments(candidates, limit) {
const covered = new Set()
const chosen = []
let remaining = candidates
while (chosen.length < limit) {
const best = bestCandidate(remaining, covered)
if (best === undefined) break
chosen.push(best)
for (const feature of best.features) covered.add(feature)
remaining = remaining.filter((candidate) => candidate !== best)
}
return chosen
}
+40
View File
@@ -0,0 +1,40 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { features, selectDocuments } from './select.mjs'
function paragraph(...content) {
return { content: [{ content, type: 'paragraph' }], type: 'doc', version: 1 }
}
test('names node and mark types, attribute keys and nesting', () => {
const found = features(paragraph({ marks: [{ attrs: { href: 'x' }, type: 'link' }], text: 'a', type: 'text' }))
for (const feature of ['node:doc', 'node:paragraph', 'node:text', 'mark:link', 'attr:link.href', 'child:paragraph>text']) assert.ok(found.has(feature), feature)
})
test('names marks held together', () => {
assert.ok(features(paragraph({ marks: [{ type: 'strong' }, { type: 'em' }], text: 'a', type: 'text' })).has('marks:em+strong'))
})
test('names bold beside a code span and italics beside a hard break', () => {
const found = features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'code' }], text: 'b', type: 'text' }, { type: 'hardBreak' }, { marks: [{ type: 'em' }], text: 'c', type: 'text' }))
assert.ok(found.has('beside:text[strong]|text[code]'))
assert.ok(found.has('beside:hardBreak|text[em]'))
})
test('takes the smaller of two candidates showing the same features, and stops when nothing is new', () => {
const big = { features: new Set(['a', 'b']), size: 100 }
const small = { features: new Set(['a', 'b']), size: 10 }
const other = { features: new Set(['c']), size: 50 }
assert.deepEqual(selectDocuments([big, small, other], 10), [small, other])
})
test('takes a candidate showing a new node or mark type ahead of any other', () => {
const rare = { features: new Set(['mark:breakout']), size: 1000 }
const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
assert.deepEqual(selectDocuments([cheap, rare], 1), [rare])
})
test('stops at the limit', () => {
const candidates = [{ features: new Set(['a']), size: 1 }, { features: new Set(['b']), size: 1 }]
assert.equal(selectDocuments(candidates, 1).length, 1)
})
+72
View File
@@ -0,0 +1,72 @@
[
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-SA-4.0",
"licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
}
},
"site": "lf-onap"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-opendaylight"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-akraino"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-anuket"
},
{
"products": {
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-hyperledger"
},
{
"products": {
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-edgexfoundry"
}
]
+53
View File
@@ -0,0 +1,53 @@
// Usage: node write-corpus.mjs <sites.json> <raw folder> <documents folder> <sources file>. The raw folder holds fetch.mjs's documents and run.sh's draws.
import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
import { features, selectDocuments } from './select.mjs'
const documentLimit = 150
const largestDocument = 64 * 1024
const [sitesPath, rawFolder, corpusFolder, sourcesPath] = process.argv.slice(2)
function lines(name) {
return readFileSync(join(rawFolder, 'draws', `${name}.txt`), 'utf8').split('\n').filter((line) => line !== '')
}
function sorted(value) {
if (Array.isArray(value)) return value.map(sorted)
if (typeof value !== 'object' || value === null) return value
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]))
}
function canonical(value) {
return `${JSON.stringify(sorted(value), null, 2)}\n`
}
const licences = new Map(JSON.parse(readFileSync(sitesPath, 'utf8')).flatMap(({ products, site }) => Object.entries(products).map(([product, licence]) => [`${site}-${product}`, licence])))
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), persons: lines('persons'), uuids: lines('uuids'), words: lines('words') }
const documentsFolder = join(rawFolder, 'documents')
const candidates = readdirSync(documentsFolder).flatMap((file) => {
const path = join(documentsFolder, file)
const text = readFileSync(path, 'utf8')
if (text.length > largestDocument) return []
const { document, source } = JSON.parse(text)
const name = file.slice(0, -'.json'.length)
const written = anonymize(document, draws)
const differences = shapeDifferences(document, written)
const leaks = leakedTokens(document, written)
if (differences.length > 0 || leaks.length > 0) {
console.error(`${name} left out: ${differences.length} shape differences, ${leaks.length} tokens still held`)
return []
}
return [{ features: features(document), name, retrieved: statSync(path).mtime.toISOString().slice(0, 10), size: text.length, source, written }]
})
rmSync(corpusFolder, { force: true, recursive: true })
mkdirSync(corpusFolder, { recursive: true })
const sources = {}
for (const { name, retrieved, source, written } of selectDocuments(candidates, documentLimit)) {
writeFileSync(join(corpusFolder, `${name}.json`), canonical(written))
const [, site, product] = /^(.+)-(jira|wiki)-/.exec(name)
sources[name] = { ...licences.get(`${site}-${product}`), retrieved, url: source }
}
writeFileSync(sourcesPath, canonical(sources))
console.error(`${Object.keys(sources).length} documents of ${candidates.length} candidates`)