3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-20 15:15:44 +01:00

Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo

This commit is contained in:
2026-11-05 00:17:32 +01:00
committed by Lillem4n
parent 8fee35a63b
commit 1e3d6e5d65
51 changed files with 3164 additions and 1055 deletions
+46 -49
View File
@@ -1,6 +1,6 @@
// A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
const formatKeys = new Set([
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode',
'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
])
@@ -10,7 +10,7 @@ const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12
const uuid = new RegExp(`^${uuidPattern}$`, 'i')
const uuidOrToken = new RegExp(`(?<![\\p{L}\\p{N}])${uuidPattern}(?![\\p{L}\\p{M}\\p{N}])|${token.source}`, 'giu')
const uuidVersionAt = 14
// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
// A chance match below this length identifies nobody, and a document may hold every such token, so redrawing around them might never end.
const shortestLeak = 3
const redraws = 64
@@ -22,47 +22,45 @@ function tokensOf(text) {
return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
}
// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
function walk(value, path, visit) {
// Calls `visit` with each leaf, its steps from the document and its kind: `kept`, or `free`, which the anonymizer replaces.
function walk(value, steps, visit) {
if (Array.isArray(value)) {
value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
value.forEach((item, index) => walk(item, [...steps, index], visit))
} else if (typeof value === 'object' && value !== null) {
for (const [key, member] of Object.entries(value)) {
const memberPath = `${path}/${key}`
if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
else walkFree(member, memberPath, visit)
const memberSteps = [...steps, key]
if (key === 'attrs') walkAttributes(member, memberSteps, visit)
else if (key === 'content' || key === 'marks') walk(member, memberSteps, visit)
else if (key === 'type' || key === 'version') visit(member, memberSteps, 'kept')
else walkFree(member, memberSteps, visit)
}
} else {
visit(value, path, 'free')
visit(value, steps, 'free')
}
}
// `<app id>/<environment id>/...`, so one more id, where any other extension key names a macro.
// A Forge key reads `<app id>/<environment id>/...`, so it holds ids to replace. Any other extension key names a macro and stays.
function isForgeKey(key, value) {
return key === 'extensionKey' && String(value).includes('/')
}
function walkAttributes(attributes, path, visit, nodeType) {
function walkAttributes(attributes, steps, visit) {
for (const [key, member] of Object.entries(attributes)) {
const memberPath = `${path}/${key}`
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept')
else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
else walkFree(member, memberPath, visit)
const memberSteps = [...steps, key]
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberSteps, 'kept')
else walkFree(member, memberSteps, visit)
}
}
function walkFree(value, path, visit) {
if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
else visit(value, path, 'free')
function walkFree(value, steps, visit) {
if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, [...steps, key], visit)
else visit(value, steps, 'free')
}
function replacedStrings(document) {
const strings = []
walk(document, '', (value, _path, kind) => {
if (kind !== 'kept' && typeof value === 'string') strings.push(value)
walk(document, [], (value, _steps, kind) => {
if (kind === 'free' && typeof value === 'string') strings.push(value)
})
return strings
}
@@ -101,7 +99,7 @@ function wordDrawer(words) {
}
function withCaseOf(original, word) {
if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
if (original === original.toUpperCase() && original !== original.toLowerCase()) return word.toUpperCase()
if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
return word
}
@@ -111,7 +109,6 @@ function replacer(document, draws) {
const digits = stream(draws.digits)
const hex = stream(draws.hex)
const letters = stream(draws.letters)
const nextPerson = cursor(draws.persons)
const nextUuid = cursor(draws.uuids)
const nextWord = wordDrawer(draws.words)
const spelled = new Map()
@@ -139,7 +136,7 @@ function replacer(document, draws) {
function word(original) {
if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
let tries = 0
// A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
// A long document can hold every word of a length, so the last two of the draws `fresh` makes gain a suffix.
return withCaseOf(original, fresh(() => {
tries += 1
return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
@@ -172,42 +169,34 @@ function replacer(document, draws) {
})
}
function person(original) {
return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
}
function scalar(value) {
if (typeof value === 'string') return string(value)
if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
return value
}
return { person, scalar }
return scalar
}
function setAt(target, path, value) {
const steps = path.split('/').slice(1)
const last = steps.pop()
let holder = target
for (const step of steps) holder = holder[step]
holder[last] = value
function setAt(target, steps, value) {
const holder = steps.slice(0, -1).reduce((held, step) => held[step], target)
holder[steps.at(-1)] = value
}
// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
// `draws` is fejkdata's output, as run.sh renders it.
export function anonymize(document, draws) {
const { person, scalar } = replacer(document, draws)
const scalar = replacer(document, draws)
const written = structuredClone(document)
walk(document, '', (value, path, kind) => {
if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
else if (kind === 'free') setAt(written, path, scalar(value))
walk(document, [], (value, steps, kind) => {
if (kind === 'free') setAt(written, steps, scalar(value))
})
return written
}
function structure(document) {
const found = new Map()
walk(document, '', (value, path, kind) => {
found.set(path, { kind, value })
walk(document, [], (value, steps, kind) => {
found.set(`/${steps.join('/')}`, { kind, value })
})
return found
}
@@ -217,12 +206,12 @@ function skeleton(text) {
}
function holdsSameShape(before, after) {
if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
if (before.kind === 'kept') return JSON.stringify(before.value) === JSON.stringify(after.value)
if (typeof before.value !== typeof after.value) return false
return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
return typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
}
// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
// The paths where `after` differs from `before` in anything but the tokens the anonymizer replaces.
export function shapeDifferences(before, after) {
const beforeValues = structure(before)
const afterValues = structure(after)
@@ -231,11 +220,19 @@ export function shapeDifferences(before, after) {
const first = beforeValues.get(path)
const second = afterValues.get(path)
return first === undefined || second === undefined || !holdsSameShape(first, second)
}).sort()
})
}
// The tokens of `before`'s replaced values that `after`'s replaced values still hold.
export function leakedTokens(before, after) {
const held = heldTokens(after)
return [...heldTokens(before)].filter((found) => held.has(found)).sort()
return [...heldTokens(before)].filter((found) => held.has(found))
}
// The anonymized document, or what keeps it out of the corpus: a slip here would publish real text, and nothing later could see it.
export function anonymizeChecked(document, draws) {
const written = anonymize(document, draws)
const differences = shapeDifferences(document, written)
const leaks = leakedTokens(document, written)
return differences.length > 0 || leaks.length > 0 ? { differences, leaks, ok: false } : { ok: true, written }
}
+133 -12
View File
@@ -1,12 +1,12 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
import { anonymize, anonymizeChecked, leakedTokens, shapeDifferences } from './anonymize.mjs'
const draws = {
digits: ['7391', '2846'],
hex: ['9f3ac07be1d4'],
letters: ['qkxwzj'],
persons: ['Pat Smith', 'Lo Berg'],
uuids: ['0190a1b2-c3d4-4e5f-8a9b-0c1d2e3f4a5b', '0190a1b2-c3d4-4e5f-8a9b-1c1d2e3f4a5c'],
words: ['amber', 'clever', 'cobalt', 'merry', 'otter', 'cliff', 'tundra', 'pumice', 'lantern', 'harbour'],
}
@@ -22,7 +22,7 @@ function textOf(document) {
test('replaces each word and keeps the whitespace and punctuation around it', () => {
const written = textOf(anonymize(paragraph({ text: ' Deploy *the* secret-name, now!\n', type: 'text' }), draws))
assert.match(written, /^ {2}[A-Z][a-z]+ \*[a-z]+\* [a-z]+-[a-z]+, [a-z]+!\n$/)
for (const word of ['Deploy', 'the', 'secret', 'name', 'now']) assert.ok(!written.includes(word))
assert.doesNotMatch(written, /Deploy|the|secret|name|now/)
})
test('keeps a word in capitals in capitals', () => {
@@ -81,13 +81,13 @@ test('replaces a person, an account id and a uuid, keeping each shape', () => {
assert.ok(!mention.attrs.text.includes('Person'))
})
test('keeps the version of every uuid it replaces', () => {
for (const [version, original] of [['1', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], ['3', '6fa459ea-ee8a-3ca4-894e-db77e160355e'], ['4', '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e']]) {
for (const [version, original] of [['1', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], ['3', '6fa459ea-ee8a-3ca4-894e-db77e160355e'], ['4', '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e']]) {
test(`keeps the version of a v${version} uuid it replaces`, () => {
const [mention] = anonymize(paragraph({ attrs: { localId: original }, type: 'mention' }), draws).content[0].content
assert.match(mention.attrs.localId, new RegExp(`^[0-9a-f]{8}-[0-9a-f]{4}-${version}[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$`))
assert.notEqual(mention.attrs.localId, original)
}
})
})
}
test('keeps a uuid in capitals in capitals', () => {
const [mention] = anonymize(paragraph({ attrs: { localId: '6FA459EA-EE8A-3CA4-894E-DB77E160355E' }, type: 'mention' }), draws).content[0].content
@@ -95,7 +95,7 @@ test('keeps a uuid in capitals in capitals', () => {
assert.ok(/[A-F]/.test(mention.attrs.localId))
})
test('keeps a uuid inside a longer value a uuid of its version', () => {
test('keeps the version of a uuid inside a longer value', () => {
const inside = '[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}'
const accountId = '712020:abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
const href = 'https://wiki.example.org/page?id=abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
@@ -111,7 +111,7 @@ test('reports a uuid whose version changed', () => {
assert.deepEqual(shapeDifferences(before, after), ['/content/0/content/0/attrs/id'])
})
test('keeps a date timestamp a number of the same length', () => {
test('replaces a date timestamp with a number of the same length', () => {
const [date] = anonymize(paragraph({ attrs: { timestamp: '1700000000000' }, type: 'date' }), draws).content[0].content
assert.match(date.attrs.timestamp, /^[1-9]\d{12}$/)
assert.notEqual(date.attrs.timestamp, '1700000000000')
@@ -160,9 +160,104 @@ test('never draws a uuid holding a part of an original one', () => {
assert.equal(mention.attrs.localId, draws.uuids[0])
})
test('never runs out of draws, however many words a document holds', () => {
const text = Array.from({ length: 500 }, (_, index) => `word${'x'.repeat(index % 9)}`).join(' ')
assert.equal(textOf(anonymize(paragraph({ text, type: 'text' }), draws)).split(' ').length, 500)
test('never runs out of draws when a document holds every word of the pool', () => {
const document = paragraph({ text: draws.words.join(' '), type: 'text' })
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
})
test('ends and reports the leak when every draw holds the original', () => {
const document = paragraph({ text: '123', type: 'text' })
assert.deepEqual(leakedTokens(document, anonymize(document, { ...draws, digits: ['123'] })), ['123'])
})
test('replaces a one-word mention with one word', () => {
const [mention] = anonymize(paragraph({ attrs: { id: 'x', text: '@Cher' }, type: 'mention' }), draws).content[0].content
assert.match(mention.attrs.text, /^@[A-Z][a-z]+$/)
})
test('spells a name the same way in a mention and in text', () => {
const [mention, text] = anonymize(paragraph({ attrs: { text: '@Bob' }, type: 'mention' }, { text: 'Bob said', type: 'text' }), draws).content[0].content
assert.equal(mention.attrs.text.slice(1), text.text.split(' ')[0])
})
test('keeps every attribute naming a format, and the version', () => {
const attrs = Object.fromEntries(['__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode', 'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth', 'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType'].map((key) => [key, 'zebra']))
const [written] = anonymize({ content: [{ attrs: { ...attrs, language: 'c/c++' }, type: 'someNode' }], type: 'doc', version: 1 }, draws).content
assert.deepEqual(written.attrs, { ...attrs, language: 'c/c++' })
assert.equal(anonymize(paragraph(), draws).version, 1)
})
test('spells a word with the nearest length the pool holds, in its case', () => {
const pool = { ...draws, words: ['amber', 'clever', 'lantern'] }
assert.equal(textOf(anonymize(paragraph({ text: 'Alpha', type: 'text' }), pool)), 'Amber')
assert.equal(textOf(anonymize(paragraph({ text: 'channel', type: 'text' }), pool)), 'lantern')
assert.equal(textOf(anonymize(paragraph({ text: 'NINELONGS', type: 'text' }), pool)), 'LANTERN')
assert.equal(textOf(anonymize(paragraph({ text: 'tiny', type: 'text' }), pool)), 'amber')
assert.equal(textOf(anonymize(paragraph({ text: 'Bob', type: 'text' }), pool)), 'Amber')
})
test('spells a short word from the letters, keeping its case', () => {
assert.equal(textOf(anonymize(paragraph({ text: 'Al', type: 'text' }), draws)), 'Qk')
})
test('reads a word with an accent as a word', () => {
assert.equal(textOf(anonymize(paragraph({ text: 'caf\u00e9', type: 'text' }), { ...draws, words: ['amber'] })), 'amber')
})
test('spells a word of a caseless script in small letters', () => {
assert.match(textOf(anonymize(paragraph({ text: '\u6f22\u5b57 \u6f22\u5b57\u6f22', type: 'text' }), draws)), /^[a-z]+ [a-z]+$/)
})
test('keeps letters and digits in their places in a mixed token', () => {
assert.match(textOf(anonymize(paragraph({ text: 'beta2 2beta a1x xa1', type: 'text' }), draws)), /^[a-z]{4}\d \d[a-z]{4} [a-z]\d[a-z] [a-z]{2}\d$/)
})
test('spells a hex token from the hex draws', () => {
assert.equal(textOf(anonymize(paragraph({ text: '3f2a', type: 'text' }), draws)), '9f3a')
})
test('keeps a combining mark in a mixed token', () => {
assert.match(textOf(anonymize(paragraph({ text: 'a1e\u0301', type: 'text' }), draws)), /^[a-z]\d[a-z]\u0301$/)
})
test('keeps a scheme only before its colon', () => {
assert.match(textOf(anonymize(paragraph({ text: 'file:///x ftp://x http://x https://x mailto:x tel:x', type: 'text' }), draws)), /^file:\/\/\/[a-z] ftp:\/\/[a-z] http:\/\/[a-z] https:\/\/[a-z] mailto:[a-z] tel:[a-z]$/)
assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'secret: value', type: 'text' }), draws)), /secret/)
assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'file https', type: 'text' }), draws)), /file|https/)
})
test('does not call a kept scheme a leak', () => {
const document = paragraph({ marks: [{ attrs: { href: 'https://example.org' }, type: 'link' }], text: 'x', type: 'text' })
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
})
test('keeps a boolean and a null, and replaces the strings beside them', () => {
const [node] = anonymize(paragraph({ attrs: { flag: true, list: [false, 'secret'], none: null }, type: 'someNode' }), draws).content[0].content
assert.equal(node.attrs.flag, true)
assert.equal(node.attrs.list[0], false)
assert.notEqual(node.attrs.list[1], 'secret')
assert.equal(node.attrs.none, null)
})
test('replaces a string among a content array, and keeps a null there', () => {
const [held] = anonymize({ content: [{ content: ['secret', null], type: 'someNode' }], type: 'doc', version: 1 }, draws).content
assert.notEqual(held.content[0], 'secret')
assert.equal(held.content[1], null)
})
test('replaces a value whose key holds a slash, and the value beside it', () => {
const [node] = anonymize(paragraph({ attrs: { a: { b: 4242 }, 'a/b': 4243 }, type: 'someNode' }), draws).content[0].content
assert.notEqual(node.attrs['a/b'], 4243)
assert.notEqual(node.attrs.a.b, 4242)
assert.deepEqual(Object.keys(node.attrs), ['a', 'a/b'])
})
test('gives a number no leading zero, so it keeps its length', () => {
assert.equal(textOf(anonymize(paragraph({ text: '2024 0123', type: 'text' }), { ...draws, digits: ['00000000'] })), '1000 0000')
})
test('moves on through the digits between numbers', () => {
assert.equal(textOf(anonymize(paragraph({ text: '1234 5678', type: 'text' }), draws)), '7391 2846')
})
test('finds no difference between a document and its anonymized form', () => {
@@ -171,6 +266,17 @@ test('finds no difference between a document and its anonymized form', () => {
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
})
test('finds no difference in a kept array or a replaced number', () => {
const document = paragraph({ attrs: { colwidth: [100, 200], pageId: 42871 }, type: 'tableCell' })
assert.deepEqual(shapeDifferences(document, anonymize(document, draws)), [])
})
test('reports a number turned into a string, an added key and a removed key', () => {
const before = paragraph({ attrs: { kept: 12, removed: 1 }, type: 'someNode' })
const after = paragraph({ attrs: { added: 1, kept: '12' }, type: 'someNode' })
assert.deepEqual(shapeDifferences(before, after).sort(), ['/content/0/content/0/attrs/added', '/content/0/content/0/attrs/kept', '/content/0/content/0/attrs/removed'])
})
test('reports a changed structure with its path', () => {
const before = paragraph({ marks: [{ type: 'em' }], text: 'a b', type: 'text' })
const after = paragraph({ marks: [{ type: 'strong' }], text: 'c-d', type: 'text' })
@@ -183,6 +289,21 @@ test('reports a word the output still holds', () => {
assert.deepEqual(leakedTokens(before, after), ['private'])
})
test('counts a token of three letters as a leak, and a shorter one not', () => {
const document = paragraph({ text: 'Bob Al', type: 'text' })
assert.deepEqual(leakedTokens(document, document), ['bob'])
})
test('gives the anonymized document when it holds the shape and no original token', () => {
const document = paragraph({ text: 'private word', type: 'text' })
assert.deepEqual(anonymizeChecked(document, draws), { ok: true, written: anonymize(document, draws) })
})
test('keeps out a document whose output still holds an original token', () => {
const document = paragraph({ text: '123', type: 'text' })
assert.deepEqual(anonymizeChecked(document, { ...draws, digits: ['123'] }), { differences: [], leaks: ['123'], ok: false })
})
test('does not call a kept attribute value a leak', () => {
const document = { attrs: {}, content: [{ attrs: { language: 'python' }, content: [{ text: 'python', type: 'text' }], type: 'codeBlock' }], type: 'doc', version: 1 }
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
+33 -19
View File
@@ -1,22 +1,28 @@
// Usage: node fetch.mjs <sites.json> <raw folder>. The folder stays outside the repo: its documents hold the sites' real text.
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { setTimeout as delay } from 'node:timers/promises'
// Robots.txt on these sites asks for one second between requests.
const crawlDelay = 1000
const pagesPerSource = 20
const [sitesPath, rawFolder] = process.argv.slice(2)
const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
const documentsFolder = join(rawFolder, 'documents')
mkdirSync(documentsFolder, { recursive: true })
import { jiraUrl, wikiUrl } from './source-urls.mjs'
async function getJson(url) {
// Robots.txt on these sites allows these paths and asks for one second between requests.
const crawlDelay = 1000
const defaultRetryAfter = 60
const [sitesPath, rawFolder] = process.argv.slice(2)
// Keeps a fetch to minutes.
const pagesPerListing = 20
const retries = 5
const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
const partialFolder = join(rawFolder, 'documents.partial')
rmSync(partialFolder, { force: true, recursive: true })
mkdirSync(partialFolder, { recursive: true })
async function getJson(url, retriesLeft = retries) {
await delay(crawlDelay)
const response = await fetch(url, { headers: { Accept: 'application/json', 'User-Agent': 'adf-codec-corpus-gathering (https://github.com/larvit/adf-codec)' } })
if (response.status === 429) {
await delay(Number(response.headers.get('retry-after') ?? 60) * 1000)
return getJson(url)
if (response.status === 429 && retriesLeft > 0) {
const asked = Number(response.headers.get('retry-after'))
await delay((asked > 0 ? asked : defaultRetryAfter) * 1000)
return getJson(url, retriesLeft - 1)
}
if (!response.ok) throw new Error(`${url} answered ${response.status}`)
return response.json()
@@ -26,18 +32,18 @@ let written = 0
function keep(name, source, document) {
if (document?.type !== 'doc' || !Array.isArray(document.content) || document.content.length === 0) return
writeFileSync(join(documentsFolder, `${name}.json`), JSON.stringify({ document, source }))
writeFileSync(join(partialFolder, `${name}.json`), JSON.stringify({ document, retrieved: new Date().toISOString(), source }))
written += 1
}
async function fetchConfluence(site, collection) {
const base = `https://${site}.atlassian.net`
let next = `/wiki/api/v2/${collection}?limit=100&body-format=atlas_doc_format`
for (let page = 0; next !== undefined && page < pagesPerSource; page += 1) {
for (let page = 0; next !== undefined && page < pagesPerListing; page += 1) {
const answer = await getJson(`${base}${next}`)
for (const result of answer.results) {
const body = result.body?.atlas_doc_format?.value
if (body) keep(`${site}-wiki-${collection}-${result.id}`, `${base}/wiki/api/v2/${collection}/${result.id}`, JSON.parse(body))
if (body) keep(`${site}-wiki-${collection}-${result.id}`, wikiUrl(base, result._links.webui), JSON.parse(body))
}
next = answer._links?.next
}
@@ -46,18 +52,19 @@ async function fetchConfluence(site, collection) {
async function fetchJira(site) {
const base = `https://${site}.atlassian.net`
let token
for (let page = 0; page < pagesPerSource; page += 1) {
for (let page = 0; page < pagesPerListing; page += 1) {
const query = new URLSearchParams({ fields: 'comment,description', jql: 'created >= "2015-01-01" ORDER BY created DESC', maxResults: '100', ...(token ? { nextPageToken: token } : {}) })
const answer = await getJson(`${base}/rest/api/3/search/jql?${query}`)
for (const issue of answer.issues) {
keep(`${site}-jira-${issue.key}-description`, `${base}/rest/api/3/issue/${issue.key}?fields=description`, issue.fields.description)
for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, `${base}/rest/api/3/issue/${issue.key}/comment/${comment.id}`, comment.body)
keep(`${site}-jira-${issue.key}-description`, jiraUrl(base, issue.key), issue.fields.description)
for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, jiraUrl(base, issue.key, comment.id), comment.body)
}
token = answer.nextPageToken
if (token === undefined) break
}
}
const failures = []
for (const { products, site } of sites) {
for (const product of Object.keys(products)) {
try {
@@ -69,7 +76,14 @@ for (const { products, site } of sites) {
}
} catch (error) {
console.error(`${site} ${product}: ${error.message}`)
failures.push(`${site} ${product}`)
}
console.error(`${site} ${product}: ${written} documents so far`)
}
}
if (failures.length > 0) {
console.error(`No corpus source is complete without: ${failures.join(', ')}. Fetch again, or take the site out of sites.json.`)
process.exit(1)
}
renameSync(partialFolder, join(rawFolder, 'documents'))
+4 -5
View File
@@ -1,12 +1,12 @@
#!/bin/sh
# Usage: corpus-gathering/run.sh <raw folder>. Keep the raw folder outside the repo: its documents hold the sites' real text.
set -eu
here=$(cd "$(dirname "$0")" && pwd)
raw=$(mkdir -p "$1" && cd "$1" && pwd)
[ $# -eq 1 ] || { echo 'Usage: corpus-gathering/run.sh <raw folder>' >&2; exit 2; }
here=$(cd "$(dirname "$0")" && pwd -P)
raw=$(mkdir -p "$1" && cd "$1" && pwd -P)
case "$raw/" in "$(cd "$here/.." && pwd -P)"/*) echo "run.sh: keep the raw folder outside the repo: its documents hold the sites' real text" >&2; exit 2 ;; esac
fejkdata_image=larvit/fejkdata:0.2.0
node_image=node:26.13.0-alpine3.24
# Takes how to mount the repo, then the command.
in_node() {
repo_mount=$1
shift
@@ -25,7 +25,6 @@ mkdir -p "$raw/draws"
draw digits 5000 '{digits(32)}'
draw hex 5000 '{hex(32)}'
draw letters 5000 '{lower(32)}'
draw persons 2000 '{/en_US.person}'
draw uuids 20000 '{/misc.uuid}'
draw words 20000 '{/en_US.word}'
+25
View File
@@ -0,0 +1,25 @@
import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import { test } from 'node:test'
const repo = new URL('..', import.meta.url).pathname
function run(...args) {
return spawnSync('sh', ['corpus-gathering/run.sh', ...args], { cwd: repo, encoding: 'utf8' })
}
test('refuses a raw folder inside the repo', () => {
const { status, stderr } = run('corpus')
assert.equal(status, 2)
assert.match(stderr, /outside the repo/)
})
test('refuses the repo itself as the raw folder', () => {
assert.equal(run('.').status, 2)
})
test('names its argument when it is missing', () => {
const { status, stderr } = run()
assert.equal(status, 2)
assert.match(stderr, /Usage: corpus-gathering\/run\.sh <raw folder>/)
})
+3 -5
View File
@@ -23,7 +23,7 @@ function collect(node, parentType, found) {
})
}
// What a document shows the corpus: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
// A document's features: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
export function features(document) {
const found = new Set()
collect(document, undefined, found)
@@ -47,7 +47,7 @@ function bestCandidate(candidates, covered) {
showsType ||= isType(feature)
}
const score = added / candidate.size
if (added > 0 && (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore)) {
if (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore) {
best = candidate
bestScore = score
bestShowsType = showsType
@@ -60,13 +60,11 @@ function bestCandidate(candidates, covered) {
export function selectDocuments(candidates, limit) {
const covered = new Set()
const chosen = []
let remaining = candidates
while (chosen.length < limit) {
const best = bestCandidate(remaining, covered)
const best = bestCandidate(candidates, covered)
if (best === undefined) break
chosen.push(best)
for (const feature of best.features) covered.add(feature)
remaining = remaining.filter((candidate) => candidate !== best)
}
return chosen
}
+26
View File
@@ -1,5 +1,6 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { features, selectDocuments } from './select.mjs'
function paragraph(...content) {
@@ -11,10 +12,23 @@ test('names node and mark types, attribute keys and nesting', () => {
for (const feature of ['node:doc', 'node:paragraph', 'node:text', 'mark:link', 'attr:link.href', 'child:paragraph>text']) assert.ok(found.has(feature), feature)
})
test('names a node attribute', () => {
assert.ok(features(paragraph({ attrs: { level: 2 }, type: 'heading' })).has('attr:heading.level'))
})
test('names no mark for a node without marks, and no parent for the document', () => {
assert.deepEqual([...features({ type: 'doc' })], ['node:doc'])
})
test('names marks held together', () => {
assert.ok(features(paragraph({ marks: [{ type: 'strong' }, { type: 'em' }], text: 'a', type: 'text' })).has('marks:em+strong'))
})
test('names no marks held together for one mark, or for marks on a node other than text', () => {
const found = [...features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'strong' }, { type: 'em' }], type: 'mediaInline' }))]
assert.deepEqual(found.filter((feature) => feature.startsWith('marks:')), [])
})
test('names bold beside a code span and italics beside a hard break', () => {
const found = features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'code' }], text: 'b', type: 'text' }, { type: 'hardBreak' }, { marks: [{ type: 'em' }], text: 'c', type: 'text' }))
assert.ok(found.has('beside:text[strong]|text[code]'))
@@ -34,6 +48,18 @@ test('takes a candidate showing a new node or mark type ahead of any other', ()
assert.deepEqual(selectDocuments([cheap, rare], 1), [rare])
})
test('takes a candidate showing a new type wherever it stands among the candidates', () => {
const rare = { features: new Set(['mark:breakout']), size: 1000 }
const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
assert.deepEqual(selectDocuments([rare, cheap], 1), [rare])
})
test('counts a feature naming a type only by its opening', () => {
const rare = { features: new Set(['beside:node:x']), size: 1000 }
const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
assert.deepEqual(selectDocuments([rare, cheap], 1), [cheap])
})
test('stops at the limit', () => {
const candidates = [{ features: new Set(['a']), size: 1 }, { features: new Set(['b']), size: 1 }]
assert.equal(selectDocuments(candidates, 1).length, 1)
+28 -28
View File
@@ -1,30 +1,4 @@
[
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-SA-4.0",
"licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
}
},
"site": "lf-onap"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-opendaylight"
},
{
"products": {
"jira": {
@@ -58,7 +32,7 @@
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-hyperledger"
"site": "lf-edgexfoundry"
},
{
"products": {
@@ -67,6 +41,32 @@
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-edgexfoundry"
"site": "lf-hyperledger"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-SA-4.0",
"licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
}
},
"site": "lf-onap"
},
{
"products": {
"jira": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
},
"wiki": {
"licence": "CC-BY-3.0",
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
}
},
"site": "lf-opendaylight"
}
]
+10
View File
@@ -0,0 +1,10 @@
// The pages a person reads, which name each document's creator. A wiki's web link also holds the page's title, and can hold an attachment's name, which the repo must not.
export function jiraUrl(base, key, commentId) {
return `${base}/browse/${key}${commentId === undefined ? '' : `?focusedCommentId=${commentId}`}`
}
export function wikiUrl(base, webLink) {
const [path, query] = webLink.split('?')
const commentId = new URLSearchParams(query).get('focusedCommentId')
return `${base}/wiki${path.replace(/(\/\d+)\/[^/]+$/, '$1')}${commentId === null ? '' : `?focusedCommentId=${commentId}`}`
}
+32
View File
@@ -0,0 +1,32 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { jiraUrl, wikiUrl } from './source-urls.mjs'
const base = 'https://site.atlassian.net'
test('links a Jira issue, and a comment on it', () => {
assert.equal(jiraUrl(base, 'KEY-1'), 'https://site.atlassian.net/browse/KEY-1')
assert.equal(jiraUrl(base, 'KEY-1', '42'), 'https://site.atlassian.net/browse/KEY-1?focusedCommentId=42')
})
test('links a wiki page without the title its web link holds', () => {
assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13')
})
test('links a wiki comment without the title of its page', () => {
assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13?focusedCommentId=42')
})
test('links a comment on a blog post without its title', () => {
assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/blog/2020/01/02/13?focusedCommentId=42')
})
test('wikiUrl drops a preview, which names an attached file', () => {
assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?preview=%2F13%2F14%2FSecret.jpeg'), `${base}/wiki/spaces/AK/blog/2020/01/02/13`)
assert.equal(wikiUrl(base, '/spaces/AK/overview?preview=%2F13%2F14%2FSecret.jpeg&focusedCommentId=42'), `${base}/wiki/spaces/AK/overview?focusedCommentId=42`)
})
test('wikiUrl keeps a space overview, which holds no title', () => {
assert.equal(wikiUrl(base, '/spaces/AK/overview'), `${base}/wiki/spaces/AK/overview`)
})
+11 -13
View File
@@ -1,9 +1,10 @@
// Usage: node write-corpus.mjs <sites.json> <raw folder> <documents folder> <sources file>. The raw folder holds fetch.mjs's documents and run.sh's draws.
import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { anonymizeChecked } from './anonymize.mjs'
import { features, selectDocuments } from './select.mjs'
// Selection prefers small documents, which keeps the corpus near 660 KB.
const documentLimit = 150
const largestDocument = 64 * 1024
const [sitesPath, rawFolder, corpusFolder, sourcesPath] = process.argv.slice(2)
@@ -23,22 +24,19 @@ function canonical(value) {
}
const licences = new Map(JSON.parse(readFileSync(sitesPath, 'utf8')).flatMap(({ products, site }) => Object.entries(products).map(([product, licence]) => [`${site}-${product}`, licence])))
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), persons: lines('persons'), uuids: lines('uuids'), words: lines('words') }
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), uuids: lines('uuids'), words: lines('words') }
const documentsFolder = join(rawFolder, 'documents')
const candidates = readdirSync(documentsFolder).flatMap((file) => {
const path = join(documentsFolder, file)
const text = readFileSync(path, 'utf8')
const text = readFileSync(join(documentsFolder, file), 'utf8')
if (text.length > largestDocument) return []
const { document, source } = JSON.parse(text)
const { document, retrieved, source } = JSON.parse(text)
const name = file.slice(0, -'.json'.length)
const written = anonymize(document, draws)
const differences = shapeDifferences(document, written)
const leaks = leakedTokens(document, written)
if (differences.length > 0 || leaks.length > 0) {
console.error(`${name} left out: ${differences.length} shape differences, ${leaks.length} tokens still held`)
const checked = anonymizeChecked(document, draws)
if (!checked.ok) {
console.error(`${name} left out: ${checked.differences.length} shape differences, ${checked.leaks.length} tokens still held`)
return []
}
return [{ features: features(document), name, retrieved: statSync(path).mtime.toISOString().slice(0, 10), size: text.length, source, written }]
return [{ features: features(document), name, retrieved, size: text.length, source, written: checked.written }]
})
rmSync(corpusFolder, { force: true, recursive: true })