mirror of
https://github.com/larvit/adf-codec.git
synced 2026-11-20 15:15:44 +01:00
Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
// A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
|
||||
const formatKeys = new Set([
|
||||
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
|
||||
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode',
|
||||
'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
|
||||
'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
|
||||
])
|
||||
@@ -10,7 +10,7 @@ const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12
|
||||
const uuid = new RegExp(`^${uuidPattern}$`, 'i')
|
||||
const uuidOrToken = new RegExp(`(?<![\\p{L}\\p{N}])${uuidPattern}(?![\\p{L}\\p{M}\\p{N}])|${token.source}`, 'giu')
|
||||
const uuidVersionAt = 14
|
||||
// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
|
||||
// A chance match below this length identifies nobody, and a document may hold every such token, so redrawing around them might never end.
|
||||
const shortestLeak = 3
|
||||
const redraws = 64
|
||||
|
||||
@@ -22,47 +22,45 @@ function tokensOf(text) {
|
||||
return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
|
||||
}
|
||||
|
||||
// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
|
||||
function walk(value, path, visit) {
|
||||
// Calls `visit` with each leaf, its steps from the document and its kind: `kept`, or `free`, which the anonymizer replaces.
|
||||
function walk(value, steps, visit) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
|
||||
value.forEach((item, index) => walk(item, [...steps, index], visit))
|
||||
} else if (typeof value === 'object' && value !== null) {
|
||||
for (const [key, member] of Object.entries(value)) {
|
||||
const memberPath = `${path}/${key}`
|
||||
if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
|
||||
else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
|
||||
else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
|
||||
else walkFree(member, memberPath, visit)
|
||||
const memberSteps = [...steps, key]
|
||||
if (key === 'attrs') walkAttributes(member, memberSteps, visit)
|
||||
else if (key === 'content' || key === 'marks') walk(member, memberSteps, visit)
|
||||
else if (key === 'type' || key === 'version') visit(member, memberSteps, 'kept')
|
||||
else walkFree(member, memberSteps, visit)
|
||||
}
|
||||
} else {
|
||||
visit(value, path, 'free')
|
||||
visit(value, steps, 'free')
|
||||
}
|
||||
}
|
||||
|
||||
// `<app id>/<environment id>/...`, so one more id, where any other extension key names a macro.
|
||||
// A Forge key reads `<app id>/<environment id>/...`, so it holds ids to replace. Any other extension key names a macro and stays.
|
||||
function isForgeKey(key, value) {
|
||||
return key === 'extensionKey' && String(value).includes('/')
|
||||
}
|
||||
|
||||
function walkAttributes(attributes, path, visit, nodeType) {
|
||||
function walkAttributes(attributes, steps, visit) {
|
||||
for (const [key, member] of Object.entries(attributes)) {
|
||||
const memberPath = `${path}/${key}`
|
||||
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept')
|
||||
else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
|
||||
else walkFree(member, memberPath, visit)
|
||||
const memberSteps = [...steps, key]
|
||||
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberSteps, 'kept')
|
||||
else walkFree(member, memberSteps, visit)
|
||||
}
|
||||
}
|
||||
|
||||
function walkFree(value, path, visit) {
|
||||
if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
|
||||
else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
|
||||
else visit(value, path, 'free')
|
||||
function walkFree(value, steps, visit) {
|
||||
if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, [...steps, key], visit)
|
||||
else visit(value, steps, 'free')
|
||||
}
|
||||
|
||||
function replacedStrings(document) {
|
||||
const strings = []
|
||||
walk(document, '', (value, _path, kind) => {
|
||||
if (kind !== 'kept' && typeof value === 'string') strings.push(value)
|
||||
walk(document, [], (value, _steps, kind) => {
|
||||
if (kind === 'free' && typeof value === 'string') strings.push(value)
|
||||
})
|
||||
return strings
|
||||
}
|
||||
@@ -101,7 +99,7 @@ function wordDrawer(words) {
|
||||
}
|
||||
|
||||
function withCaseOf(original, word) {
|
||||
if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
|
||||
if (original === original.toUpperCase() && original !== original.toLowerCase()) return word.toUpperCase()
|
||||
if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
|
||||
return word
|
||||
}
|
||||
@@ -111,7 +109,6 @@ function replacer(document, draws) {
|
||||
const digits = stream(draws.digits)
|
||||
const hex = stream(draws.hex)
|
||||
const letters = stream(draws.letters)
|
||||
const nextPerson = cursor(draws.persons)
|
||||
const nextUuid = cursor(draws.uuids)
|
||||
const nextWord = wordDrawer(draws.words)
|
||||
const spelled = new Map()
|
||||
@@ -139,7 +136,7 @@ function replacer(document, draws) {
|
||||
function word(original) {
|
||||
if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
|
||||
let tries = 0
|
||||
// A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
|
||||
// A long document can hold every word of a length, so the last two of the draws `fresh` makes gain a suffix.
|
||||
return withCaseOf(original, fresh(() => {
|
||||
tries += 1
|
||||
return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
|
||||
@@ -172,42 +169,34 @@ function replacer(document, draws) {
|
||||
})
|
||||
}
|
||||
|
||||
function person(original) {
|
||||
return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
|
||||
}
|
||||
|
||||
function scalar(value) {
|
||||
if (typeof value === 'string') return string(value)
|
||||
if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
|
||||
return value
|
||||
}
|
||||
|
||||
return { person, scalar }
|
||||
return scalar
|
||||
}
|
||||
|
||||
function setAt(target, path, value) {
|
||||
const steps = path.split('/').slice(1)
|
||||
const last = steps.pop()
|
||||
let holder = target
|
||||
for (const step of steps) holder = holder[step]
|
||||
holder[last] = value
|
||||
function setAt(target, steps, value) {
|
||||
const holder = steps.slice(0, -1).reduce((held, step) => held[step], target)
|
||||
holder[steps.at(-1)] = value
|
||||
}
|
||||
|
||||
// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
|
||||
// `draws` is fejkdata's output, as run.sh renders it.
|
||||
export function anonymize(document, draws) {
|
||||
const { person, scalar } = replacer(document, draws)
|
||||
const scalar = replacer(document, draws)
|
||||
const written = structuredClone(document)
|
||||
walk(document, '', (value, path, kind) => {
|
||||
if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
|
||||
else if (kind === 'free') setAt(written, path, scalar(value))
|
||||
walk(document, [], (value, steps, kind) => {
|
||||
if (kind === 'free') setAt(written, steps, scalar(value))
|
||||
})
|
||||
return written
|
||||
}
|
||||
|
||||
function structure(document) {
|
||||
const found = new Map()
|
||||
walk(document, '', (value, path, kind) => {
|
||||
found.set(path, { kind, value })
|
||||
walk(document, [], (value, steps, kind) => {
|
||||
found.set(`/${steps.join('/')}`, { kind, value })
|
||||
})
|
||||
return found
|
||||
}
|
||||
@@ -217,12 +206,12 @@ function skeleton(text) {
|
||||
}
|
||||
|
||||
function holdsSameShape(before, after) {
|
||||
if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
|
||||
if (before.kind === 'kept') return JSON.stringify(before.value) === JSON.stringify(after.value)
|
||||
if (typeof before.value !== typeof after.value) return false
|
||||
return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
|
||||
return typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
|
||||
}
|
||||
|
||||
// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
|
||||
// The paths where `after` differs from `before` in anything but the tokens the anonymizer replaces.
|
||||
export function shapeDifferences(before, after) {
|
||||
const beforeValues = structure(before)
|
||||
const afterValues = structure(after)
|
||||
@@ -231,11 +220,19 @@ export function shapeDifferences(before, after) {
|
||||
const first = beforeValues.get(path)
|
||||
const second = afterValues.get(path)
|
||||
return first === undefined || second === undefined || !holdsSameShape(first, second)
|
||||
}).sort()
|
||||
})
|
||||
}
|
||||
|
||||
// The tokens of `before`'s replaced values that `after`'s replaced values still hold.
|
||||
export function leakedTokens(before, after) {
|
||||
const held = heldTokens(after)
|
||||
return [...heldTokens(before)].filter((found) => held.has(found)).sort()
|
||||
return [...heldTokens(before)].filter((found) => held.has(found))
|
||||
}
|
||||
|
||||
// The anonymized document, or what keeps it out of the corpus: a slip here would publish real text, and nothing later could see it.
|
||||
export function anonymizeChecked(document, draws) {
|
||||
const written = anonymize(document, draws)
|
||||
const differences = shapeDifferences(document, written)
|
||||
const leaks = leakedTokens(document, written)
|
||||
return differences.length > 0 || leaks.length > 0 ? { differences, leaks, ok: false } : { ok: true, written }
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
|
||||
|
||||
import { anonymize, anonymizeChecked, leakedTokens, shapeDifferences } from './anonymize.mjs'
|
||||
|
||||
const draws = {
|
||||
digits: ['7391', '2846'],
|
||||
hex: ['9f3ac07be1d4'],
|
||||
letters: ['qkxwzj'],
|
||||
persons: ['Pat Smith', 'Lo Berg'],
|
||||
uuids: ['0190a1b2-c3d4-4e5f-8a9b-0c1d2e3f4a5b', '0190a1b2-c3d4-4e5f-8a9b-1c1d2e3f4a5c'],
|
||||
words: ['amber', 'clever', 'cobalt', 'merry', 'otter', 'cliff', 'tundra', 'pumice', 'lantern', 'harbour'],
|
||||
}
|
||||
@@ -22,7 +22,7 @@ function textOf(document) {
|
||||
test('replaces each word and keeps the whitespace and punctuation around it', () => {
|
||||
const written = textOf(anonymize(paragraph({ text: ' Deploy *the* secret-name, now!\n', type: 'text' }), draws))
|
||||
assert.match(written, /^ {2}[A-Z][a-z]+ \*[a-z]+\* [a-z]+-[a-z]+, [a-z]+!\n$/)
|
||||
for (const word of ['Deploy', 'the', 'secret', 'name', 'now']) assert.ok(!written.includes(word))
|
||||
assert.doesNotMatch(written, /Deploy|the|secret|name|now/)
|
||||
})
|
||||
|
||||
test('keeps a word in capitals in capitals', () => {
|
||||
@@ -81,13 +81,13 @@ test('replaces a person, an account id and a uuid, keeping each shape', () => {
|
||||
assert.ok(!mention.attrs.text.includes('Person'))
|
||||
})
|
||||
|
||||
test('keeps the version of every uuid it replaces', () => {
|
||||
for (const [version, original] of [['1', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], ['3', '6fa459ea-ee8a-3ca4-894e-db77e160355e'], ['4', '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e']]) {
|
||||
for (const [version, original] of [['1', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], ['3', '6fa459ea-ee8a-3ca4-894e-db77e160355e'], ['4', '3f2b8c1e-9d4a-4c7b-8e21-5a6f7b8c9d0e']]) {
|
||||
test(`keeps the version of a v${version} uuid it replaces`, () => {
|
||||
const [mention] = anonymize(paragraph({ attrs: { localId: original }, type: 'mention' }), draws).content[0].content
|
||||
assert.match(mention.attrs.localId, new RegExp(`^[0-9a-f]{8}-[0-9a-f]{4}-${version}[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$`))
|
||||
assert.notEqual(mention.attrs.localId, original)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
test('keeps a uuid in capitals in capitals', () => {
|
||||
const [mention] = anonymize(paragraph({ attrs: { localId: '6FA459EA-EE8A-3CA4-894E-DB77E160355E' }, type: 'mention' }), draws).content[0].content
|
||||
@@ -95,7 +95,7 @@ test('keeps a uuid in capitals in capitals', () => {
|
||||
assert.ok(/[A-F]/.test(mention.attrs.localId))
|
||||
})
|
||||
|
||||
test('keeps a uuid inside a longer value a uuid of its version', () => {
|
||||
test('keeps the version of a uuid inside a longer value', () => {
|
||||
const inside = '[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}'
|
||||
const accountId = '712020:abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
|
||||
const href = 'https://wiki.example.org/page?id=abcdabcd-ebfa-3cde-8abc-abcdefabcdef'
|
||||
@@ -111,7 +111,7 @@ test('reports a uuid whose version changed', () => {
|
||||
assert.deepEqual(shapeDifferences(before, after), ['/content/0/content/0/attrs/id'])
|
||||
})
|
||||
|
||||
test('keeps a date timestamp a number of the same length', () => {
|
||||
test('replaces a date timestamp with a number of the same length', () => {
|
||||
const [date] = anonymize(paragraph({ attrs: { timestamp: '1700000000000' }, type: 'date' }), draws).content[0].content
|
||||
assert.match(date.attrs.timestamp, /^[1-9]\d{12}$/)
|
||||
assert.notEqual(date.attrs.timestamp, '1700000000000')
|
||||
@@ -160,9 +160,104 @@ test('never draws a uuid holding a part of an original one', () => {
|
||||
assert.equal(mention.attrs.localId, draws.uuids[0])
|
||||
})
|
||||
|
||||
test('never runs out of draws, however many words a document holds', () => {
|
||||
const text = Array.from({ length: 500 }, (_, index) => `word${'x'.repeat(index % 9)}`).join(' ')
|
||||
assert.equal(textOf(anonymize(paragraph({ text, type: 'text' }), draws)).split(' ').length, 500)
|
||||
test('never runs out of draws when a document holds every word of the pool', () => {
|
||||
const document = paragraph({ text: draws.words.join(' '), type: 'text' })
|
||||
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
|
||||
})
|
||||
|
||||
test('ends and reports the leak when every draw holds the original', () => {
|
||||
const document = paragraph({ text: '123', type: 'text' })
|
||||
assert.deepEqual(leakedTokens(document, anonymize(document, { ...draws, digits: ['123'] })), ['123'])
|
||||
})
|
||||
|
||||
test('replaces a one-word mention with one word', () => {
|
||||
const [mention] = anonymize(paragraph({ attrs: { id: 'x', text: '@Cher' }, type: 'mention' }), draws).content[0].content
|
||||
assert.match(mention.attrs.text, /^@[A-Z][a-z]+$/)
|
||||
})
|
||||
|
||||
test('spells a name the same way in a mention and in text', () => {
|
||||
const [mention, text] = anonymize(paragraph({ attrs: { text: '@Bob' }, type: 'mention' }, { text: 'Bob said', type: 'text' }), draws).content[0].content
|
||||
assert.equal(mention.attrs.text.slice(1), text.text.split(' ')[0])
|
||||
})
|
||||
|
||||
test('keeps every attribute naming a format, and the version', () => {
|
||||
const attrs = Object.fromEntries(['__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode', 'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth', 'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType'].map((key) => [key, 'zebra']))
|
||||
const [written] = anonymize({ content: [{ attrs: { ...attrs, language: 'c/c++' }, type: 'someNode' }], type: 'doc', version: 1 }, draws).content
|
||||
assert.deepEqual(written.attrs, { ...attrs, language: 'c/c++' })
|
||||
assert.equal(anonymize(paragraph(), draws).version, 1)
|
||||
})
|
||||
|
||||
test('spells a word with the nearest length the pool holds, in its case', () => {
|
||||
const pool = { ...draws, words: ['amber', 'clever', 'lantern'] }
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'Alpha', type: 'text' }), pool)), 'Amber')
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'channel', type: 'text' }), pool)), 'lantern')
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'NINELONGS', type: 'text' }), pool)), 'LANTERN')
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'tiny', type: 'text' }), pool)), 'amber')
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'Bob', type: 'text' }), pool)), 'Amber')
|
||||
})
|
||||
|
||||
test('spells a short word from the letters, keeping its case', () => {
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'Al', type: 'text' }), draws)), 'Qk')
|
||||
})
|
||||
|
||||
test('reads a word with an accent as a word', () => {
|
||||
assert.equal(textOf(anonymize(paragraph({ text: 'caf\u00e9', type: 'text' }), { ...draws, words: ['amber'] })), 'amber')
|
||||
})
|
||||
|
||||
test('spells a word of a caseless script in small letters', () => {
|
||||
assert.match(textOf(anonymize(paragraph({ text: '\u6f22\u5b57 \u6f22\u5b57\u6f22', type: 'text' }), draws)), /^[a-z]+ [a-z]+$/)
|
||||
})
|
||||
|
||||
test('keeps letters and digits in their places in a mixed token', () => {
|
||||
assert.match(textOf(anonymize(paragraph({ text: 'beta2 2beta a1x xa1', type: 'text' }), draws)), /^[a-z]{4}\d \d[a-z]{4} [a-z]\d[a-z] [a-z]{2}\d$/)
|
||||
})
|
||||
|
||||
test('spells a hex token from the hex draws', () => {
|
||||
assert.equal(textOf(anonymize(paragraph({ text: '3f2a', type: 'text' }), draws)), '9f3a')
|
||||
})
|
||||
|
||||
test('keeps a combining mark in a mixed token', () => {
|
||||
assert.match(textOf(anonymize(paragraph({ text: 'a1e\u0301', type: 'text' }), draws)), /^[a-z]\d[a-z]\u0301$/)
|
||||
})
|
||||
|
||||
test('keeps a scheme only before its colon', () => {
|
||||
assert.match(textOf(anonymize(paragraph({ text: 'file:///x ftp://x http://x https://x mailto:x tel:x', type: 'text' }), draws)), /^file:\/\/\/[a-z] ftp:\/\/[a-z] http:\/\/[a-z] https:\/\/[a-z] mailto:[a-z] tel:[a-z]$/)
|
||||
assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'secret: value', type: 'text' }), draws)), /secret/)
|
||||
assert.doesNotMatch(textOf(anonymize(paragraph({ text: 'file https', type: 'text' }), draws)), /file|https/)
|
||||
})
|
||||
|
||||
test('does not call a kept scheme a leak', () => {
|
||||
const document = paragraph({ marks: [{ attrs: { href: 'https://example.org' }, type: 'link' }], text: 'x', type: 'text' })
|
||||
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
|
||||
})
|
||||
|
||||
test('keeps a boolean and a null, and replaces the strings beside them', () => {
|
||||
const [node] = anonymize(paragraph({ attrs: { flag: true, list: [false, 'secret'], none: null }, type: 'someNode' }), draws).content[0].content
|
||||
assert.equal(node.attrs.flag, true)
|
||||
assert.equal(node.attrs.list[0], false)
|
||||
assert.notEqual(node.attrs.list[1], 'secret')
|
||||
assert.equal(node.attrs.none, null)
|
||||
})
|
||||
|
||||
test('replaces a string among a content array, and keeps a null there', () => {
|
||||
const [held] = anonymize({ content: [{ content: ['secret', null], type: 'someNode' }], type: 'doc', version: 1 }, draws).content
|
||||
assert.notEqual(held.content[0], 'secret')
|
||||
assert.equal(held.content[1], null)
|
||||
})
|
||||
|
||||
test('replaces a value whose key holds a slash, and the value beside it', () => {
|
||||
const [node] = anonymize(paragraph({ attrs: { a: { b: 4242 }, 'a/b': 4243 }, type: 'someNode' }), draws).content[0].content
|
||||
assert.notEqual(node.attrs['a/b'], 4243)
|
||||
assert.notEqual(node.attrs.a.b, 4242)
|
||||
assert.deepEqual(Object.keys(node.attrs), ['a', 'a/b'])
|
||||
})
|
||||
|
||||
test('gives a number no leading zero, so it keeps its length', () => {
|
||||
assert.equal(textOf(anonymize(paragraph({ text: '2024 0123', type: 'text' }), { ...draws, digits: ['00000000'] })), '1000 0000')
|
||||
})
|
||||
|
||||
test('moves on through the digits between numbers', () => {
|
||||
assert.equal(textOf(anonymize(paragraph({ text: '1234 5678', type: 'text' }), draws)), '7391 2846')
|
||||
})
|
||||
|
||||
test('finds no difference between a document and its anonymized form', () => {
|
||||
@@ -171,6 +266,17 @@ test('finds no difference between a document and its anonymized form', () => {
|
||||
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
|
||||
})
|
||||
|
||||
test('finds no difference in a kept array or a replaced number', () => {
|
||||
const document = paragraph({ attrs: { colwidth: [100, 200], pageId: 42871 }, type: 'tableCell' })
|
||||
assert.deepEqual(shapeDifferences(document, anonymize(document, draws)), [])
|
||||
})
|
||||
|
||||
test('reports a number turned into a string, an added key and a removed key', () => {
|
||||
const before = paragraph({ attrs: { kept: 12, removed: 1 }, type: 'someNode' })
|
||||
const after = paragraph({ attrs: { added: 1, kept: '12' }, type: 'someNode' })
|
||||
assert.deepEqual(shapeDifferences(before, after).sort(), ['/content/0/content/0/attrs/added', '/content/0/content/0/attrs/kept', '/content/0/content/0/attrs/removed'])
|
||||
})
|
||||
|
||||
test('reports a changed structure with its path', () => {
|
||||
const before = paragraph({ marks: [{ type: 'em' }], text: 'a b', type: 'text' })
|
||||
const after = paragraph({ marks: [{ type: 'strong' }], text: 'c-d', type: 'text' })
|
||||
@@ -183,6 +289,21 @@ test('reports a word the output still holds', () => {
|
||||
assert.deepEqual(leakedTokens(before, after), ['private'])
|
||||
})
|
||||
|
||||
test('counts a token of three letters as a leak, and a shorter one not', () => {
|
||||
const document = paragraph({ text: 'Bob Al', type: 'text' })
|
||||
assert.deepEqual(leakedTokens(document, document), ['bob'])
|
||||
})
|
||||
|
||||
test('gives the anonymized document when it holds the shape and no original token', () => {
|
||||
const document = paragraph({ text: 'private word', type: 'text' })
|
||||
assert.deepEqual(anonymizeChecked(document, draws), { ok: true, written: anonymize(document, draws) })
|
||||
})
|
||||
|
||||
test('keeps out a document whose output still holds an original token', () => {
|
||||
const document = paragraph({ text: '123', type: 'text' })
|
||||
assert.deepEqual(anonymizeChecked(document, { ...draws, digits: ['123'] }), { differences: [], leaks: ['123'], ok: false })
|
||||
})
|
||||
|
||||
test('does not call a kept attribute value a leak', () => {
|
||||
const document = { attrs: {}, content: [{ attrs: { language: 'python' }, content: [{ text: 'python', type: 'text' }], type: 'codeBlock' }], type: 'doc', version: 1 }
|
||||
assert.deepEqual(leakedTokens(document, anonymize(document, draws)), [])
|
||||
|
||||
+33
-19
@@ -1,22 +1,28 @@
|
||||
// Usage: node fetch.mjs <sites.json> <raw folder>. The folder stays outside the repo: its documents hold the sites' real text.
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
|
||||
// Robots.txt on these sites asks for one second between requests.
|
||||
const crawlDelay = 1000
|
||||
const pagesPerSource = 20
|
||||
const [sitesPath, rawFolder] = process.argv.slice(2)
|
||||
const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
|
||||
const documentsFolder = join(rawFolder, 'documents')
|
||||
mkdirSync(documentsFolder, { recursive: true })
|
||||
import { jiraUrl, wikiUrl } from './source-urls.mjs'
|
||||
|
||||
async function getJson(url) {
|
||||
// Robots.txt on these sites allows these paths and asks for one second between requests.
|
||||
const crawlDelay = 1000
|
||||
const defaultRetryAfter = 60
|
||||
const [sitesPath, rawFolder] = process.argv.slice(2)
|
||||
// Keeps a fetch to minutes.
|
||||
const pagesPerListing = 20
|
||||
const retries = 5
|
||||
const sites = JSON.parse(readFileSync(sitesPath, 'utf8'))
|
||||
const partialFolder = join(rawFolder, 'documents.partial')
|
||||
rmSync(partialFolder, { force: true, recursive: true })
|
||||
mkdirSync(partialFolder, { recursive: true })
|
||||
|
||||
async function getJson(url, retriesLeft = retries) {
|
||||
await delay(crawlDelay)
|
||||
const response = await fetch(url, { headers: { Accept: 'application/json', 'User-Agent': 'adf-codec-corpus-gathering (https://github.com/larvit/adf-codec)' } })
|
||||
if (response.status === 429) {
|
||||
await delay(Number(response.headers.get('retry-after') ?? 60) * 1000)
|
||||
return getJson(url)
|
||||
if (response.status === 429 && retriesLeft > 0) {
|
||||
const asked = Number(response.headers.get('retry-after'))
|
||||
await delay((asked > 0 ? asked : defaultRetryAfter) * 1000)
|
||||
return getJson(url, retriesLeft - 1)
|
||||
}
|
||||
if (!response.ok) throw new Error(`${url} answered ${response.status}`)
|
||||
return response.json()
|
||||
@@ -26,18 +32,18 @@ let written = 0
|
||||
|
||||
function keep(name, source, document) {
|
||||
if (document?.type !== 'doc' || !Array.isArray(document.content) || document.content.length === 0) return
|
||||
writeFileSync(join(documentsFolder, `${name}.json`), JSON.stringify({ document, source }))
|
||||
writeFileSync(join(partialFolder, `${name}.json`), JSON.stringify({ document, retrieved: new Date().toISOString(), source }))
|
||||
written += 1
|
||||
}
|
||||
|
||||
async function fetchConfluence(site, collection) {
|
||||
const base = `https://${site}.atlassian.net`
|
||||
let next = `/wiki/api/v2/${collection}?limit=100&body-format=atlas_doc_format`
|
||||
for (let page = 0; next !== undefined && page < pagesPerSource; page += 1) {
|
||||
for (let page = 0; next !== undefined && page < pagesPerListing; page += 1) {
|
||||
const answer = await getJson(`${base}${next}`)
|
||||
for (const result of answer.results) {
|
||||
const body = result.body?.atlas_doc_format?.value
|
||||
if (body) keep(`${site}-wiki-${collection}-${result.id}`, `${base}/wiki/api/v2/${collection}/${result.id}`, JSON.parse(body))
|
||||
if (body) keep(`${site}-wiki-${collection}-${result.id}`, wikiUrl(base, result._links.webui), JSON.parse(body))
|
||||
}
|
||||
next = answer._links?.next
|
||||
}
|
||||
@@ -46,18 +52,19 @@ async function fetchConfluence(site, collection) {
|
||||
async function fetchJira(site) {
|
||||
const base = `https://${site}.atlassian.net`
|
||||
let token
|
||||
for (let page = 0; page < pagesPerSource; page += 1) {
|
||||
for (let page = 0; page < pagesPerListing; page += 1) {
|
||||
const query = new URLSearchParams({ fields: 'comment,description', jql: 'created >= "2015-01-01" ORDER BY created DESC', maxResults: '100', ...(token ? { nextPageToken: token } : {}) })
|
||||
const answer = await getJson(`${base}/rest/api/3/search/jql?${query}`)
|
||||
for (const issue of answer.issues) {
|
||||
keep(`${site}-jira-${issue.key}-description`, `${base}/rest/api/3/issue/${issue.key}?fields=description`, issue.fields.description)
|
||||
for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, `${base}/rest/api/3/issue/${issue.key}/comment/${comment.id}`, comment.body)
|
||||
keep(`${site}-jira-${issue.key}-description`, jiraUrl(base, issue.key), issue.fields.description)
|
||||
for (const comment of issue.fields.comment?.comments ?? []) keep(`${site}-jira-${issue.key}-comment-${comment.id}`, jiraUrl(base, issue.key, comment.id), comment.body)
|
||||
}
|
||||
token = answer.nextPageToken
|
||||
if (token === undefined) break
|
||||
}
|
||||
}
|
||||
|
||||
const failures = []
|
||||
for (const { products, site } of sites) {
|
||||
for (const product of Object.keys(products)) {
|
||||
try {
|
||||
@@ -69,7 +76,14 @@ for (const { products, site } of sites) {
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`${site} ${product}: ${error.message}`)
|
||||
failures.push(`${site} ${product}`)
|
||||
}
|
||||
console.error(`${site} ${product}: ${written} documents so far`)
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`No corpus source is complete without: ${failures.join(', ')}. Fetch again, or take the site out of sites.json.`)
|
||||
process.exit(1)
|
||||
}
|
||||
renameSync(partialFolder, join(rawFolder, 'documents'))
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Usage: corpus-gathering/run.sh <raw folder>. Keep the raw folder outside the repo: its documents hold the sites' real text.
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
raw=$(mkdir -p "$1" && cd "$1" && pwd)
|
||||
[ $# -eq 1 ] || { echo 'Usage: corpus-gathering/run.sh <raw folder>' >&2; exit 2; }
|
||||
here=$(cd "$(dirname "$0")" && pwd -P)
|
||||
raw=$(mkdir -p "$1" && cd "$1" && pwd -P)
|
||||
case "$raw/" in "$(cd "$here/.." && pwd -P)"/*) echo "run.sh: keep the raw folder outside the repo: its documents hold the sites' real text" >&2; exit 2 ;; esac
|
||||
fejkdata_image=larvit/fejkdata:0.2.0
|
||||
node_image=node:26.13.0-alpine3.24
|
||||
|
||||
# Takes how to mount the repo, then the command.
|
||||
in_node() {
|
||||
repo_mount=$1
|
||||
shift
|
||||
@@ -25,7 +25,6 @@ mkdir -p "$raw/draws"
|
||||
draw digits 5000 '{digits(32)}'
|
||||
draw hex 5000 '{hex(32)}'
|
||||
draw letters 5000 '{lower(32)}'
|
||||
draw persons 2000 '{/en_US.person}'
|
||||
draw uuids 20000 '{/misc.uuid}'
|
||||
draw words 20000 '{/en_US.word}'
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { test } from 'node:test'
|
||||
|
||||
const repo = new URL('..', import.meta.url).pathname
|
||||
|
||||
function run(...args) {
|
||||
return spawnSync('sh', ['corpus-gathering/run.sh', ...args], { cwd: repo, encoding: 'utf8' })
|
||||
}
|
||||
|
||||
test('refuses a raw folder inside the repo', () => {
|
||||
const { status, stderr } = run('corpus')
|
||||
assert.equal(status, 2)
|
||||
assert.match(stderr, /outside the repo/)
|
||||
})
|
||||
|
||||
test('refuses the repo itself as the raw folder', () => {
|
||||
assert.equal(run('.').status, 2)
|
||||
})
|
||||
|
||||
test('names its argument when it is missing', () => {
|
||||
const { status, stderr } = run()
|
||||
assert.equal(status, 2)
|
||||
assert.match(stderr, /Usage: corpus-gathering\/run\.sh <raw folder>/)
|
||||
})
|
||||
@@ -23,7 +23,7 @@ function collect(node, parentType, found) {
|
||||
})
|
||||
}
|
||||
|
||||
// What a document shows the corpus: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
|
||||
// A document's features: its node and mark types, attribute keys, nesting, marks held together and neighbours such as bold beside a code span.
|
||||
export function features(document) {
|
||||
const found = new Set()
|
||||
collect(document, undefined, found)
|
||||
@@ -47,7 +47,7 @@ function bestCandidate(candidates, covered) {
|
||||
showsType ||= isType(feature)
|
||||
}
|
||||
const score = added / candidate.size
|
||||
if (added > 0 && (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore)) {
|
||||
if (showsType && !bestShowsType || showsType === bestShowsType && score > bestScore) {
|
||||
best = candidate
|
||||
bestScore = score
|
||||
bestShowsType = showsType
|
||||
@@ -60,13 +60,11 @@ function bestCandidate(candidates, covered) {
|
||||
export function selectDocuments(candidates, limit) {
|
||||
const covered = new Set()
|
||||
const chosen = []
|
||||
let remaining = candidates
|
||||
while (chosen.length < limit) {
|
||||
const best = bestCandidate(remaining, covered)
|
||||
const best = bestCandidate(candidates, covered)
|
||||
if (best === undefined) break
|
||||
chosen.push(best)
|
||||
for (const feature of best.features) covered.add(feature)
|
||||
remaining = remaining.filter((candidate) => candidate !== best)
|
||||
}
|
||||
return chosen
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import { features, selectDocuments } from './select.mjs'
|
||||
|
||||
function paragraph(...content) {
|
||||
@@ -11,10 +12,23 @@ test('names node and mark types, attribute keys and nesting', () => {
|
||||
for (const feature of ['node:doc', 'node:paragraph', 'node:text', 'mark:link', 'attr:link.href', 'child:paragraph>text']) assert.ok(found.has(feature), feature)
|
||||
})
|
||||
|
||||
test('names a node attribute', () => {
|
||||
assert.ok(features(paragraph({ attrs: { level: 2 }, type: 'heading' })).has('attr:heading.level'))
|
||||
})
|
||||
|
||||
test('names no mark for a node without marks, and no parent for the document', () => {
|
||||
assert.deepEqual([...features({ type: 'doc' })], ['node:doc'])
|
||||
})
|
||||
|
||||
test('names marks held together', () => {
|
||||
assert.ok(features(paragraph({ marks: [{ type: 'strong' }, { type: 'em' }], text: 'a', type: 'text' })).has('marks:em+strong'))
|
||||
})
|
||||
|
||||
test('names no marks held together for one mark, or for marks on a node other than text', () => {
|
||||
const found = [...features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'strong' }, { type: 'em' }], type: 'mediaInline' }))]
|
||||
assert.deepEqual(found.filter((feature) => feature.startsWith('marks:')), [])
|
||||
})
|
||||
|
||||
test('names bold beside a code span and italics beside a hard break', () => {
|
||||
const found = features(paragraph({ marks: [{ type: 'strong' }], text: 'a', type: 'text' }, { marks: [{ type: 'code' }], text: 'b', type: 'text' }, { type: 'hardBreak' }, { marks: [{ type: 'em' }], text: 'c', type: 'text' }))
|
||||
assert.ok(found.has('beside:text[strong]|text[code]'))
|
||||
@@ -34,6 +48,18 @@ test('takes a candidate showing a new node or mark type ahead of any other', ()
|
||||
assert.deepEqual(selectDocuments([cheap, rare], 1), [rare])
|
||||
})
|
||||
|
||||
test('takes a candidate showing a new type wherever it stands among the candidates', () => {
|
||||
const rare = { features: new Set(['mark:breakout']), size: 1000 }
|
||||
const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
|
||||
assert.deepEqual(selectDocuments([rare, cheap], 1), [rare])
|
||||
})
|
||||
|
||||
test('counts a feature naming a type only by its opening', () => {
|
||||
const rare = { features: new Set(['beside:node:x']), size: 1000 }
|
||||
const cheap = { features: new Set(['beside:a|b', 'beside:b|c']), size: 10 }
|
||||
assert.deepEqual(selectDocuments([rare, cheap], 1), [cheap])
|
||||
})
|
||||
|
||||
test('stops at the limit', () => {
|
||||
const candidates = [{ features: new Set(['a']), size: 1 }, { features: new Set(['b']), size: 1 }]
|
||||
assert.equal(selectDocuments(candidates, 1).length, 1)
|
||||
|
||||
+28
-28
@@ -1,30 +1,4 @@
|
||||
[
|
||||
{
|
||||
"products": {
|
||||
"jira": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
},
|
||||
"wiki": {
|
||||
"licence": "CC-BY-SA-4.0",
|
||||
"licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
|
||||
}
|
||||
},
|
||||
"site": "lf-onap"
|
||||
},
|
||||
{
|
||||
"products": {
|
||||
"jira": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
},
|
||||
"wiki": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
}
|
||||
},
|
||||
"site": "lf-opendaylight"
|
||||
},
|
||||
{
|
||||
"products": {
|
||||
"jira": {
|
||||
@@ -58,7 +32,7 @@
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
}
|
||||
},
|
||||
"site": "lf-hyperledger"
|
||||
"site": "lf-edgexfoundry"
|
||||
},
|
||||
{
|
||||
"products": {
|
||||
@@ -67,6 +41,32 @@
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
}
|
||||
},
|
||||
"site": "lf-edgexfoundry"
|
||||
"site": "lf-hyperledger"
|
||||
},
|
||||
{
|
||||
"products": {
|
||||
"jira": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
},
|
||||
"wiki": {
|
||||
"licence": "CC-BY-SA-4.0",
|
||||
"licenceSource": "https://lf-onap.atlassian.net/wiki/spaces/DW/pages/16220224/Open+Source"
|
||||
}
|
||||
},
|
||||
"site": "lf-onap"
|
||||
},
|
||||
{
|
||||
"products": {
|
||||
"jira": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
},
|
||||
"wiki": {
|
||||
"licence": "CC-BY-3.0",
|
||||
"licenceSource": "https://www.linuxfoundation.org/legal/terms"
|
||||
}
|
||||
},
|
||||
"site": "lf-opendaylight"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// The pages a person reads, which name each document's creator. A wiki's web link also holds the page's title, and can hold an attachment's name, which the repo must not.
|
||||
export function jiraUrl(base, key, commentId) {
|
||||
return `${base}/browse/${key}${commentId === undefined ? '' : `?focusedCommentId=${commentId}`}`
|
||||
}
|
||||
|
||||
export function wikiUrl(base, webLink) {
|
||||
const [path, query] = webLink.split('?')
|
||||
const commentId = new URLSearchParams(query).get('focusedCommentId')
|
||||
return `${base}/wiki${path.replace(/(\/\d+)\/[^/]+$/, '$1')}${commentId === null ? '' : `?focusedCommentId=${commentId}`}`
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import { jiraUrl, wikiUrl } from './source-urls.mjs'
|
||||
|
||||
const base = 'https://site.atlassian.net'
|
||||
|
||||
test('links a Jira issue, and a comment on it', () => {
|
||||
assert.equal(jiraUrl(base, 'KEY-1'), 'https://site.atlassian.net/browse/KEY-1')
|
||||
assert.equal(jiraUrl(base, 'KEY-1', '42'), 'https://site.atlassian.net/browse/KEY-1?focusedCommentId=42')
|
||||
})
|
||||
|
||||
test('links a wiki page without the title its web link holds', () => {
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13')
|
||||
})
|
||||
|
||||
test('links a wiki comment without the title of its page', () => {
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/pages/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/pages/13?focusedCommentId=42')
|
||||
})
|
||||
|
||||
test('links a comment on a blog post without its title', () => {
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?focusedCommentId=42'), 'https://site.atlassian.net/wiki/spaces/AK/blog/2020/01/02/13?focusedCommentId=42')
|
||||
})
|
||||
|
||||
test('wikiUrl drops a preview, which names an attached file', () => {
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/blog/2020/01/02/13/Secret+Title?preview=%2F13%2F14%2FSecret.jpeg'), `${base}/wiki/spaces/AK/blog/2020/01/02/13`)
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/overview?preview=%2F13%2F14%2FSecret.jpeg&focusedCommentId=42'), `${base}/wiki/spaces/AK/overview?focusedCommentId=42`)
|
||||
})
|
||||
|
||||
test('wikiUrl keeps a space overview, which holds no title', () => {
|
||||
assert.equal(wikiUrl(base, '/spaces/AK/overview'), `${base}/wiki/spaces/AK/overview`)
|
||||
})
|
||||
@@ -1,9 +1,10 @@
|
||||
// Usage: node write-corpus.mjs <sites.json> <raw folder> <documents folder> <sources file>. The raw folder holds fetch.mjs's documents and run.sh's draws.
|
||||
import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { anonymize, leakedTokens, shapeDifferences } from './anonymize.mjs'
|
||||
import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
|
||||
import { anonymizeChecked } from './anonymize.mjs'
|
||||
import { features, selectDocuments } from './select.mjs'
|
||||
|
||||
// Selection prefers small documents, which keeps the corpus near 660 KB.
|
||||
const documentLimit = 150
|
||||
const largestDocument = 64 * 1024
|
||||
const [sitesPath, rawFolder, corpusFolder, sourcesPath] = process.argv.slice(2)
|
||||
@@ -23,22 +24,19 @@ function canonical(value) {
|
||||
}
|
||||
|
||||
const licences = new Map(JSON.parse(readFileSync(sitesPath, 'utf8')).flatMap(({ products, site }) => Object.entries(products).map(([product, licence]) => [`${site}-${product}`, licence])))
|
||||
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), persons: lines('persons'), uuids: lines('uuids'), words: lines('words') }
|
||||
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), uuids: lines('uuids'), words: lines('words') }
|
||||
const documentsFolder = join(rawFolder, 'documents')
|
||||
const candidates = readdirSync(documentsFolder).flatMap((file) => {
|
||||
const path = join(documentsFolder, file)
|
||||
const text = readFileSync(path, 'utf8')
|
||||
const text = readFileSync(join(documentsFolder, file), 'utf8')
|
||||
if (text.length > largestDocument) return []
|
||||
const { document, source } = JSON.parse(text)
|
||||
const { document, retrieved, source } = JSON.parse(text)
|
||||
const name = file.slice(0, -'.json'.length)
|
||||
const written = anonymize(document, draws)
|
||||
const differences = shapeDifferences(document, written)
|
||||
const leaks = leakedTokens(document, written)
|
||||
if (differences.length > 0 || leaks.length > 0) {
|
||||
console.error(`${name} left out: ${differences.length} shape differences, ${leaks.length} tokens still held`)
|
||||
const checked = anonymizeChecked(document, draws)
|
||||
if (!checked.ok) {
|
||||
console.error(`${name} left out: ${checked.differences.length} shape differences, ${checked.leaks.length} tokens still held`)
|
||||
return []
|
||||
}
|
||||
return [{ features: features(document), name, retrieved: statSync(path).mtime.toISOString().slice(0, 10), size: text.length, source, written }]
|
||||
return [{ features: features(document), name, retrieved, size: text.length, source, written: checked.written }]
|
||||
})
|
||||
|
||||
rmSync(corpusFolder, { force: true, recursive: true })
|
||||
|
||||
Reference in New Issue
Block a user