mirror of
https://github.com/larvit/adf-codec.git
synced 2026-11-20 23:25:43 +01:00
Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
// A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
|
||||
const formatKeys = new Set([
|
||||
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
|
||||
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode',
|
||||
'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
|
||||
'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
|
||||
])
|
||||
@@ -10,7 +10,7 @@ const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12
|
||||
const uuid = new RegExp(`^${uuidPattern}$`, 'i')
|
||||
const uuidOrToken = new RegExp(`(?<![\\p{L}\\p{N}])${uuidPattern}(?![\\p{L}\\p{M}\\p{N}])|${token.source}`, 'giu')
|
||||
const uuidVersionAt = 14
|
||||
// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
|
||||
// A chance match below this length identifies nobody, and a document may hold every such token, so redrawing around them might never end.
|
||||
const shortestLeak = 3
|
||||
const redraws = 64
|
||||
|
||||
@@ -22,47 +22,45 @@ function tokensOf(text) {
|
||||
return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
|
||||
}
|
||||
|
||||
// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
|
||||
function walk(value, path, visit) {
|
||||
// Calls `visit` with each leaf, its steps from the document and its kind: `kept`, or `free`, which the anonymizer replaces.
|
||||
function walk(value, steps, visit) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
|
||||
value.forEach((item, index) => walk(item, [...steps, index], visit))
|
||||
} else if (typeof value === 'object' && value !== null) {
|
||||
for (const [key, member] of Object.entries(value)) {
|
||||
const memberPath = `${path}/${key}`
|
||||
if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
|
||||
else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
|
||||
else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
|
||||
else walkFree(member, memberPath, visit)
|
||||
const memberSteps = [...steps, key]
|
||||
if (key === 'attrs') walkAttributes(member, memberSteps, visit)
|
||||
else if (key === 'content' || key === 'marks') walk(member, memberSteps, visit)
|
||||
else if (key === 'type' || key === 'version') visit(member, memberSteps, 'kept')
|
||||
else walkFree(member, memberSteps, visit)
|
||||
}
|
||||
} else {
|
||||
visit(value, path, 'free')
|
||||
visit(value, steps, 'free')
|
||||
}
|
||||
}
|
||||
|
||||
// `<app id>/<environment id>/...`, so one more id, where any other extension key names a macro.
|
||||
// A Forge key reads `<app id>/<environment id>/...`, so it holds ids to replace. Any other extension key names a macro and stays.
|
||||
function isForgeKey(key, value) {
|
||||
return key === 'extensionKey' && String(value).includes('/')
|
||||
}
|
||||
|
||||
function walkAttributes(attributes, path, visit, nodeType) {
|
||||
function walkAttributes(attributes, steps, visit) {
|
||||
for (const [key, member] of Object.entries(attributes)) {
|
||||
const memberPath = `${path}/${key}`
|
||||
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept')
|
||||
else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
|
||||
else walkFree(member, memberPath, visit)
|
||||
const memberSteps = [...steps, key]
|
||||
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberSteps, 'kept')
|
||||
else walkFree(member, memberSteps, visit)
|
||||
}
|
||||
}
|
||||
|
||||
function walkFree(value, path, visit) {
|
||||
if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
|
||||
else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
|
||||
else visit(value, path, 'free')
|
||||
function walkFree(value, steps, visit) {
|
||||
if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, [...steps, key], visit)
|
||||
else visit(value, steps, 'free')
|
||||
}
|
||||
|
||||
function replacedStrings(document) {
|
||||
const strings = []
|
||||
walk(document, '', (value, _path, kind) => {
|
||||
if (kind !== 'kept' && typeof value === 'string') strings.push(value)
|
||||
walk(document, [], (value, _steps, kind) => {
|
||||
if (kind === 'free' && typeof value === 'string') strings.push(value)
|
||||
})
|
||||
return strings
|
||||
}
|
||||
@@ -101,7 +99,7 @@ function wordDrawer(words) {
|
||||
}
|
||||
|
||||
function withCaseOf(original, word) {
|
||||
if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
|
||||
if (original === original.toUpperCase() && original !== original.toLowerCase()) return word.toUpperCase()
|
||||
if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
|
||||
return word
|
||||
}
|
||||
@@ -111,7 +109,6 @@ function replacer(document, draws) {
|
||||
const digits = stream(draws.digits)
|
||||
const hex = stream(draws.hex)
|
||||
const letters = stream(draws.letters)
|
||||
const nextPerson = cursor(draws.persons)
|
||||
const nextUuid = cursor(draws.uuids)
|
||||
const nextWord = wordDrawer(draws.words)
|
||||
const spelled = new Map()
|
||||
@@ -139,7 +136,7 @@ function replacer(document, draws) {
|
||||
function word(original) {
|
||||
if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
|
||||
let tries = 0
|
||||
// A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
|
||||
// A long document can hold every word of a length, so the last two of the draws `fresh` makes gain a suffix.
|
||||
return withCaseOf(original, fresh(() => {
|
||||
tries += 1
|
||||
return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
|
||||
@@ -172,42 +169,34 @@ function replacer(document, draws) {
|
||||
})
|
||||
}
|
||||
|
||||
function person(original) {
|
||||
return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
|
||||
}
|
||||
|
||||
function scalar(value) {
|
||||
if (typeof value === 'string') return string(value)
|
||||
if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
|
||||
return value
|
||||
}
|
||||
|
||||
return { person, scalar }
|
||||
return scalar
|
||||
}
|
||||
|
||||
function setAt(target, path, value) {
|
||||
const steps = path.split('/').slice(1)
|
||||
const last = steps.pop()
|
||||
let holder = target
|
||||
for (const step of steps) holder = holder[step]
|
||||
holder[last] = value
|
||||
function setAt(target, steps, value) {
|
||||
const holder = steps.slice(0, -1).reduce((held, step) => held[step], target)
|
||||
holder[steps.at(-1)] = value
|
||||
}
|
||||
|
||||
// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
|
||||
// `draws` is fejkdata's output, as run.sh renders it.
|
||||
export function anonymize(document, draws) {
|
||||
const { person, scalar } = replacer(document, draws)
|
||||
const scalar = replacer(document, draws)
|
||||
const written = structuredClone(document)
|
||||
walk(document, '', (value, path, kind) => {
|
||||
if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
|
||||
else if (kind === 'free') setAt(written, path, scalar(value))
|
||||
walk(document, [], (value, steps, kind) => {
|
||||
if (kind === 'free') setAt(written, steps, scalar(value))
|
||||
})
|
||||
return written
|
||||
}
|
||||
|
||||
function structure(document) {
|
||||
const found = new Map()
|
||||
walk(document, '', (value, path, kind) => {
|
||||
found.set(path, { kind, value })
|
||||
walk(document, [], (value, steps, kind) => {
|
||||
found.set(`/${steps.join('/')}`, { kind, value })
|
||||
})
|
||||
return found
|
||||
}
|
||||
@@ -217,12 +206,12 @@ function skeleton(text) {
|
||||
}
|
||||
|
||||
function holdsSameShape(before, after) {
|
||||
if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
|
||||
if (before.kind === 'kept') return JSON.stringify(before.value) === JSON.stringify(after.value)
|
||||
if (typeof before.value !== typeof after.value) return false
|
||||
return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
|
||||
return typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
|
||||
}
|
||||
|
||||
// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
|
||||
// The paths where `after` differs from `before` in anything but the tokens the anonymizer replaces.
|
||||
export function shapeDifferences(before, after) {
|
||||
const beforeValues = structure(before)
|
||||
const afterValues = structure(after)
|
||||
@@ -231,11 +220,19 @@ export function shapeDifferences(before, after) {
|
||||
const first = beforeValues.get(path)
|
||||
const second = afterValues.get(path)
|
||||
return first === undefined || second === undefined || !holdsSameShape(first, second)
|
||||
}).sort()
|
||||
})
|
||||
}
|
||||
|
||||
// The tokens of `before`'s replaced values that `after`'s replaced values still hold.
|
||||
export function leakedTokens(before, after) {
|
||||
const held = heldTokens(after)
|
||||
return [...heldTokens(before)].filter((found) => held.has(found)).sort()
|
||||
return [...heldTokens(before)].filter((found) => held.has(found))
|
||||
}
|
||||
|
||||
// The anonymized document, or what keeps it out of the corpus: a slip here would publish real text, and nothing later could see it.
|
||||
export function anonymizeChecked(document, draws) {
|
||||
const written = anonymize(document, draws)
|
||||
const differences = shapeDifferences(document, written)
|
||||
const leaks = leakedTokens(document, written)
|
||||
return differences.length > 0 || leaks.length > 0 ? { differences, leaks, ok: false } : { ok: true, written }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user