3
0
mirror of https://github.com/larvit/adf-codec.git synced 2026-11-20 23:25:43 +01:00

Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo

This commit is contained in:
2026-11-05 00:17:32 +01:00
committed by Lillem4n
parent 8fee35a63b
commit 1e3d6e5d65
51 changed files with 3164 additions and 1055 deletions
+46 -49
View File
@@ -1,6 +1,6 @@
// A node's or mark's direct attribute under one of these keys names a format or a layout, never someone's words, so its value stays.
const formatKeys = new Set([
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'colwidth', 'columnRuleStyle', 'displayMode',
'__autoSize', 'accessLevel', 'align', 'annotationType', 'background', 'breakoutWidth', 'color', 'colspan', 'columnRuleStyle', 'colwidth', 'displayMode',
'extensionKey', 'extensionType', 'height', 'isNumberColumnEnabled', 'language', 'layout', 'level', 'mode', 'order', 'originalHeight', 'originalWidth',
'panelColor', 'panelType', 'rowspan', 'shadowColor', 'size', 'state', 'style', 'type', 'userType', 'valign', 'width', 'widthType',
])
@@ -10,7 +10,7 @@ const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12
const uuid = new RegExp(`^${uuidPattern}$`, 'i')
const uuidOrToken = new RegExp(`(?<![\\p{L}\\p{N}])${uuidPattern}(?![\\p{L}\\p{M}\\p{N}])|${token.source}`, 'giu')
const uuidVersionAt = 14
// A shorter token cannot leak anything, and a document may hold all of them, so drawing around it could never end.
// A chance match below this length identifies nobody, and a document may hold every such token, so redrawing around them might never end.
const shortestLeak = 3
const redraws = 64
@@ -22,47 +22,45 @@ function tokensOf(text) {
return [...text.matchAll(token)].filter((found) => !isScheme(text, found[0], found.index)).map((found) => found[0])
}
// Calls `visit` with each leaf and its path, as `kept`, `person` or `free`, the last two being what the anonymizer replaces.
function walk(value, path, visit) {
// Calls `visit` with each leaf, its steps from the document and its kind: `kept`, or `free`, which the anonymizer replaces.
function walk(value, steps, visit) {
if (Array.isArray(value)) {
value.forEach((item, index) => walk(item, `${path}/${index}`, visit))
value.forEach((item, index) => walk(item, [...steps, index], visit))
} else if (typeof value === 'object' && value !== null) {
for (const [key, member] of Object.entries(value)) {
const memberPath = `${path}/${key}`
if (key === 'attrs') walkAttributes(member, memberPath, visit, value.type)
else if (key === 'content' || key === 'marks') walk(member, memberPath, visit)
else if (key === 'type' || key === 'version') visit(member, memberPath, 'kept')
else walkFree(member, memberPath, visit)
const memberSteps = [...steps, key]
if (key === 'attrs') walkAttributes(member, memberSteps, visit)
else if (key === 'content' || key === 'marks') walk(member, memberSteps, visit)
else if (key === 'type' || key === 'version') visit(member, memberSteps, 'kept')
else walkFree(member, memberSteps, visit)
}
} else {
visit(value, path, 'free')
visit(value, steps, 'free')
}
}
// `<app id>/<environment id>/...`, so one more id, where any other extension key names a macro.
// A Forge key reads `<app id>/<environment id>/...`, so it holds ids to replace. Any other extension key names a macro and stays.
function isForgeKey(key, value) {
return key === 'extensionKey' && String(value).includes('/')
}
function walkAttributes(attributes, path, visit, nodeType) {
function walkAttributes(attributes, steps, visit) {
for (const [key, member] of Object.entries(attributes)) {
const memberPath = `${path}/${key}`
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept')
else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person')
else walkFree(member, memberPath, visit)
const memberSteps = [...steps, key]
if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberSteps, 'kept')
else walkFree(member, memberSteps, visit)
}
}
function walkFree(value, path, visit) {
if (Array.isArray(value)) value.forEach((item, index) => walkFree(item, `${path}/${index}`, visit))
else if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, `${path}/${key}`, visit)
else visit(value, path, 'free')
function walkFree(value, steps, visit) {
if (typeof value === 'object' && value !== null) for (const [key, member] of Object.entries(value)) walkFree(member, [...steps, key], visit)
else visit(value, steps, 'free')
}
function replacedStrings(document) {
const strings = []
walk(document, '', (value, _path, kind) => {
if (kind !== 'kept' && typeof value === 'string') strings.push(value)
walk(document, [], (value, _steps, kind) => {
if (kind === 'free' && typeof value === 'string') strings.push(value)
})
return strings
}
@@ -101,7 +99,7 @@ function wordDrawer(words) {
}
function withCaseOf(original, word) {
if (original.length > 1 && original === original.toUpperCase()) return word.toUpperCase()
if (original === original.toUpperCase() && original !== original.toLowerCase()) return word.toUpperCase()
if (original[0] !== original[0].toLowerCase()) return word[0].toUpperCase() + word.slice(1)
return word
}
@@ -111,7 +109,6 @@ function replacer(document, draws) {
const digits = stream(draws.digits)
const hex = stream(draws.hex)
const letters = stream(draws.letters)
const nextPerson = cursor(draws.persons)
const nextUuid = cursor(draws.uuids)
const nextWord = wordDrawer(draws.words)
const spelled = new Map()
@@ -139,7 +136,7 @@ function replacer(document, draws) {
function word(original) {
if (original.length < shortestLeak) return withCaseOf(original, letters(original.length))
let tries = 0
// A long document can hold every word of the pool, so a redraw past the pool's size gains a suffix.
// A long document can hold every word of a length, so the last two of the draws `fresh` makes gain a suffix.
return withCaseOf(original, fresh(() => {
tries += 1
return tries < redraws ? nextWord(original.length) : nextWord(original.length) + letters(2)
@@ -172,42 +169,34 @@ function replacer(document, draws) {
})
}
function person(original) {
return memo(original, () => `${original.startsWith('@') ? '@' : ''}${fresh(nextPerson)}`)
}
function scalar(value) {
if (typeof value === 'string') return string(value)
if (typeof value === 'number') return Number(String(value).replaceAll(/\d+/g, number))
return value
}
return { person, scalar }
return scalar
}
function setAt(target, path, value) {
const steps = path.split('/').slice(1)
const last = steps.pop()
let holder = target
for (const step of steps) holder = holder[step]
holder[last] = value
function setAt(target, steps, value) {
const holder = steps.slice(0, -1).reduce((held, step) => held[step], target)
holder[steps.at(-1)] = value
}
// Every draw comes from `draws`: fejkdata's output, as run.sh renders it.
// `draws` is fejkdata's output, as run.sh renders it.
export function anonymize(document, draws) {
const { person, scalar } = replacer(document, draws)
const scalar = replacer(document, draws)
const written = structuredClone(document)
walk(document, '', (value, path, kind) => {
if (kind === 'person' && typeof value === 'string') setAt(written, path, person(value))
else if (kind === 'free') setAt(written, path, scalar(value))
walk(document, [], (value, steps, kind) => {
if (kind === 'free') setAt(written, steps, scalar(value))
})
return written
}
function structure(document) {
const found = new Map()
walk(document, '', (value, path, kind) => {
found.set(path, { kind, value })
walk(document, [], (value, steps, kind) => {
found.set(`/${steps.join('/')}`, { kind, value })
})
return found
}
@@ -217,12 +206,12 @@ function skeleton(text) {
}
function holdsSameShape(before, after) {
if (before.kind === 'kept') return before.value === after.value || JSON.stringify(before.value) === JSON.stringify(after.value)
if (before.kind === 'kept') return JSON.stringify(before.value) === JSON.stringify(after.value)
if (typeof before.value !== typeof after.value) return false
return before.kind === 'person' || typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
return typeof before.value !== 'string' || skeleton(before.value) === skeleton(after.value)
}
// The paths where `after` differs from `before` in anything but the words the anonymizer replaces.
// The paths where `after` differs from `before` in anything but the tokens the anonymizer replaces.
export function shapeDifferences(before, after) {
const beforeValues = structure(before)
const afterValues = structure(after)
@@ -231,11 +220,19 @@ export function shapeDifferences(before, after) {
const first = beforeValues.get(path)
const second = afterValues.get(path)
return first === undefined || second === undefined || !holdsSameShape(first, second)
}).sort()
})
}
// The tokens of `before`'s replaced values that `after`'s replaced values still hold.
export function leakedTokens(before, after) {
const held = heldTokens(after)
return [...heldTokens(before)].filter((found) => held.has(found)).sort()
return [...heldTokens(before)].filter((found) => held.has(found))
}
// The anonymized document, or what keeps it out of the corpus: a slip here would publish real text, and nothing later could see it.
export function anonymizeChecked(document, draws) {
const written = anonymize(document, draws)
const differences = shapeDifferences(document, written)
const leaks = leakedTokens(document, written)
return differences.length > 0 || leaks.length > 0 ? { differences, leaks, ok: false } : { ok: true, written }
}