From 8fee35a63b6878f47a64d39dd5fb7f5643c050be Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 4 Nov 2026 23:31:59 +0100 Subject: [PATCH] =?UTF-8?q?Answer=20security=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20replace=20F?= =?UTF-8?q?orge=20keys=20and=20digits=20of=20any=20script,=20and=20fetch?= =?UTF-8?q?=20with=20the=20repo=20read-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- corpus-gathering/anonymize.mjs | 9 +++++++-- corpus-gathering/anonymize.test.mjs | 19 +++++++++++++++++++ corpus-gathering/run.sh | 10 +++++++--- .../lf-onap-wiki-pages-16224503.json | 14 +++++++------- docs/decisions.md | 2 +- 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/corpus-gathering/anonymize.mjs b/corpus-gathering/anonymize.mjs index a2e8fe4..92b431e 100644 --- a/corpus-gathering/anonymize.mjs +++ b/corpus-gathering/anonymize.mjs @@ -39,10 +39,15 @@ function walk(value, path, visit) { } } +// `//...`, so one more id, where any other extension key names a macro. +function isForgeKey(key, value) { + return key === 'extensionKey' && String(value).includes('/') +} + function walkAttributes(attributes, path, visit, nodeType) { for (const [key, member] of Object.entries(attributes)) { const memberPath = `${path}/${key}` - if (formatKeys.has(key)) visit(member, memberPath, 'kept') + if (formatKeys.has(key) && !isForgeKey(key, member)) visit(member, memberPath, 'kept') else if (nodeType === 'mention' && key === 'text') visit(member, memberPath, 'person') else walkFree(member, memberPath, visit) } @@ -125,7 +130,7 @@ function replacer(document, draws) { function mixed(original) { if (/^[0-9a-f]+$/.test(original)) return fresh(() => hex(original.length)) return fresh(() => [...original].map((character) => { - if (/\d/.test(character)) return digits(1) + if (/\p{N}/u.test(character)) return digits(1) if (!/\p{L}/u.test(character)) return character return character === character.toLowerCase() ? letters(1) : letters(1).toUpperCase() }).join('')) diff --git a/corpus-gathering/anonymize.test.mjs b/corpus-gathering/anonymize.test.mjs index cc13f37..bf3a118 100644 --- a/corpus-gathering/anonymize.test.mjs +++ b/corpus-gathering/anonymize.test.mjs @@ -129,6 +129,25 @@ test('replaces everything inside a free-form attribute, whatever its keys', () = assert.match(String(written.attrs.parameters.pageId), /^[1-9]\d{4}$/) }) +test('replaces a Forge extension key, which opens with its app and environment ids', () => { + const extensionKey = '6ba7b810-9dad-11d1-80b4-00c04fd430c8/6fa459ea-ee8a-3ca4-894e-db77e160355e/static/report-macro' + const document = { content: [{ attrs: { extensionKey, extensionType: 'com.atlassian.ecosystem' }, type: 'extension' }], type: 'doc', version: 1 } + const written = anonymize(document, draws) + const [app, environment, ...module] = written.content[0].attrs.extensionKey.split('/') + assert.match(app, /^[0-9a-f]{8}-[0-9a-f]{4}-1[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/) + assert.match(environment, /^[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/) + assert.ok(!extensionKey.startsWith(app)) + assert.ok(!module.join('/').includes('report')) + assert.equal(written.content[0].attrs.extensionType, 'com.atlassian.ecosystem') + assert.deepEqual(shapeDifferences(document, written), []) + assert.deepEqual(leakedTokens(document, written), []) +}) + +test('replaces a digit of any script', () => { + const written = textOf(anonymize(paragraph({ text: '電話１３８００１３ ٣٤٥', type: 'text' }), draws)) + assert.match(written, /^[a-z0-9]+ [0-9]+$/) +}) + test('never draws a word the document already holds', () => { const written = textOf(anonymize(paragraph({ text: 'amber clever', type: 'text' }), draws)) assert.ok(!written.includes('amber')) diff --git a/corpus-gathering/run.sh b/corpus-gathering/run.sh index 3ad0617..787a255 100755 --- a/corpus-gathering/run.sh +++ b/corpus-gathering/run.sh @@ -6,8 +6,11 @@ raw=$(mkdir -p "$1" && cd "$1" && pwd) fejkdata_image=larvit/fejkdata:0.2.0 node_image=node:26.13.0-alpine3.24 +# Takes how to mount the repo, then the command. in_node() { - docker run --rm --memory 2g --memory-swap 2g --ulimit core=0 -u "$(id -u):$(id -g)" -e HOME=/tmp -v "$here/..:/repo" -v "$raw:/raw" -w /repo "$node_image" "$@" + repo_mount=$1 + shift + docker run --rm --memory 2g --memory-swap 2g --ulimit core=0 -u "$(id -u):$(id -g)" -e HOME=/tmp -v "$here/..:/repo:$repo_mount" -v "$raw:/raw" -w /repo "$node_image" "$@" } # Takes a draws file's name, the number of lines and the template of a line. @@ -15,7 +18,8 @@ draw() { printf '%s\n' "$3" | docker run --rm -i --memory 256m --memory-swap 256m "$fejkdata_image" --seed 163 --repeat "$2" > "$raw/draws/$1.txt" } -[ -d "$raw/documents" ] || in_node node corpus-gathering/fetch.mjs corpus-gathering/sites.json /raw +# The fetch's file names come from the sites' answers, so it gets the repo read-only. +[ -d "$raw/documents" ] || in_node ro node corpus-gathering/fetch.mjs corpus-gathering/sites.json /raw mkdir -p "$raw/draws" draw digits 5000 '{digits(32)}' @@ -25,4 +29,4 @@ draw persons 2000 '{/en_US.person}' draw uuids 20000 '{/misc.uuid}' draw words 20000 '{/en_US.word}' -in_node node corpus-gathering/write-corpus.mjs corpus-gathering/sites.json /raw corpus/real-world corpus/real-world-sources.json +in_node rw node corpus-gathering/write-corpus.mjs corpus-gathering/sites.json /raw corpus/real-world corpus/real-world-sources.json diff --git a/corpus/real-world/lf-onap-wiki-pages-16224503.json b/corpus/real-world/lf-onap-wiki-pages-16224503.json index a434b4c..c9fb3ab 100644 --- a/corpus/real-world/lf-onap-wiki-pages-16224503.json +++ b/corpus/real-world/lf-onap-wiki-pages-16224503.json @@ -432,12 +432,12 @@ "content": [ { "attrs": { - "extensionKey": "eb366d75-b34a-4e71-b861-8347bfc59cd1/46ecedef-5278-43ce-a739-7c0d50f0a75c/static/create-from-template-pro-in", + "extensionKey": "10596ea3-8914-4e72-9f94-d7c43524c9c8/844e7334-657b-4f3c-b35b-1d7537dcfb5a/ledger/beacon-lily-tranquil-ash-zj", "extensionType": "com.atlassian.ecosystem", - "localId": "wren-ivy-silver", + "localId": "wren-ivy-beacon", "parameters": { - "extensionId": "ash:coral:woodland::lavender/10596ea3-8914-4e72-9f94-d7c43524c9c8/844e7334-657b-4f3c-b35b-1d7537dcfb5a/vesper/silver-lily-rosemary-ash-zj", - "extensionTitle": "Ledger lily Tranquil Ash (Beacon)", + "extensionId": "ash:coral:lavender::rosemary/10596ea3-8914-4e72-9f94-d7c43524c9c8/844e7334-657b-4f3c-b35b-1d7537dcfb5a/ledger/beacon-lily-tranquil-ash-zj", + "extensionTitle": "Silver lily Woodland Ash (Vesper)", "guestParams": { "icon": "ivy-able", "iconPosition": "quiet", @@ -445,14 +445,14 @@ "parentPageId": "109548136", "spaceKey": "MR", "templateId": "321477982", - "text": "Ledger ash ASH Jasmine Vine", + "text": "Silver ash ASH Jasmine Vine", "titleTemplate": "7241-BM-FP ASH Jasmine", "type": "harvest" }, - "localId": "wren-ivy-silver", + "localId": "wren-ivy-beacon", "measuredHeight": 77.66431900783672 }, - "text": "Ledger lily Tranquil Ash (Beacon)" + "text": "Silver lily Woodland Ash (Vesper)" }, "type": "inlineExtension" } diff --git a/docs/decisions.md b/docs/decisions.md index 86dc8f0..52b9a22 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1205,7 +1205,7 @@ A site joins only with a published licence. All six are hosted by the Linux Foun The tool stops at 20 result pages of 100 per source, one request a second as the sites' robots.txt asks, and 150 documents of at most 64 KiB. Those numbers keep a fetch to minutes and the corpus -near 1 MB. Selection takes new node and mark types first, then the most new features per byte. +under 1 MB. Selection takes new node and mark types first, then the most new features per byte. 22,769 fetched documents hold no `bodiedSyncBlock`, `decisionItem`, `decisionList`, `extensionFrame`, `fragment`, `multiBodiedExtension` or `syncBlock`, and neither did the sites'