mirror of
https://github.com/larvit/adf-codec.git
synced 2026-11-12 11:48:29 +01:00
52 lines
2.4 KiB
JavaScript
52 lines
2.4 KiB
JavaScript
import { join } from 'node:path'
|
|
import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
|
|
import { anonymizeChecked } from './anonymize.mjs'
|
|
import { features, selectDocuments } from './select.mjs'
|
|
|
|
// Selection prefers small documents, which keeps the corpus near 660 KB.
|
|
const documentLimit = 150
|
|
const largestDocument = 64 * 1024
|
|
const [sitesPath, rawFolder, corpusFolder, sourcesPath] = process.argv.slice(2)
|
|
|
|
function lines(name) {
|
|
return readFileSync(join(rawFolder, 'draws', `${name}.txt`), 'utf8').split('\n').filter((line) => line !== '')
|
|
}
|
|
|
|
function sorted(value) {
|
|
if (Array.isArray(value)) return value.map(sorted)
|
|
if (typeof value !== 'object' || value === null) return value
|
|
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]))
|
|
}
|
|
|
|
function canonical(value) {
|
|
return `${JSON.stringify(sorted(value), null, 2)}\n`
|
|
}
|
|
|
|
const licences = new Map(JSON.parse(readFileSync(sitesPath, 'utf8')).flatMap(({ products, site }) => Object.entries(products).map(([product, licence]) => [`${site}-${product}`, licence])))
|
|
const draws = { digits: lines('digits'), hex: lines('hex'), letters: lines('letters'), uuids: lines('uuids'), words: lines('words') }
|
|
const documentsFolder = join(rawFolder, 'documents')
|
|
const candidates = readdirSync(documentsFolder).flatMap((file) => {
|
|
const text = readFileSync(join(documentsFolder, file), 'utf8')
|
|
if (text.length > largestDocument) return []
|
|
const { document, retrieved, source } = JSON.parse(text)
|
|
const name = file.slice(0, -'.json'.length)
|
|
const checked = anonymizeChecked(document, draws)
|
|
if (!checked.ok) {
|
|
console.error(`${name} left out: ${checked.differences.length} shape differences, ${checked.leaks.length} tokens still held`)
|
|
return []
|
|
}
|
|
return [{ features: features(document), name, retrieved, size: text.length, source, written: checked.written }]
|
|
})
|
|
|
|
rmSync(corpusFolder, { force: true, recursive: true })
|
|
mkdirSync(corpusFolder, { recursive: true })
|
|
const sources = {}
|
|
for (const { name, retrieved, source, written } of selectDocuments(candidates, documentLimit)) {
|
|
writeFileSync(join(corpusFolder, `${name}.json`), canonical(written))
|
|
const [, site, product] = /^(.+)-(jira|wiki)-/.exec(name)
|
|
sources[name] = { ...licences.get(`${site}-${product}`), retrieved, url: source }
|
|
}
|
|
writeFileSync(sourcesPath, canonical(sources))
|
|
console.error(`${Object.keys(sources).length} documents of ${candidates.length} candidates`)
|