mirror of
https://github.com/larvit/adf-codec.git
synced 2026-11-21 07:32:58 +01:00
Answer product-owner round 1: 1 🔴, 5 🟡 (0 misread); add the licence notice, link each document to its page, stop a failed fetch reaching the corpus, keep raw folders out of the repo
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Usage: corpus-gathering/run.sh <raw folder>. Keep the raw folder outside the repo: its documents hold the sites' real text.
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
raw=$(mkdir -p "$1" && cd "$1" && pwd)
|
||||
[ $# -eq 1 ] || { echo 'Usage: corpus-gathering/run.sh <raw folder>' >&2; exit 2; }
|
||||
here=$(cd "$(dirname "$0")" && pwd -P)
|
||||
raw=$(mkdir -p "$1" && cd "$1" && pwd -P)
|
||||
case "$raw/" in "$(cd "$here/.." && pwd -P)"/*) echo "run.sh: keep the raw folder outside the repo: its documents hold the sites' real text" >&2; exit 2 ;; esac
|
||||
fejkdata_image=larvit/fejkdata:0.2.0
|
||||
node_image=node:26.13.0-alpine3.24
|
||||
|
||||
# Takes how to mount the repo, then the command.
|
||||
in_node() {
|
||||
repo_mount=$1
|
||||
shift
|
||||
@@ -25,7 +25,6 @@ mkdir -p "$raw/draws"
|
||||
draw digits 5000 '{digits(32)}'
|
||||
draw hex 5000 '{hex(32)}'
|
||||
draw letters 5000 '{lower(32)}'
|
||||
draw persons 2000 '{/en_US.person}'
|
||||
draw uuids 20000 '{/misc.uuid}'
|
||||
draw words 20000 '{/en_US.word}'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user