From f210c7a76410322a6df3ff4690e6d070114ef426 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 1 Sep 2026 23:07:15 +0200 Subject: [PATCH] Embed the shipped data, load any fs.FS, New takes options, Fake is safe for concurrent use --- README.md | 104 ++++++++++++++++++++++--------------------- cmd/fejkdata/main.go | 41 ++++++++++------- data.go | 86 ++++++++++++++++++++++------------- fejkdata.go | 92 +++++++++++++++++++++++++------------- render.go | 2 + 5 files changed, 197 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 08ef2d2..e189c01 100644 --- a/README.md +++ b/README.md @@ -28,29 +28,29 @@ lacked the locale coverage and format control we needed. ## CLI -Install the `fejkdata` command, then give it one or more `--data-path` directories -and a path — it prints one value to stdout. Each dot segment descends one level: -folders, then the category (a JSON file), then fields inside it. +Install the `fejkdata` command and give it a path — it prints one value to stdout. +The shipped data set is built in; each dot segment descends one level: folders, +then the category (a JSON file), then fields inside it. ```sh go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest -fejkdata --data-path ./data/sv_SE person # Sara Eriksson -fejkdata --data-path ./data/sv_SE person.last # Eriksson (dotted path into a category) -fejkdata --data-path ./data sv_SE.person # point at the tree; the folder is a segment -fejkdata -d ./data/sv_SE -d ./mydata word # layer dirs; the last wins a name clash -fejkdata --seed 42 --data-path ./data/sv_SE address -fejkdata --repeat 3 --data-path ./data/sv_SE person # three values, one per line -fejkdata -n 3 --separator ', ' --data-path ./data/sv_SE word # nät, barn, sol -fejkdata --data-path ./data/sv_SE --list # every path this data offers +fejkdata sv_SE.person # Sara Eriksson +fejkdata sv_SE.person.last # Eriksson (dotted path into a category) +fejkdata --seed 42 sv_SE.address +fejkdata --repeat 3 sv_SE.person # three values, one per line +fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol +fejkdata --list # every path the data offers +fejkdata --data-path ./mydata sv_SE.word # layer a dir over the shipped data; last wins a clash +fejkdata --no-shipped-data -d ./mydata --list # only your data ``` Flags are GNU-style: `--name value` or `--name=value`, short aliases `-d`, `-n`, `-s`, `-h`, in any position; `--` ends the flags. `--data-path` is repeatable -(last wins a name clash). `--repeat N` renders the path N times — each an -independent draw — joined by `--separator` (default a newline, so values land one -per line). Not sure what a data set offers? `--list` prints every path you can ask -for; `--version` prints the build version. +(last wins a name clash) and `--no-shipped-data` leaves the built-in set out. +`--repeat N` renders the path N times — each an independent draw — joined by +`--separator` (default a newline, so values land one per line). `--list` prints +every path you can ask for; `--version` prints the build version. Without installing, run it from a checkout with `go run ./cmd/fejkdata …`. Exit codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime error @@ -59,7 +59,7 @@ codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime erro ### Generating a file from a custom template A category is just a JSON file in a data directory, so you can drop in your -own and render it — no code change. Save this as `data/sv_SE/sql.json`: +own and render it — no code change. Save this as `mydata/sql.json`: ```json { @@ -79,7 +79,7 @@ the `),(` separator; the outer `V#ALUES(…)` wraps that into one valid row list letter token — see [Data format](#data-format).) ```sh -fejkdata --seed 1 --data-path ./data/sv_SE sql +fejkdata --seed 1 --data-path ./mydata sql # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); ``` @@ -87,7 +87,7 @@ Raise the template's `repeat` for more rows per statement; use the CLI's `--repeat` for more statements — together they build a whole seed file: ```sh -fejkdata --repeat 100 --data-path ./data/sv_SE sql > seed.sql +fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` ## Library @@ -96,9 +96,9 @@ fejkdata --repeat 100 --data-path ./data/sv_SE sql > seed.sql go get gitea.larvit.se/larvit/fejkdata # requires Go 1.22+ (for math/rand/v2) ``` -Point `New` at one or more data directories, then generate values by path with -`Fake`. Each dot segment descends one level: folders, then the category (a JSON -file), then fields inside it. +`New` loads the shipped data set, plus any `WithDataPath` directories layered over +it; generate values by path with `Fake`. Each dot segment descends one level: +folders, then the category (a JSON file), then fields inside it. ```go package main @@ -111,37 +111,40 @@ import ( ) func main() { - f, err := fejkdata.New([]string{"./data/sv_SE"}) + f, err := fejkdata.New() if err != nil { log.Fatal(err) } - for _, path := range []string{"person", "address", "phone", "address.locality"} { + for _, path := range []string{"sv_SE.person", "sv_SE.address", "sv_SE.phone", "sv_SE.address.locality"} { v, err := f.Fake(path) if err != nil { log.Fatal(err) } - fmt.Printf("%-18s %s\n", path, v) + fmt.Printf("%-24s %s\n", path, v) } } ``` ``` -person Sara Eriksson -address Kungsvägen 68 - 379 17 Stockholm -phone 072-402 91 67 -address.locality Linköping +sv_SE.person Sara Eriksson +sv_SE.address Kungsvägen 68 + 379 17 Stockholm +sv_SE.phone 072-402 91 67 +sv_SE.address.locality Linköping ``` -Seed a generator for reproducible output — same seed + locale yields an identical -sequence, handy for stable tests: +Options: `WithSeed(n)` for a reproducible sequence — same seed + data yields an +identical sequence, handy for stable tests; `WithDataPath(dir)` layers a directory +(repeat it to layer several, last wins a clash); `WithDataFS(fsys)` layers an +`fs.FS`, such as your own `embed.FS`; `WithoutShippedData()` loads only what you +give. ```go -a, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -b, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -av, _ := a.Fake("person") -bv, _ := b.Fake("person") +a, _ := fejkdata.New(fejkdata.WithSeed(42)) +b, _ := fejkdata.New(fejkdata.WithSeed(42)) +av, _ := a.Fake("sv_SE.person") +bv, _ := b.Fake("sv_SE.person") av == bv // true ``` @@ -149,28 +152,27 @@ av == bv // true dotted fields and folder segments (what the CLI's `--list` prints). Every path it lists renders. -A `*Generator` is **not** safe for concurrent use — create one per goroutine. +A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only +when drawn from one goroutine. ## Data -The library ships a ready-to-use set under [`data/`](data): one folder per locale -(`en_US`, `sv_SE`) plus a locale-neutral `misc` folder. Point either tool at the -whole tree, a single folder, a copy, or your own directory — anywhere on disk. A -category or folder name must not use `.`, `|`, `(` or `}` (see [Data -format](#data-format)), and dot-prefixed entries are skipped, so a data directory -can also be a checkout. +The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) +plus a locale-neutral `misc` folder — is embedded in the library and the CLI, so +both work with no data on disk. Layer your own directories over it; a category or +folder name must not use `.`, `|`, `(` or `}` (see [Data format](#data-format)), +and dot-prefixed entries are skipped, so a data directory can also be a checkout. A directory is just a namespace. Each JSON file is a category named after the file; each subdirectory is a dot-path segment — folders nest exactly like JSON -objects do. So `data/sv_SE/person.json` is `Fake("person")` when you point at -`data/sv_SE`, or `Fake("sv_SE.person")` when you point at `data`. +objects do. So `mydata/sv_SE/person.json` is `Fake("sv_SE.person")`. -Pass several directories and they merge, left to right: matching folders combine -by their children, and any other clash is won by the last directory loaded. That -lets you layer your own data over the built-ins without copying them: +Sources merge in order: matching folders combine by their children, and any other +clash is won by the last one loaded. That lets you override a shipped category +without copying the rest: ```go -fejkdata.New([]string{"./data/sv_SE", "./mydata"}) // mydata overrides on a clash +fejkdata.New(fejkdata.WithDataPath("./mydata")) // mydata/sv_SE/person.json replaces sv_SE.person ``` Each shipped locale carries these categories, formatted per locale (e.g. `date` @@ -517,16 +519,16 @@ docker compose run --rm test # latest ## Layout ``` -fejkdata.go Generator, New, List, options, seeding +fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation render.go Fake and the recursive renderer (choices, format strings, paths, bound draws) template.go the {token} grammar: scanning, function and path tokens, validation reference.go {..path} binding across the tree, the render graph, and the walks over it builtins.go the {name()} function registry and its implementations calc.go the {calc()} arithmetic evaluator: parser, eval, validation -data.go data loading: folders/files -> namespace tree, multi-path merge +data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge cmd/fejkdata/ the `fejkdata` CLI (New + Fake/List over stdout) -data/ shipped data (JSON): locale folders + a misc folder +data/ shipped data (JSON), embedded at build: locale folders + a misc folder ``` To add a category, drop a JSON file into a data directory; to add a locale, add diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 7d34f7c..9dd65ac 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -1,10 +1,10 @@ -// Command fejkdata prints fake values from one or more data directories. +// Command fejkdata prints fake values from the shipped data and any directories +// layered over it. // -// fejkdata --data-path ./data/sv_SE person # a full person -// fejkdata --data-path ./data/sv_SE person.last # just the surname -// fejkdata --data-path ./data sv_SE.person # point at the tree, address by folder -// fejkdata --data-path ./data/sv_SE --data-path ./mydata person # layer custom data; last dir wins -// fejkdata --seed 42 --data-path ./data/sv_SE address +// fejkdata sv_SE.person # a full person +// fejkdata sv_SE.person.last # just the surname +// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins +// fejkdata --seed 42 sv_SE.address package main import ( @@ -23,13 +23,14 @@ const usage = `Usage: fejkdata [flags] a category, or a dotted path into one (person, person.last) - -d, --data-path D a data directory, e.g. ./data/sv_SE (repeatable; last wins on a clash) - -h, --help print this help, then exit - --list list the paths the data offers, then exit - -n, --repeat N render the path N times (default 1) - -s, --seed N seed for reproducible output - --separator S string between repeated values (default newline) - --version print the version, then exit + -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) + -h, --help print this help, then exit + --list list the paths the data offers, then exit + --no-shipped-data load only the --data-path directories + -n, --repeat N render the path N times (default 1) + -s, --seed N seed for reproducible output + --separator S string between repeated values (default newline) + --version print the version, then exit Flags may come before or after ; -- ends the flags. ` @@ -38,6 +39,7 @@ type invocation struct { dirs []string help bool list bool + noShipped bool paths []string repeat int seed uint64 @@ -57,6 +59,7 @@ var flagDefs = []flagDef{ {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, + {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, {"repeat", "n", true, func(in *invocation, v string) error { n, err := strconv.Atoi(v) if err != nil || n < 1 { @@ -167,8 +170,8 @@ func run(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stdout, "fejkdata "+buildVersion()) return 0 } - if len(in.dirs) == 0 { - return misuse(stderr, errors.New("--data-path is required")) + if in.noShipped && len(in.dirs) == 0 { + return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path")) } if in.list && len(in.paths) > 0 { return misuse(stderr, errors.New("--list takes no path")) @@ -178,10 +181,16 @@ func run(args []string, stdout, stderr io.Writer) int { } var opts []fejkdata.Option + if in.noShipped { + opts = append(opts, fejkdata.WithoutShippedData()) + } + for _, dir := range in.dirs { + opts = append(opts, fejkdata.WithDataPath(dir)) + } if in.seeded { opts = append(opts, fejkdata.WithSeed(in.seed)) } - f, err := fejkdata.New(in.dirs, opts...) + f, err := fejkdata.New(opts...) if err != nil { fmt.Fprintln(stderr, err) return 1 diff --git a/data.go b/data.go index a5b0403..0be4a02 100644 --- a/data.go +++ b/data.go @@ -3,29 +3,60 @@ package fejkdata import ( "encoding/json" "fmt" + "io/fs" "os" - "path/filepath" + "path" "strings" ) -// loadData loads every given directory into one namespace tree and returns its -// root children. A directory becomes a group; each *.json file in it compiles to -// a node keyed by its base name (address.json -> "address"); each subdirectory -// becomes a nested group, so folders turn into dot-path segments. Paths are -// merged left to right: matching groups merge by their children, and any other -// clash (a leaf, or a leaf-vs-group) is won by the last directory loaded. Once -// merged, linkRefs binds every {..path} reference against the final tree. -func loadData(paths []string) (map[string]node, error) { +// dataSource is one tree to load: an fs.FS and the directory in it to start from. +// label prefixes file names in errors; path, when set, is a directory on disk that +// must exist. +type dataSource struct { + fsys fs.FS + label string + path string + root string +} + +func (s dataSource) name(p string) string { + if s.label == "" { + return p + } + return path.Join(s.label, p) +} + +// loadData loads every source into one namespace tree and returns its root +// children. A directory becomes a group; each *.json file in it compiles to a node +// keyed by its base name (address.json -> "address"); each subdirectory becomes a +// nested group, so folders turn into dot-path segments. Sources merge left to right: +// matching groups merge by their children, and any other clash is won by the last +// source loaded. Once merged, linkRefs binds every {..path} reference against the +// final tree. +func loadData(sources []dataSource) (map[string]node, error) { root := map[string]node{} - for _, p := range paths { - g, err := loadDir(p) + for _, src := range sources { + if src.path != "" { + info, err := os.Stat(src.path) + if err != nil { + return nil, fmt.Errorf("%s: no such directory", src.path) + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", src.path) + } + } + dir := src.root + if dir == "" { + dir = "." + } + g, err := loadDir(src, dir) if err != nil { return nil, err } mergeChildren(root, g.children) } if len(root) == 0 { - return nil, fmt.Errorf("no .json data found in %v", paths) + return nil, fmt.Errorf("no .json data found") } if err := linkRefs(root); err != nil { return nil, err @@ -39,29 +70,22 @@ func loadData(paths []string) (map[string]node, error) { return root, nil } -// loadDir compiles one directory into a group. os.ReadDir yields entries sorted +// loadDir compiles one directory into a group. fs.ReadDir yields entries sorted // by name, so the tree is built deterministically. Empty subdirectories (no JSON // anywhere under them) are skipped rather than added as empty namespaces. -func loadDir(dir string) (*group, error) { - info, err := os.Stat(dir) +func loadDir(src dataSource, dir string) (*group, error) { + entries, err := fs.ReadDir(src.fsys, dir) if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, fmt.Errorf("%s is not a directory", dir) - } - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", src.name(dir), err) } g := &group{children: map[string]node{}} for _, e := range entries { if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data continue } - full := filepath.Join(dir, e.Name()) + full := path.Join(dir, e.Name()) if e.IsDir() { - child, err := loadDir(full) + child, err := loadDir(src, full) if err != nil { return nil, err } @@ -69,7 +93,7 @@ func loadDir(dir string) (*group, error) { continue } if err := checkName(e.Name()); err != nil { - return nil, fmt.Errorf("%s: folder %w", full, err) + return nil, fmt.Errorf("%s: folder %w", src.name(full), err) } g.children[e.Name()] = child continue @@ -79,19 +103,19 @@ func loadDir(dir string) (*group, error) { } name := strings.TrimSuffix(e.Name(), ".json") if err := checkName(name); err != nil { - return nil, fmt.Errorf("%s: category %w", full, err) + return nil, fmt.Errorf("%s: category %w", src.name(full), err) } - b, err := os.ReadFile(full) + b, err := fs.ReadFile(src.fsys, full) if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", src.name(full), err) } var raw any if err := json.Unmarshal(b, &raw); err != nil { - return nil, fmt.Errorf("%s: %w", full, err) + return nil, fmt.Errorf("%s: %w", src.name(full), err) } n, err := compile(raw) if err != nil { - return nil, fmt.Errorf("%s: %w", full, err) + return nil, fmt.Errorf("%s: %w", src.name(full), err) } g.children[name] = n } diff --git a/fejkdata.go b/fejkdata.go index e9f8319..def1fe6 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -1,78 +1,110 @@ // Package fejkdata generates fake data from recursive JSON templates. // -// Data lives in JSON on disk, not in Go. Point [New] at one or more data -// directories; folders and files become a dot-path namespace, then generate -// values by path: +// Data lives in JSON, not in Go. A generator starts from the shipped data set and +// layers any directories you add; folders and files become a dot-path namespace, +// then generate values by path: // -// f, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -// f.Fake("address") // "Storgatan 12\n234 56 Göteborg" -// f.Fake("address.locality") // "Göteborg" +// f, _ := fejkdata.New(fejkdata.WithSeed(42)) +// f.Fake("sv_SE.address") // "Storgatan 12\n234 56 Göteborg" +// f.Fake("sv_SE.address.locality") // "Göteborg" // -// A subdirectory is a namespace segment, so pointing at "./data" instead reaches -// a category as "sv_SE.address". Several directories are merged left to right; -// the last one wins on a name clash, so you can layer custom data over the -// built-ins. The JSON template format is documented in the README. -// -// A [Generator] is not safe for concurrent use; create one per goroutine. +// Several sources merge in order, the last winning a name clash, so custom data +// layers over the built-ins. The JSON template format is documented in the README. package fejkdata import ( crand "crypto/rand" + "embed" "encoding/binary" "fmt" + "io/fs" "math/rand/v2" + "os" "sort" + "sync" ) +//go:embed data +var shippedFS embed.FS + // Generator generates fake data from a loaded namespace tree. Create one with [New]. +// It is safe for concurrent use; a seeded sequence is reproducible only when drawn +// from one goroutine. type Generator struct { + mu sync.Mutex rand *session - categories map[string]node // root namespace: name -> compiled node tree + categories map[string]node } // session is one generator's mutable render state: the seeded rng plus the {seq()} -// counters. Scoping it to the Generator means sequences (and randomness) belong to -// that generator and reset when you create a new one. Embedding *rand.Rand makes a -// *session satisfy the rng interface the renderer draws from. +// counters. type session struct { *rand.Rand counters map[string]uint64 } -// next returns the next value (counting from 1) of the named {seq()} counter. func (s *session) next(key string) uint64 { s.counters[key]++ return s.counters[key] } type config struct { - seed uint64 - seeded bool + seed uint64 + seeded bool + shipped bool + sources []dataSource } // Option configures a [Generator]. type Option func(*config) -// WithSeed makes output reproducible: two generators with the same seed and locale +// WithSeed makes output reproducible: two generators with the same seed and data // emit identical sequences. func WithSeed(seed uint64) Option { return func(c *config) { c.seed, c.seeded = seed, true } } -// New builds a generator from one or more data directories (e.g. "./data/sv_SE"). -// Each JSON file becomes a category named after the file (address.json -> -// "address") and each subdirectory a namespace segment; directories are merged -// in order, the last winning a name clash. It errors on a missing directory, -// invalid JSON, or no data found. -func New(paths []string, opts ...Option) (*Generator, error) { - cats, err := loadData(paths) - if err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) +// WithDataPath layers a data directory over what is loaded before it. Repeat it to +// layer several; the last wins a name clash. +func WithDataPath(dir string) Option { + return func(c *config) { + c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, path: dir}) } - var c config +} + +// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own. +func WithDataFS(fsys fs.FS) Option { + return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) } +} + +// WithoutShippedData leaves the shipped data set out, so only the sources given +// with [WithDataPath] and [WithDataFS] load. +func WithoutShippedData() Option { + return func(c *config) { c.shipped = false } +} + +// New builds a generator from the shipped data set and the options' sources, merged +// in order with the last winning a name clash. Each JSON file becomes a category +// named after the file (address.json -> "address") and each subdirectory a +// namespace segment. It errors on a missing directory, invalid JSON, invalid data, +// or no data at all. +func New(opts ...Option) (*Generator, error) { + c := config{shipped: true} for _, opt := range opts { opt(&c) } + var sources []dataSource + if c.shipped { + sources = append(sources, dataSource{fsys: shippedFS, root: "data"}) + } + sources = append(sources, c.sources...) + if len(sources) == 0 { + return nil, fmt.Errorf("fejkdata: no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") + } + cats, err := loadData(sources) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } return &Generator{rand: newRand(c.seed, c.seeded), categories: cats}, nil } diff --git a/render.go b/render.go index d1985c2..13ca4bd 100644 --- a/render.go +++ b/render.go @@ -18,6 +18,8 @@ type rng interface { // e.g. "sv_SE.address" or "sv_SE.address.street". Choices along the way are // resolved at random. A path naming a folder (no value of its own) is an error. func (f *Generator) Fake(path string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) if err != nil { return "", fmt.Errorf("fejkdata: %s: %w", path, err)