From f773738243d512ba7426211bbd08aa77990264e5 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 1 Sep 2026 23:01:04 +0200 Subject: [PATCH 01/38] Tests for GNU-style flags --- cmd/fejkdata/main_test.go | 149 +++++++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 50 deletions(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 1ced7b8..79eaa18 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -11,7 +11,6 @@ const ( enUS = "../../data/en_US" ) -// runOut runs the CLI and returns exit code, stdout, stderr. func runOut(args ...string) (int, string, string) { var out, errb bytes.Buffer code := run(args, &out, &errb) @@ -19,7 +18,7 @@ func runOut(args ...string) (int, string, string) { } func TestRunOutputsValue(t *testing.T) { - code, out, errb := runOut("-data-path", svSE, "person") + code, out, errb := runOut("--data-path", svSE, "person") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -32,12 +31,11 @@ func TestRunOutputsValue(t *testing.T) { } func TestRunDotPath(t *testing.T) { - // person.last descends to just a surname — never the full "First Last". - code, full, _ := runOut("-seed", "7", "-data-path", svSE, "person") + code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person") if code != 0 { t.Fatalf("person run = %d", code) } - code, last, errb := runOut("-seed", "7", "-data-path", svSE, "person.last") + code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last") if code != 0 { t.Fatalf("person.last run = %d, stderr=%q", code, errb) } @@ -49,17 +47,80 @@ func TestRunDotPath(t *testing.T) { } } -func TestRunSeedDeterministic(t *testing.T) { - _, a, _ := runOut("-seed", "42", "-data-path", svSE, "address") - _, b, _ := runOut("-seed", "42", "-data-path", svSE, "address") - if a != b { - t.Errorf("same seed diverged: %q != %q", a, b) +func TestRunSeedSpellings(t *testing.T) { + _, want, _ := runOut("--seed", "42", "--data-path", svSE, "address") + for _, args := range [][]string{ + {"--seed=42", "--data-path", svSE, "address"}, + {"-s", "42", "-d", svSE, "address"}, + {"--data-path", svSE, "address", "--seed", "42"}, + } { + code, got, errb := runOut(args...) + if code != 0 { + t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb) + } + if got != want { + t.Errorf("run(%v) = %q, want %q", args, got, want) + } + } +} + +func TestRunDoubleDashEndsFlags(t *testing.T) { + code, out, errb := runOut("--data-path", svSE, "--", "person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb) + } + code, _, errb = runOut("--data-path", svSE, "--", "--list") + if code != 1 || !strings.Contains(errb, "--list") { + t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb) + } +} + +func TestRunSingleDashLongIsRejected(t *testing.T) { + for _, c := range []struct { + args []string + want string + }{ + {[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"}, + {[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"}, + {[]string{"-data-path", svSE, "person"}, "use --data-path"}, + {[]string{"-list", "-d", svSE}, "use --list"}, + {[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"}, + {[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"}, + } { + code, _, errb := runOut(c.args...) + if code != 2 { + t.Errorf("run(%v) = %d, want 2", c.args, code) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } + } +} + +func TestRunFlagValues(t *testing.T) { + for _, c := range []struct { + args []string + want string + }{ + {[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`}, + {[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"}, + {[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"}, + {[]string{"--list=1", "-d", svSE}, "--list takes no value"}, + {[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"}, + {[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"}, + } { + code, _, errb := runOut(c.args...) + if code != 2 { + t.Errorf("run(%v) = %d, want 2", c.args, code) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } } } func TestRunRepeat(t *testing.T) { - // -repeat N prints N values, one per line by default. - code, out, errb := runOut("-seed", "1", "-repeat", "3", "-data-path", svSE, "word") + code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -67,11 +128,14 @@ func TestRunRepeat(t *testing.T) { if len(lines) != 3 { t.Errorf("repeat=3 gave %d lines: %q", len(lines), out) } + _, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word") + if short != out { + t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out) + } } func TestRunSeparator(t *testing.T) { - // -separator joins the repeated values instead of newlines. - code, out, errb := runOut("-repeat", "3", "-separator", ",", "-data-path", svSE, "word") + code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -84,8 +148,7 @@ func TestRunSeparator(t *testing.T) { } func TestRunRepeatAdvancesRNG(t *testing.T) { - // Each repeat is a fresh draw, not the same value N times. - code, out, errb := runOut("-seed", "1", "-repeat", "5", "-data-path", svSE, "person") + code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -98,35 +161,23 @@ func TestRunRepeatAdvancesRNG(t *testing.T) { } } -func TestRunRepeatInvalid(t *testing.T) { - // A non-positive repeat is misuse. - for _, r := range []string{"0", "-1"} { - code, _, errb := runOut("-repeat", r, "-data-path", svSE, "word") - if code != 2 { - t.Errorf("repeat=%s = %d, want 2", r, code) - } - if errb == "" { - t.Errorf("repeat=%s: want an error message", r) - } - } -} - -func TestRunUsageOnMissingArgs(t *testing.T) { - // Need at least one -data-path and exactly one positional path; else misuse. - for _, args := range [][]string{{}, {"-data-path", svSE}, {"person"}} { - code, _, errb := runOut(args...) +func TestRunMisuse(t *testing.T) { + for _, args := range [][]string{{}, {"--data-path", svSE}, {"person"}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}} { + code, out, errb := runOut(args...) if code != 2 { t.Errorf("run(%v) = %d, want 2", args, code) } - if !strings.Contains(errb, "Usage") { - t.Errorf("run(%v) stderr = %q, want usage", args, errb) + if !strings.Contains(errb, "try 'fejkdata --help'") { + t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb) + } + if out != "" { + t.Errorf("run(%v) stdout = %q, want nothing", args, out) } } } func TestRunMultipleDataPaths(t *testing.T) { - // -data-path repeats: dirs merge, last wins; the path stays positional. - code, out, errb := runOut("-data-path", enUS, "-data-path", svSE, "person") + code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person") if code != 0 { t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb) } @@ -136,7 +187,7 @@ func TestRunMultipleDataPaths(t *testing.T) { } func TestRunUnknownCategoryFails(t *testing.T) { - code, _, errb := runOut("-data-path", svSE, "nope") + code, _, errb := runOut("--data-path", svSE, "nope") if code != 1 { t.Fatalf("run = %d, want 1", code) } @@ -146,8 +197,7 @@ func TestRunUnknownCategoryFails(t *testing.T) { } func TestRunList(t *testing.T) { - // -list prints the discoverable paths and exits 0, no positional path needed. - code, out, errb := runOut("-data-path", svSE, "-list") + code, out, errb := runOut("--data-path", svSE, "--list") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -158,32 +208,31 @@ func TestRunList(t *testing.T) { } } -func TestRunHelpExitsZero(t *testing.T) { - // -h/-help is success (0), not misuse (2), so scripts under `set -e` survive. - for _, h := range []string{"-h", "-help"} { - code, _, errb := runOut(h) +func TestRunHelpOnStdout(t *testing.T) { + for _, h := range []string{"-h", "--help"} { + code, out, errb := runOut(h) if code != 0 { t.Errorf("%s = %d, want 0", h, code) } - if !strings.Contains(errb, "Usage") { - t.Errorf("%s: stderr = %q, want usage", h, errb) + if !strings.Contains(out, "Usage") || errb != "" { + t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb) } } } func TestRunVersion(t *testing.T) { - code, out, errb := runOut("-version") + code, out, errb := runOut("--version") if code != 0 { - t.Fatalf("-version = %d, stderr=%q", code, errb) + t.Fatalf("--version = %d, stderr=%q", code, errb) } version, ok := strings.CutPrefix(out, "fejkdata ") if !ok || strings.TrimSpace(version) == "" { - t.Errorf("-version = %q, want the command name and a version on stdout", out) + t.Errorf("--version = %q, want the command name and a version on stdout", out) } } func TestRunMissingDirFails(t *testing.T) { - code, _, errb := runOut("-data-path", "../../data/nope", "person") + code, _, errb := runOut("--data-path", "../../data/nope", "person") if code != 1 { t.Fatalf("run = %d, want 1", code) } -- 2.52.0 From cf9de214956583b7ba9e44f699276d5a861e87e6 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 1 Sep 2026 23:02:19 +0200 Subject: [PATCH 02/38] GNU-style flags: --name, short aliases, any position, help on stdout --- README.md | 39 +++---- cmd/fejkdata/main.go | 244 +++++++++++++++++++++++++++++-------------- 2 files changed, 188 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 13aeb6f..08ef2d2 100644 --- a/README.md +++ b/README.md @@ -28,31 +28,32 @@ lacked the locale coverage and format control we needed. ## CLI -Install the `fejkdata` command, then give it one or more `-data-path` directories +Install the `fejkdata` command, then give it one or more `--data-path` directories and a path — it prints one value to stdout. Each dot segment descends one level: folders, then the category (a JSON file), then fields inside it. ```sh go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest -fejkdata -data-path ./data/sv_SE person # Sara Eriksson -fejkdata -data-path ./data/sv_SE person.last # Eriksson (dotted path into a category) -fejkdata -data-path ./data sv_SE.person # point at the tree; the folder is a segment -fejkdata -data-path ./data/sv_SE -data-path ./mydata word # layer dirs; the last wins a name clash -fejkdata -seed 42 -data-path ./data/sv_SE address -fejkdata -repeat 3 -data-path ./data/sv_SE person # three values, one per line -fejkdata -repeat 3 -separator ', ' -data-path ./data/sv_SE word # nät, barn, sol -fejkdata -data-path ./data/sv_SE -list # every path this data offers +fejkdata --data-path ./data/sv_SE person # Sara Eriksson +fejkdata --data-path ./data/sv_SE person.last # Eriksson (dotted path into a category) +fejkdata --data-path ./data sv_SE.person # point at the tree; the folder is a segment +fejkdata -d ./data/sv_SE -d ./mydata word # layer dirs; the last wins a name clash +fejkdata --seed 42 --data-path ./data/sv_SE address +fejkdata --repeat 3 --data-path ./data/sv_SE person # three values, one per line +fejkdata -n 3 --separator ', ' --data-path ./data/sv_SE word # nät, barn, sol +fejkdata --data-path ./data/sv_SE --list # every path this data offers ``` -`-data-path` is repeatable (last wins a name clash) and the path comes last — -all flags must precede it. `-repeat N` renders the path N times — each an -independent draw — joined by `-separator` (default a newline, so values land one -per line). Not sure what a data set offers? `-list` prints every path you can ask -for; `-version` prints the build version. +Flags are GNU-style: `--name value` or `--name=value`, short aliases `-d`, `-n`, +`-s`, `-h`, in any position; `--` ends the flags. `--data-path` is repeatable +(last wins a name clash). `--repeat N` renders the path N times — each an +independent draw — joined by `--separator` (default a newline, so values land one +per line). Not sure what a data set offers? `--list` prints every path you can ask +for; `--version` prints the build version. Without installing, run it from a checkout with `go run ./cmd/fejkdata …`. Exit -codes: `0` success (including `-list`, `-version`, `-h`), `1` runtime error +codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime error (missing dir, unknown path), `2` misuse. ### Generating a file from a custom template @@ -78,15 +79,15 @@ the `),(` separator; the outer `V#ALUES(…)` wraps that into one valid row list letter token — see [Data format](#data-format).) ```sh -fejkdata -seed 1 -data-path ./data/sv_SE sql +fejkdata --seed 1 --data-path ./data/sv_SE sql # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); ``` Raise the template's `repeat` for more rows per statement; use the CLI's -`-repeat` for more statements — together they build a whole seed file: +`--repeat` for more statements — together they build a whole seed file: ```sh -fejkdata -repeat 100 -data-path ./data/sv_SE sql > seed.sql +fejkdata --repeat 100 --data-path ./data/sv_SE sql > seed.sql ``` ## Library @@ -145,7 +146,7 @@ av == bv // true ``` `f.List()` returns the sorted paths the loaded data offers — the categories, their -dotted fields and folder segments (what the CLI's `-list` prints). Every path it +dotted fields and folder segments (what the CLI's `--list` prints). Every path it lists renders. A `*Generator` is **not** safe for concurrent use — create one per goroutine. diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 24ac085..7d34f7c 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -1,122 +1,214 @@ -// Command fejkdata prints one fake value from one or more data directories. +// Command fejkdata prints fake values from one or more data directories. // -// fejkdata -data-path ./data/sv_SE person # a full person -// fejkdata -data-path ./data/sv_SE person.last # just the surname (dotted path) -// fejkdata -data-path ./data sv_SE.person # point at the tree, address by folder -// fejkdata -data-path ./data/sv_SE -data-path ./mydata person # layer custom data; last dir wins -// fejkdata -seed 42 -data-path ./data/sv_SE address -// -// It is a thin CLI over the fejkdata library: New(dirs) then Fake(path). +// fejkdata --data-path ./data/sv_SE person # a full person +// fejkdata --data-path ./data/sv_SE person.last # just the surname +// fejkdata --data-path ./data sv_SE.person # point at the tree, address by folder +// fejkdata --data-path ./data/sv_SE --data-path ./mydata person # layer custom data; last dir wins +// fejkdata --seed 42 --data-path ./data/sv_SE address package main import ( "errors" - "flag" "fmt" "io" "os" "runtime/debug" + "strconv" "strings" "gitea.larvit.se/larvit/fejkdata" ) -const usage = `Usage: fejkdata -data-path D [-data-path D]... [-seed N] [-repeat N] [-separator S] +const usage = `Usage: fejkdata [flags] - -data-path D a data directory, e.g. ./data/sv_SE (repeatable; last wins on clash) - a category, or a dotted path into one (person, person.last) - -seed N seed for reproducible output - -repeat N render the path N times (default 1) - -separator S string between repeated values (default newline) - -list list the paths the data offers, then exit - -version print the version, then exit + a category, or a dotted path into one (person, person.last) -Flags must come before .` + -d, --data-path D a data directory, e.g. ./data/sv_SE (repeatable; last wins on a clash) + -h, --help print this help, then exit + --list list the paths the data offers, then exit + -n, --repeat N render the path N times (default 1) + -s, --seed N seed for reproducible output + --separator S string between repeated values (default newline) + --version print the version, then exit -// stringList collects a repeatable string flag, preserving order. -type stringList []string +Flags may come before or after ; -- ends the flags. +` -func (s *stringList) String() string { return strings.Join(*s, ",") } +type invocation struct { + dirs []string + help bool + list bool + paths []string + repeat int + seed uint64 + seeded bool + separator string + version bool +} -func (s *stringList) Set(v string) error { *s = append(*s, v); return nil } +type flagDef struct { + long string + short string + value bool + set func(*invocation, string) error +} + +var flagDefs = []flagDef{ + {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, + {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, + {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, + {"repeat", "n", true, func(in *invocation, v string) error { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return fmt.Errorf("--repeat needs a positive integer, got %q", v) + } + in.repeat = n + return nil + }}, + {"seed", "s", true, func(in *invocation, v string) error { + n, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return fmt.Errorf("--seed needs an unsigned integer, got %q", v) + } + in.seed, in.seeded = n, true + return nil + }}, + {"separator", "", true, func(in *invocation, v string) error { in.separator = v; return nil }}, + {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, +} + +func flagByLong(name string) *flagDef { + for i := range flagDefs { + if flagDefs[i].long == name { + return &flagDefs[i] + } + } + return nil +} + +func flagByShort(name string) *flagDef { + for i := range flagDefs { + if flagDefs[i].short == name { + return &flagDefs[i] + } + } + return nil +} + +func parseArgs(argv []string) (invocation, error) { + in := invocation{repeat: 1, separator: "\n"} + for i := 0; i < len(argv); i++ { + arg := argv[i] + switch { + case arg == "--": + in.paths = append(in.paths, argv[i+1:]...) + return in, nil + case strings.HasPrefix(arg, "--"): + name, value, hasValue := strings.Cut(arg[2:], "=") + def := flagByLong(name) + if def == nil { + return in, fmt.Errorf("unknown flag --%s", name) + } + if !def.value { + if hasValue { + return in, fmt.Errorf("--%s takes no value", name) + } + _ = def.set(&in, "") + continue + } + if !hasValue { + if i++; i >= len(argv) { + return in, fmt.Errorf("--%s needs a value", name) + } + value = argv[i] + } + if err := def.set(&in, value); err != nil { + return in, err + } + case len(arg) > 1 && arg[0] == '-': + name, _, _ := strings.Cut(arg[1:], "=") + def := flagByShort(name) + if def == nil { + if flagByLong(name) != nil { + return in, fmt.Errorf("unknown flag %s; use --%s", arg, name) + } + return in, fmt.Errorf("unknown flag %s", arg) + } + if !def.value { + _ = def.set(&in, "") + continue + } + if i++; i >= len(argv) { + return in, fmt.Errorf("--%s needs a value", def.long) + } + if err := def.set(&in, argv[i]); err != nil { + return in, err + } + default: + in.paths = append(in.paths, arg) + } + } + return in, nil +} func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -// run is main's testable core: it returns the process exit code (0 ok, 1 -// runtime error, 2 misuse) and writes only to the given streams. +// run returns the exit code: 0 ok, 1 runtime error, 2 misuse. func run(args []string, stdout, stderr io.Writer) int { - fs := flag.NewFlagSet("fejkdata", flag.ContinueOnError) - fs.SetOutput(stderr) - fs.Usage = func() { fmt.Fprintln(stderr, usage) } - seed := fs.Uint64("seed", 0, "seed for reproducible output") - repeat := fs.Int("repeat", 1, "render the path this many times") - sep := fs.String("separator", "\n", "string between repeated values") - list := fs.Bool("list", false, "list the paths the data offers, then exit") - showVersion := fs.Bool("version", false, "print the version, then exit") - var dirs stringList - fs.Var(&dirs, "data-path", "a data directory to load (repeatable)") - if err := fs.Parse(args); err != nil { - if errors.Is(err, flag.ErrHelp) { // -h/-help already printed usage - return 0 - } - return 2 + in, err := parseArgs(args) + if err != nil { + return misuse(stderr, err) } - if *showVersion { + if in.help { + fmt.Fprint(stdout, usage) + return 0 + } + if in.version { fmt.Fprintln(stdout, "fejkdata "+buildVersion()) return 0 } - if len(dirs) == 0 { - fs.Usage() - return 2 + if len(in.dirs) == 0 { + return misuse(stderr, errors.New("--data-path is required")) + } + if in.list && len(in.paths) > 0 { + return misuse(stderr, errors.New("--list takes no path")) + } + if !in.list && len(in.paths) != 1 { + return misuse(stderr, fmt.Errorf("expected one path, got %d", len(in.paths))) } var opts []fejkdata.Option - fs.Visit(func(fl *flag.Flag) { - if fl.Name == "seed" { - opts = append(opts, fejkdata.WithSeed(*seed)) - } - }) - - if *list { - f, err := fejkdata.New(dirs, opts...) - if err != nil { - fmt.Fprintln(stderr, err) - return 1 - } + if in.seeded { + opts = append(opts, fejkdata.WithSeed(in.seed)) + } + f, err := fejkdata.New(in.dirs, opts...) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if in.list { for _, p := range f.List() { fmt.Fprintln(stdout, p) } return 0 } - if fs.NArg() != 1 { - fs.Usage() - return 2 - } - if *repeat < 1 { - fmt.Fprintln(stderr, "repeat must be a positive integer") - return 2 - } - - path := fs.Arg(0) - f, err := fejkdata.New(dirs, opts...) - if err != nil { - fmt.Fprintln(stderr, err) - return 1 - } - vals := make([]string, *repeat) + vals := make([]string, in.repeat) for i := range vals { - if vals[i], err = f.Fake(path); err != nil { + if vals[i], err = f.Fake(in.paths[0]); err != nil { fmt.Fprintln(stderr, err) return 1 } } - fmt.Fprintln(stdout, strings.Join(vals, *sep)) + fmt.Fprintln(stdout, strings.Join(vals, in.separator)) return 0 } -// buildVersion reports the module version stamped into the binary by `go install` -// (or "devel" for a local build), read from the build info — no version constant -// to bump, no extra dependency. +func misuse(stderr io.Writer, err error) int { + fmt.Fprintf(stderr, "fejkdata: %v\ntry 'fejkdata --help'\n", err) + return 2 +} + +// buildVersion is the module version go install stamps into the binary, or "devel". func buildVersion() string { if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" { return info.Main.Version -- 2.52.0 From 91cbae2a4496a391b51d647702efa54730d999a6 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 1 Sep 2026 23:03:53 +0200 Subject: [PATCH 03/38] Tests for the shipped data layer, New options and concurrent Fake --- Dockerfile | 2 +- bench_test.go | 5 +- bound_test.go | 76 ++++++++++++------------- cmd/fejkdata/main_test.go | 31 ++++++++++- compose.yaml | 2 +- edge_test.go | 10 ++-- fejkdata_test.go | 8 ++- loading_test.go | 2 +- reference_test.go | 6 +- shipped_data_test.go | 114 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 201 insertions(+), 55 deletions(-) create mode 100644 shipped_data_test.go diff --git a/Dockerfile b/Dockerfile index 63be7a8..227d6da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,4 @@ COPY . . RUN go vet ./... && \ { unformatted="$(gofmt -l .)"; test -z "$unformatted" || \ { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \ - go test ./... + go test -race ./... diff --git a/bench_test.go b/bench_test.go index 9c69865..6022f31 100644 --- a/bench_test.go +++ b/bench_test.go @@ -8,7 +8,7 @@ import ( func benchGenerator(b *testing.B, dir string) *Generator { b.Helper() - f, err := New([]string{dir}, WithSeed(1)) + f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1)) if err != nil { b.Fatal(err) } @@ -91,11 +91,10 @@ func BenchmarkBoundWide(b *testing.B) { benchPath(b, dir, "row") } -// BenchmarkNew measures load+compile+validate of the whole shipped tree. func BenchmarkNew(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - if _, err := New([]string{"data"}, WithSeed(1)); err != nil { + if _, err := New(WithSeed(1)); err != nil { b.Fatal(err) } } diff --git a/bound_test.go b/bound_test.go index b6eeac0..36ea30c 100644 --- a/bound_test.go +++ b/bound_test.go @@ -54,7 +54,7 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { "either order": `{"format":"{p.first} + {p}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil { t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name) continue @@ -64,9 +64,9 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { } } // The same path twice is one spelling, so it stays legal. - if _, err := New([]string{writeData(t, map[string]string{ + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{p.first} + {p.first}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, - })}); err != nil { + }))); err != nil { t.Errorf("New = %v, want one path read twice accepted", err) } } @@ -75,17 +75,17 @@ func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) { // A calc operand renders its field, so naming a bound level in one is the same // overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a // path into, and the two disagree. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` + `{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Fatalf("New = %v, want the calc operand rejected as an overlap", err) } // Arithmetic over fields no path names is untouched. - if _, err := New([]string{writeData(t, map[string]string{ + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`, - })}); err != nil { + }))); err != nil { t.Errorf("New = %v, want plain arithmetic accepted", err) } } @@ -104,16 +104,16 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { `"inner":{"format":"{..cat.p}"}}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err) } } // A reference to anything this format does not bind is untouched. - if _, err := New([]string{writeData(t, map[string]string{ + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{p.first} {..surname}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, "surname": `["Eriksson","Lindqvist"]`, - })}); err != nil { + }))); err != nil { t.Errorf("New = %v, want a reference outside the bound level accepted", err) } } @@ -122,9 +122,9 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { // A path token renders what it lands on, not the level it started from, so the // cycle walk has to follow it there. A head whose own format names nothing // would otherwise hide the cycle until render, where it is fatal. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "a": `{"format":"{p.x}","p":{"format":"static","x":{"format":"{..a}"}}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) } @@ -133,10 +133,10 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) { // {p.a} renders q, so it is a route to the level {q.x} holds — even though p's // own format names nothing. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":{"format":"{..thing.q}"}},` + `"q":{"format":"{x}","x":["1","2"]}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Fatalf("New = %v, want the second route to q rejected", err) } @@ -212,7 +212,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { }, } for name, c := range rejected { - _, err := New([]string{writeData(t, c.files)}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") { t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err) continue @@ -259,7 +259,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { }, } for name, files := range accepted { - if _, err := New([]string{writeData(t, files)}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { t.Errorf("%s: New = %v, want it accepted", name, err) } } @@ -282,7 +282,7 @@ func TestAPathReachesEveryVariantItMightDraw(t *testing.T) { }, } for name, c := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": c.file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file}))) if err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) } @@ -300,7 +300,7 @@ func TestALevelAPathNeverRendersIsAccepted(t *testing.T) { `"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {..thing.p}","a":["1","2"]}}`, } for name, file := range accepted { - if _, err := New([]string{writeData(t, map[string]string{"thing": file})}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"thing": file}))); err != nil { t.Errorf("%s: New = %v, want it accepted", name, err) } } @@ -315,7 +315,7 @@ func TestADeepDiamondChainLoads(t *testing.T) { `{"format":"{a}{b}","a":{"format":"{..l%d}"},"b":{"format":"{..l%d}"}}`, i-1, i-1) } files["thing"] = `{"format":"{p.first} {..l30}","p":{"format":"x","first":["A","B"]}}` - if _, err := New([]string{writeData(t, files)}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { t.Fatalf("New = %v, want a deep diamond chain to load", err) } } @@ -324,11 +324,11 @@ func TestASharedNodeIsWalkedOnce(t *testing.T) { // Two fields reaching one node make the render graph a diamond, not a tree. // The search past a bound level must take that in its stride rather than walk // the shared node once per route. - f, err := New([]string{writeData(t, map[string]string{ + f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{p.first}|{q}","p":[{"format":"{first}","first":["Anna","Bo"]}],` + `"q":{"format":"{a}{b}","a":{"format":"{..shared}"},"b":{"format":"{..shared}"}}}`, "shared": `["x"]`, - })}, WithSeed(1)) + })), WithSeed(1)) if err != nil { t.Fatalf("New = %v, want a shared node accepted", err) } @@ -341,9 +341,9 @@ func TestTheEarlierReaderIsNamed(t *testing.T) { // A token and a calc operand can name one level. The one the format writes // first is the one reported, so the error points at the same place a reader // looking at the format would start. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":[{"format":"{first}","first":["1","2"]}]}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) { t.Fatalf("New = %v, want the calc operand named, being written first", err) } @@ -353,10 +353,10 @@ func TestReferenceToAMatchingStringIsAccepted(t *testing.T) { // A literal renders one fixed string, so no draw of it can disagree with a // held one. Two unrelated literals that merely spell the same text must not // read as the same node — the trap when comparing a value type. - if _, err := New([]string{writeData(t, map[string]string{ + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{p.city} {..other.tag}","p":{"format":"{city}","city":"Stockholm"}}`, "other": `{"format":"x","tag":"Stockholm"}`, - })}); err != nil { + }))); err != nil { t.Fatalf("New = %v, want a reference to a matching string accepted", err) } } @@ -378,9 +378,9 @@ func TestNestedChoiceDrawsOneVariant(t *testing.T) { func TestRepeatingLevelIsNamedInTheError(t *testing.T) { // The level carrying the repeat is the one to fix, so the error names it // rather than the head the path started from. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":["z"]}}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), `"p.a"`) { t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err) } @@ -390,9 +390,9 @@ func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) { // A choice of rows is the shape this feature is for, so the repeat rule has to // reach inside one — otherwise the direct spelling is a load error and the same // mistake behind a choice silently drops the repeat. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":["x"]},{"format":"{a}","a":["y"]}]}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want the repeat behind a choice rejected", err) } @@ -401,9 +401,9 @@ func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) { func TestPathIntoARepeatingLevelIsRejected(t *testing.T) { // A path reads one level's draw, so it can never apply that level's repeat. // The engine rejects options that cannot take effect, and this is one. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":["z"]}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want a path into a repeating level rejected", err) } @@ -412,9 +412,9 @@ func TestPathIntoARepeatingLevelIsRejected(t *testing.T) { func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) { // A head that is one template, not a choice, reports the missing segment // directly — there are no variants to compare. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), `no field "nope"`) { t.Fatalf("New = %v, want it to name the missing field", err) } @@ -532,7 +532,7 @@ func TestDottedTokenErrors(t *testing.T) { }, } for name, c := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": c.file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file}))) if err == nil { t.Errorf("%s: New = nil error, want it rejected at load", name) continue @@ -665,7 +665,7 @@ func TestEmptyPathSegmentIsRejected(t *testing.T) { "double dot": `{"format":"[{a..b}]","a":{"format":"x"}}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil { t.Errorf("%s: New = nil error, want the unfinished path rejected", name) continue @@ -680,9 +680,9 @@ func TestCycleThroughAPathTokenIsRejected(t *testing.T) { // A path token is a render edge like any other, so a cycle routed through one // must be caught at New. Reaching render would be fatal: the recursion never // terminates, and a stack overflow cannot be recovered. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "a": `{"format":"{p.x}","p":{"format":"{x}","x":{"format":"{..a}"}}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) } @@ -692,7 +692,7 @@ func TestBoundPathIsReachableByFake(t *testing.T) { // Binding changes how a format reads a sibling, not what List and Fake offer: // the sub-fields stay addressable on their own. dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")}) - f, err := New([]string{dir}, WithSeed(9)) + f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9)) if err != nil { t.Fatalf("New = %v", err) } diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 79eaa18..56cb624 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -162,7 +162,7 @@ func TestRunRepeatAdvancesRNG(t *testing.T) { } func TestRunMisuse(t *testing.T) { - for _, args := range [][]string{{}, {"--data-path", svSE}, {"person"}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}} { + for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} { code, out, errb := runOut(args...) if code != 2 { t.Errorf("run(%v) = %d, want 2", args, code) @@ -240,3 +240,32 @@ func TestRunMissingDirFails(t *testing.T) { t.Error("want an error message on stderr") } } + +func TestRunShippedDataByDefault(t *testing.T) { + code, out, errb := runOut("--seed", "1", "sv_SE.person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb) + } + code, list, _ := runOut("--list") + if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") { + t.Errorf("--list without --data-path = %d, %q", code, list) + } + code, _, errb = runOut("person") + if code != 1 || !strings.Contains(errb, "person") { + t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb) + } +} + +func TestRunNoShippedData(t *testing.T) { + code, list, errb := runOut("--no-shipped-data", "-d", svSE, "--list") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + if strings.Contains(list, "en_US") || !strings.Contains(list, "person\n") { + t.Errorf("--no-shipped-data --list = %q, want only the given dir", list) + } + code, out, _ := runOut("--no-shipped-data", "-d", svSE, "-s", "3", "person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Errorf("run = %d, out=%q", code, out) + } +} diff --git a/compose.yaml b/compose.yaml index 18f706c..fda93dd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,7 +28,7 @@ x-go: &go services: test: <<: *go - command: go test ./... + command: go test -race ./... cover: <<: *go diff --git a/edge_test.go b/edge_test.go index 61f4bb6..223e31b 100644 --- a/edge_test.go +++ b/edge_test.go @@ -56,11 +56,11 @@ func TestNewErrors(t *testing.T) { if err := os.WriteFile(file, []byte("{}"), 0o644); err != nil { t.Fatal(err) } - if _, err := New([]string{file}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(file)); err == nil { t.Error("New(file) = nil error, want not-a-directory error") } // Invalid JSON in a category file fails. - if _, err := New([]string{writeData(t, map[string]string{"broken": `{ not json`})}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"broken": `{ not json`}))); err == nil { t.Error("New(invalid JSON) = nil error") } // An option that cannot take effect, and a category or folder no dot path can @@ -168,7 +168,7 @@ func TestNewErrors(t *testing.T) { }, } for name, c := range rejected { - _, err := New([]string{writeData(t, c.files)}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err == nil { t.Errorf("%s: New = nil error, want it rejected at load", name) continue @@ -195,7 +195,7 @@ func TestNewErrors(t *testing.T) { "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":["1"]}`}, "a", "11"}, } for name, c := range accepted { - f, err := New([]string{writeData(t, c.files)}) + f, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err != nil { t.Errorf("%s: New = %v, want it accepted", name, err) continue @@ -272,7 +272,7 @@ func TestPathKeyIsUnambiguous(t *testing.T) { dir := writeData(t, map[string]string{ "cat": `[{"format":"{a.b}","a.b":["1"]},{"format":"{a}","a":{"format":"{b}","b":["2"]}}]`, }) - _, err := New([]string{dir}) + _, err := New(WithoutShippedData(), WithDataPath(dir)) if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { t.Fatalf("New = %v, want the dotted field name rejected", err) } diff --git a/fejkdata_test.go b/fejkdata_test.go index ff5f564..d85401f 100644 --- a/fejkdata_test.go +++ b/fejkdata_test.go @@ -15,7 +15,11 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator { func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator { t.Helper() - f, err := New(dirs, opts...) + all := []Option{WithoutShippedData()} + for _, dir := range dirs { + all = append(all, WithDataPath(dir)) + } + f, err := New(append(all, opts...)...) if err != nil { t.Fatalf("New(%q): %v", dirs, err) } @@ -33,7 +37,7 @@ func fake(t *testing.T, f *Generator, path string) string { } func TestNewMissingDirectory(t *testing.T) { - _, err := New([]string{"data/de_DE"}) + _, err := New(WithoutShippedData(), WithDataPath("data/de_DE")) if err == nil || !strings.Contains(err.Error(), "de_DE") { t.Fatalf("New(missing) error = %v, want it to name the path", err) } diff --git a/loading_test.go b/loading_test.go index 5727bde..d8890ca 100644 --- a/loading_test.go +++ b/loading_test.go @@ -57,7 +57,7 @@ func TestNewLoadsAnyDirName(t *testing.T) { } func TestNewEmptyDirErrors(t *testing.T) { - if _, err := New([]string{writeData(t, nil)}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, nil))); err == nil { t.Fatal("New(empty dir) = nil error") } } diff --git a/reference_test.go b/reference_test.go index 62ceada..3ed55ef 100644 --- a/reference_test.go +++ b/reference_test.go @@ -107,7 +107,7 @@ func TestReferenceErrors(t *testing.T) { "field key using the reference prefix": {"cat": `{"format":"hi","..x":{"format":"{..nope}"}}`}, } for name, files := range cases { - if _, err := New([]string{writeData(t, files)}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { t.Errorf("%s: New = nil error, want a reference error", name) } } @@ -161,7 +161,7 @@ func TestNewErrorIsDeterministic(t *testing.T) { dir := writeData(t, files) var first string for i := 0; i < 50; i++ { - _, err := New([]string{dir}) + _, err := New(WithoutShippedData(), WithDataPath(dir)) if err == nil { t.Fatalf("%s: New = nil error, want a load error", name) } @@ -198,7 +198,7 @@ func TestNewErrorPathIsCanonical(t *testing.T) { }, } for _, c := range cases { - _, err := New([]string{writeData(t, c.files)}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err == nil { t.Errorf("%s: New = nil error", c.name) continue diff --git a/shipped_data_test.go b/shipped_data_test.go new file mode 100644 index 0000000..660fccd --- /dev/null +++ b/shipped_data_test.go @@ -0,0 +1,114 @@ +package fejkdata + +import ( + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/fstest" +) + +func TestNewDefaultsToShippedData(t *testing.T) { + f, err := New(WithSeed(1)) + if err != nil { + t.Fatalf("New() = %v", err) + } + paths := f.List() + for _, p := range []string{"sv_SE.person", "en_US.address", "misc.uuid"} { + if !slices.Contains(paths, p) { + t.Errorf("List() omits shipped %q", p) + } + } + if got := fake(t, f, "sv_SE.person"); got == "" { + t.Error("sv_SE.person rendered empty") + } +} + +func TestWithDataPathLayersOverShipped(t *testing.T) { + dir := writeData(t, map[string]string{"sv_SE/word": `["only-mine"]`, "greeting": `["hej"]`}) + f, err := New(WithDataPath(dir), WithSeed(1)) + if err != nil { + t.Fatalf("New = %v", err) + } + if got := fake(t, f, "sv_SE.word"); got != "only-mine" { + t.Errorf("sv_SE.word = %q, want the layered file to win", got) + } + if got := fake(t, f, "sv_SE.person"); got == "" { + t.Error("sv_SE.person should still come from the shipped data") + } + if got := fake(t, f, "greeting"); got != "hej" { + t.Errorf("greeting = %q, want hej", got) + } +} + +func TestUserDataMayReferenceShipped(t *testing.T) { + dir := writeData(t, map[string]string{"greeting": `{"format":"Hej {..sv_SE.person}!"}`}) + f, err := New(WithDataPath(dir), WithSeed(1)) + if err != nil { + t.Fatalf("New = %v", err) + } + if got := fake(t, f, "greeting"); !strings.HasPrefix(got, "Hej ") || !strings.HasSuffix(got, "!") { + t.Errorf("greeting = %q", got) + } +} + +func TestWithoutShippedDataNeedsASource(t *testing.T) { + _, err := New(WithoutShippedData()) + if err == nil || !strings.Contains(err.Error(), "WithDataPath") { + t.Fatalf("New(WithoutShippedData()) = %v, want an error naming WithDataPath", err) + } +} + +func TestWithoutShippedDataListsOnlyOwn(t *testing.T) { + dir := writeData(t, map[string]string{"greeting": `["hej"]`}) + f := newGenerator(t, dir, WithSeed(1)) + if got := f.List(); !reflect.DeepEqual(got, []string{"greeting"}) { + t.Errorf("List() = %v, want only the loaded dir", got) + } +} + +func TestWithDataFS(t *testing.T) { + fsys := fstest.MapFS{ + "greeting.json": {Data: []byte(`["hej"]`)}, + "nested/x.json": {Data: []byte(`{"format":"{y}","y":["z"]}`)}, + ".hidden.json": {Data: []byte(`["ignored"]`)}, + "broken/no.json": {Data: []byte(`["x"]`)}, + } + f, err := New(WithoutShippedData(), WithDataFS(fsys), WithSeed(1)) + if err != nil { + t.Fatalf("New(WithDataFS) = %v", err) + } + if got := fake(t, f, "greeting"); got != "hej" { + t.Errorf("greeting = %q, want hej", got) + } + if got := fake(t, f, "nested.x.y"); got != "z" { + t.Errorf("nested.x.y = %q, want z", got) + } + bad := fstest.MapFS{"broken.json": {Data: []byte(`{ not json`)}} + if _, err := New(WithoutShippedData(), WithDataFS(bad)); err == nil || !strings.Contains(err.Error(), "broken.json") { + t.Errorf("New(bad fs) = %v, want an error naming the file", err) + } +} + +func TestFakeIsSafeForConcurrentUse(t *testing.T) { + f, err := New(WithSeed(1)) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + if _, err := f.Fake("sv_SE.person"); err != nil { + t.Error(err) + return + } + f.List() + } + }() + } + wg.Wait() +} -- 2.52.0 From f210c7a76410322a6df3ff4690e6d070114ef426 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 1 Sep 2026 23:07:15 +0200 Subject: [PATCH 04/38] Embed the shipped data, load any fs.FS, New takes options, Fake is safe for concurrent use --- README.md | 104 ++++++++++++++++++++++--------------------- cmd/fejkdata/main.go | 41 ++++++++++------- data.go | 86 ++++++++++++++++++++++------------- fejkdata.go | 92 +++++++++++++++++++++++++------------- render.go | 2 + 5 files changed, 197 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 08ef2d2..e189c01 100644 --- a/README.md +++ b/README.md @@ -28,29 +28,29 @@ lacked the locale coverage and format control we needed. ## CLI -Install the `fejkdata` command, then give it one or more `--data-path` directories -and a path — it prints one value to stdout. Each dot segment descends one level: -folders, then the category (a JSON file), then fields inside it. +Install the `fejkdata` command and give it a path — it prints one value to stdout. +The shipped data set is built in; each dot segment descends one level: folders, +then the category (a JSON file), then fields inside it. ```sh go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest -fejkdata --data-path ./data/sv_SE person # Sara Eriksson -fejkdata --data-path ./data/sv_SE person.last # Eriksson (dotted path into a category) -fejkdata --data-path ./data sv_SE.person # point at the tree; the folder is a segment -fejkdata -d ./data/sv_SE -d ./mydata word # layer dirs; the last wins a name clash -fejkdata --seed 42 --data-path ./data/sv_SE address -fejkdata --repeat 3 --data-path ./data/sv_SE person # three values, one per line -fejkdata -n 3 --separator ', ' --data-path ./data/sv_SE word # nät, barn, sol -fejkdata --data-path ./data/sv_SE --list # every path this data offers +fejkdata sv_SE.person # Sara Eriksson +fejkdata sv_SE.person.last # Eriksson (dotted path into a category) +fejkdata --seed 42 sv_SE.address +fejkdata --repeat 3 sv_SE.person # three values, one per line +fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol +fejkdata --list # every path the data offers +fejkdata --data-path ./mydata sv_SE.word # layer a dir over the shipped data; last wins a clash +fejkdata --no-shipped-data -d ./mydata --list # only your data ``` Flags are GNU-style: `--name value` or `--name=value`, short aliases `-d`, `-n`, `-s`, `-h`, in any position; `--` ends the flags. `--data-path` is repeatable -(last wins a name clash). `--repeat N` renders the path N times — each an -independent draw — joined by `--separator` (default a newline, so values land one -per line). Not sure what a data set offers? `--list` prints every path you can ask -for; `--version` prints the build version. +(last wins a name clash) and `--no-shipped-data` leaves the built-in set out. +`--repeat N` renders the path N times — each an independent draw — joined by +`--separator` (default a newline, so values land one per line). `--list` prints +every path you can ask for; `--version` prints the build version. Without installing, run it from a checkout with `go run ./cmd/fejkdata …`. Exit codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime error @@ -59,7 +59,7 @@ codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime erro ### Generating a file from a custom template A category is just a JSON file in a data directory, so you can drop in your -own and render it — no code change. Save this as `data/sv_SE/sql.json`: +own and render it — no code change. Save this as `mydata/sql.json`: ```json { @@ -79,7 +79,7 @@ the `),(` separator; the outer `V#ALUES(…)` wraps that into one valid row list letter token — see [Data format](#data-format).) ```sh -fejkdata --seed 1 --data-path ./data/sv_SE sql +fejkdata --seed 1 --data-path ./mydata sql # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); ``` @@ -87,7 +87,7 @@ Raise the template's `repeat` for more rows per statement; use the CLI's `--repeat` for more statements — together they build a whole seed file: ```sh -fejkdata --repeat 100 --data-path ./data/sv_SE sql > seed.sql +fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` ## Library @@ -96,9 +96,9 @@ fejkdata --repeat 100 --data-path ./data/sv_SE sql > seed.sql go get gitea.larvit.se/larvit/fejkdata # requires Go 1.22+ (for math/rand/v2) ``` -Point `New` at one or more data directories, then generate values by path with -`Fake`. Each dot segment descends one level: folders, then the category (a JSON -file), then fields inside it. +`New` loads the shipped data set, plus any `WithDataPath` directories layered over +it; generate values by path with `Fake`. Each dot segment descends one level: +folders, then the category (a JSON file), then fields inside it. ```go package main @@ -111,37 +111,40 @@ import ( ) func main() { - f, err := fejkdata.New([]string{"./data/sv_SE"}) + f, err := fejkdata.New() if err != nil { log.Fatal(err) } - for _, path := range []string{"person", "address", "phone", "address.locality"} { + for _, path := range []string{"sv_SE.person", "sv_SE.address", "sv_SE.phone", "sv_SE.address.locality"} { v, err := f.Fake(path) if err != nil { log.Fatal(err) } - fmt.Printf("%-18s %s\n", path, v) + fmt.Printf("%-24s %s\n", path, v) } } ``` ``` -person Sara Eriksson -address Kungsvägen 68 - 379 17 Stockholm -phone 072-402 91 67 -address.locality Linköping +sv_SE.person Sara Eriksson +sv_SE.address Kungsvägen 68 + 379 17 Stockholm +sv_SE.phone 072-402 91 67 +sv_SE.address.locality Linköping ``` -Seed a generator for reproducible output — same seed + locale yields an identical -sequence, handy for stable tests: +Options: `WithSeed(n)` for a reproducible sequence — same seed + data yields an +identical sequence, handy for stable tests; `WithDataPath(dir)` layers a directory +(repeat it to layer several, last wins a clash); `WithDataFS(fsys)` layers an +`fs.FS`, such as your own `embed.FS`; `WithoutShippedData()` loads only what you +give. ```go -a, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -b, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -av, _ := a.Fake("person") -bv, _ := b.Fake("person") +a, _ := fejkdata.New(fejkdata.WithSeed(42)) +b, _ := fejkdata.New(fejkdata.WithSeed(42)) +av, _ := a.Fake("sv_SE.person") +bv, _ := b.Fake("sv_SE.person") av == bv // true ``` @@ -149,28 +152,27 @@ av == bv // true dotted fields and folder segments (what the CLI's `--list` prints). Every path it lists renders. -A `*Generator` is **not** safe for concurrent use — create one per goroutine. +A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only +when drawn from one goroutine. ## Data -The library ships a ready-to-use set under [`data/`](data): one folder per locale -(`en_US`, `sv_SE`) plus a locale-neutral `misc` folder. Point either tool at the -whole tree, a single folder, a copy, or your own directory — anywhere on disk. A -category or folder name must not use `.`, `|`, `(` or `}` (see [Data -format](#data-format)), and dot-prefixed entries are skipped, so a data directory -can also be a checkout. +The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) +plus a locale-neutral `misc` folder — is embedded in the library and the CLI, so +both work with no data on disk. Layer your own directories over it; a category or +folder name must not use `.`, `|`, `(` or `}` (see [Data format](#data-format)), +and dot-prefixed entries are skipped, so a data directory can also be a checkout. A directory is just a namespace. Each JSON file is a category named after the file; each subdirectory is a dot-path segment — folders nest exactly like JSON -objects do. So `data/sv_SE/person.json` is `Fake("person")` when you point at -`data/sv_SE`, or `Fake("sv_SE.person")` when you point at `data`. +objects do. So `mydata/sv_SE/person.json` is `Fake("sv_SE.person")`. -Pass several directories and they merge, left to right: matching folders combine -by their children, and any other clash is won by the last directory loaded. That -lets you layer your own data over the built-ins without copying them: +Sources merge in order: matching folders combine by their children, and any other +clash is won by the last one loaded. That lets you override a shipped category +without copying the rest: ```go -fejkdata.New([]string{"./data/sv_SE", "./mydata"}) // mydata overrides on a clash +fejkdata.New(fejkdata.WithDataPath("./mydata")) // mydata/sv_SE/person.json replaces sv_SE.person ``` Each shipped locale carries these categories, formatted per locale (e.g. `date` @@ -517,16 +519,16 @@ docker compose run --rm test # latest ## Layout ``` -fejkdata.go Generator, New, List, options, seeding +fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation render.go Fake and the recursive renderer (choices, format strings, paths, bound draws) template.go the {token} grammar: scanning, function and path tokens, validation reference.go {..path} binding across the tree, the render graph, and the walks over it builtins.go the {name()} function registry and its implementations calc.go the {calc()} arithmetic evaluator: parser, eval, validation -data.go data loading: folders/files -> namespace tree, multi-path merge +data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge cmd/fejkdata/ the `fejkdata` CLI (New + Fake/List over stdout) -data/ shipped data (JSON): locale folders + a misc folder +data/ shipped data (JSON), embedded at build: locale folders + a misc folder ``` To add a category, drop a JSON file into a data directory; to add a locale, add diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 7d34f7c..9dd65ac 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -1,10 +1,10 @@ -// Command fejkdata prints fake values from one or more data directories. +// Command fejkdata prints fake values from the shipped data and any directories +// layered over it. // -// fejkdata --data-path ./data/sv_SE person # a full person -// fejkdata --data-path ./data/sv_SE person.last # just the surname -// fejkdata --data-path ./data sv_SE.person # point at the tree, address by folder -// fejkdata --data-path ./data/sv_SE --data-path ./mydata person # layer custom data; last dir wins -// fejkdata --seed 42 --data-path ./data/sv_SE address +// fejkdata sv_SE.person # a full person +// fejkdata sv_SE.person.last # just the surname +// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins +// fejkdata --seed 42 sv_SE.address package main import ( @@ -23,13 +23,14 @@ const usage = `Usage: fejkdata [flags] a category, or a dotted path into one (person, person.last) - -d, --data-path D a data directory, e.g. ./data/sv_SE (repeatable; last wins on a clash) - -h, --help print this help, then exit - --list list the paths the data offers, then exit - -n, --repeat N render the path N times (default 1) - -s, --seed N seed for reproducible output - --separator S string between repeated values (default newline) - --version print the version, then exit + -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) + -h, --help print this help, then exit + --list list the paths the data offers, then exit + --no-shipped-data load only the --data-path directories + -n, --repeat N render the path N times (default 1) + -s, --seed N seed for reproducible output + --separator S string between repeated values (default newline) + --version print the version, then exit Flags may come before or after ; -- ends the flags. ` @@ -38,6 +39,7 @@ type invocation struct { dirs []string help bool list bool + noShipped bool paths []string repeat int seed uint64 @@ -57,6 +59,7 @@ var flagDefs = []flagDef{ {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, + {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, {"repeat", "n", true, func(in *invocation, v string) error { n, err := strconv.Atoi(v) if err != nil || n < 1 { @@ -167,8 +170,8 @@ func run(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stdout, "fejkdata "+buildVersion()) return 0 } - if len(in.dirs) == 0 { - return misuse(stderr, errors.New("--data-path is required")) + if in.noShipped && len(in.dirs) == 0 { + return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path")) } if in.list && len(in.paths) > 0 { return misuse(stderr, errors.New("--list takes no path")) @@ -178,10 +181,16 @@ func run(args []string, stdout, stderr io.Writer) int { } var opts []fejkdata.Option + if in.noShipped { + opts = append(opts, fejkdata.WithoutShippedData()) + } + for _, dir := range in.dirs { + opts = append(opts, fejkdata.WithDataPath(dir)) + } if in.seeded { opts = append(opts, fejkdata.WithSeed(in.seed)) } - f, err := fejkdata.New(in.dirs, opts...) + f, err := fejkdata.New(opts...) if err != nil { fmt.Fprintln(stderr, err) return 1 diff --git a/data.go b/data.go index a5b0403..0be4a02 100644 --- a/data.go +++ b/data.go @@ -3,29 +3,60 @@ package fejkdata import ( "encoding/json" "fmt" + "io/fs" "os" - "path/filepath" + "path" "strings" ) -// loadData loads every given directory into one namespace tree and returns its -// root children. A directory becomes a group; each *.json file in it compiles to -// a node keyed by its base name (address.json -> "address"); each subdirectory -// becomes a nested group, so folders turn into dot-path segments. Paths are -// merged left to right: matching groups merge by their children, and any other -// clash (a leaf, or a leaf-vs-group) is won by the last directory loaded. Once -// merged, linkRefs binds every {..path} reference against the final tree. -func loadData(paths []string) (map[string]node, error) { +// dataSource is one tree to load: an fs.FS and the directory in it to start from. +// label prefixes file names in errors; path, when set, is a directory on disk that +// must exist. +type dataSource struct { + fsys fs.FS + label string + path string + root string +} + +func (s dataSource) name(p string) string { + if s.label == "" { + return p + } + return path.Join(s.label, p) +} + +// loadData loads every source into one namespace tree and returns its root +// children. A directory becomes a group; each *.json file in it compiles to a node +// keyed by its base name (address.json -> "address"); each subdirectory becomes a +// nested group, so folders turn into dot-path segments. Sources merge left to right: +// matching groups merge by their children, and any other clash is won by the last +// source loaded. Once merged, linkRefs binds every {..path} reference against the +// final tree. +func loadData(sources []dataSource) (map[string]node, error) { root := map[string]node{} - for _, p := range paths { - g, err := loadDir(p) + for _, src := range sources { + if src.path != "" { + info, err := os.Stat(src.path) + if err != nil { + return nil, fmt.Errorf("%s: no such directory", src.path) + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", src.path) + } + } + dir := src.root + if dir == "" { + dir = "." + } + g, err := loadDir(src, dir) if err != nil { return nil, err } mergeChildren(root, g.children) } if len(root) == 0 { - return nil, fmt.Errorf("no .json data found in %v", paths) + return nil, fmt.Errorf("no .json data found") } if err := linkRefs(root); err != nil { return nil, err @@ -39,29 +70,22 @@ func loadData(paths []string) (map[string]node, error) { return root, nil } -// loadDir compiles one directory into a group. os.ReadDir yields entries sorted +// loadDir compiles one directory into a group. fs.ReadDir yields entries sorted // by name, so the tree is built deterministically. Empty subdirectories (no JSON // anywhere under them) are skipped rather than added as empty namespaces. -func loadDir(dir string) (*group, error) { - info, err := os.Stat(dir) +func loadDir(src dataSource, dir string) (*group, error) { + entries, err := fs.ReadDir(src.fsys, dir) if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, fmt.Errorf("%s is not a directory", dir) - } - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", src.name(dir), err) } g := &group{children: map[string]node{}} for _, e := range entries { if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data continue } - full := filepath.Join(dir, e.Name()) + full := path.Join(dir, e.Name()) if e.IsDir() { - child, err := loadDir(full) + child, err := loadDir(src, full) if err != nil { return nil, err } @@ -69,7 +93,7 @@ func loadDir(dir string) (*group, error) { continue } if err := checkName(e.Name()); err != nil { - return nil, fmt.Errorf("%s: folder %w", full, err) + return nil, fmt.Errorf("%s: folder %w", src.name(full), err) } g.children[e.Name()] = child continue @@ -79,19 +103,19 @@ func loadDir(dir string) (*group, error) { } name := strings.TrimSuffix(e.Name(), ".json") if err := checkName(name); err != nil { - return nil, fmt.Errorf("%s: category %w", full, err) + return nil, fmt.Errorf("%s: category %w", src.name(full), err) } - b, err := os.ReadFile(full) + b, err := fs.ReadFile(src.fsys, full) if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", src.name(full), err) } var raw any if err := json.Unmarshal(b, &raw); err != nil { - return nil, fmt.Errorf("%s: %w", full, err) + return nil, fmt.Errorf("%s: %w", src.name(full), err) } n, err := compile(raw) if err != nil { - return nil, fmt.Errorf("%s: %w", full, err) + return nil, fmt.Errorf("%s: %w", src.name(full), err) } g.children[name] = n } diff --git a/fejkdata.go b/fejkdata.go index e9f8319..def1fe6 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -1,78 +1,110 @@ // Package fejkdata generates fake data from recursive JSON templates. // -// Data lives in JSON on disk, not in Go. Point [New] at one or more data -// directories; folders and files become a dot-path namespace, then generate -// values by path: +// Data lives in JSON, not in Go. A generator starts from the shipped data set and +// layers any directories you add; folders and files become a dot-path namespace, +// then generate values by path: // -// f, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -// f.Fake("address") // "Storgatan 12\n234 56 Göteborg" -// f.Fake("address.locality") // "Göteborg" +// f, _ := fejkdata.New(fejkdata.WithSeed(42)) +// f.Fake("sv_SE.address") // "Storgatan 12\n234 56 Göteborg" +// f.Fake("sv_SE.address.locality") // "Göteborg" // -// A subdirectory is a namespace segment, so pointing at "./data" instead reaches -// a category as "sv_SE.address". Several directories are merged left to right; -// the last one wins on a name clash, so you can layer custom data over the -// built-ins. The JSON template format is documented in the README. -// -// A [Generator] is not safe for concurrent use; create one per goroutine. +// Several sources merge in order, the last winning a name clash, so custom data +// layers over the built-ins. The JSON template format is documented in the README. package fejkdata import ( crand "crypto/rand" + "embed" "encoding/binary" "fmt" + "io/fs" "math/rand/v2" + "os" "sort" + "sync" ) +//go:embed data +var shippedFS embed.FS + // Generator generates fake data from a loaded namespace tree. Create one with [New]. +// It is safe for concurrent use; a seeded sequence is reproducible only when drawn +// from one goroutine. type Generator struct { + mu sync.Mutex rand *session - categories map[string]node // root namespace: name -> compiled node tree + categories map[string]node } // session is one generator's mutable render state: the seeded rng plus the {seq()} -// counters. Scoping it to the Generator means sequences (and randomness) belong to -// that generator and reset when you create a new one. Embedding *rand.Rand makes a -// *session satisfy the rng interface the renderer draws from. +// counters. type session struct { *rand.Rand counters map[string]uint64 } -// next returns the next value (counting from 1) of the named {seq()} counter. func (s *session) next(key string) uint64 { s.counters[key]++ return s.counters[key] } type config struct { - seed uint64 - seeded bool + seed uint64 + seeded bool + shipped bool + sources []dataSource } // Option configures a [Generator]. type Option func(*config) -// WithSeed makes output reproducible: two generators with the same seed and locale +// WithSeed makes output reproducible: two generators with the same seed and data // emit identical sequences. func WithSeed(seed uint64) Option { return func(c *config) { c.seed, c.seeded = seed, true } } -// New builds a generator from one or more data directories (e.g. "./data/sv_SE"). -// Each JSON file becomes a category named after the file (address.json -> -// "address") and each subdirectory a namespace segment; directories are merged -// in order, the last winning a name clash. It errors on a missing directory, -// invalid JSON, or no data found. -func New(paths []string, opts ...Option) (*Generator, error) { - cats, err := loadData(paths) - if err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) +// WithDataPath layers a data directory over what is loaded before it. Repeat it to +// layer several; the last wins a name clash. +func WithDataPath(dir string) Option { + return func(c *config) { + c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, path: dir}) } - var c config +} + +// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own. +func WithDataFS(fsys fs.FS) Option { + return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) } +} + +// WithoutShippedData leaves the shipped data set out, so only the sources given +// with [WithDataPath] and [WithDataFS] load. +func WithoutShippedData() Option { + return func(c *config) { c.shipped = false } +} + +// New builds a generator from the shipped data set and the options' sources, merged +// in order with the last winning a name clash. Each JSON file becomes a category +// named after the file (address.json -> "address") and each subdirectory a +// namespace segment. It errors on a missing directory, invalid JSON, invalid data, +// or no data at all. +func New(opts ...Option) (*Generator, error) { + c := config{shipped: true} for _, opt := range opts { opt(&c) } + var sources []dataSource + if c.shipped { + sources = append(sources, dataSource{fsys: shippedFS, root: "data"}) + } + sources = append(sources, c.sources...) + if len(sources) == 0 { + return nil, fmt.Errorf("fejkdata: no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") + } + cats, err := loadData(sources) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } return &Generator{rand: newRand(c.seed, c.seeded), categories: cats}, nil } diff --git a/render.go b/render.go index d1985c2..13ca4bd 100644 --- a/render.go +++ b/render.go @@ -18,6 +18,8 @@ type rng interface { // e.g. "sv_SE.address" or "sv_SE.address.street". Choices along the way are // resolved at random. A path naming a folder (no value of its own) is an error. func (f *Generator) Fake(path string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) if err != nil { return "", fmt.Errorf("fejkdata: %s: %w", path, err) -- 2.52.0 From 99d8aa0fb72aa296bea5e1bfabdf6dbe1df33979 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:26:05 +0200 Subject: [PATCH 05/38] Tests for literal format text, brace escapes and the class builtins --- bench_test.go | 10 +-- bound_test.go | 4 +- builtins_test.go | 19 ++++- edge_test.go | 25 +++--- template_stability_test.go | 6 +- template_test.go | 153 ++++++++++++++++++++++++------------- 6 files changed, 137 insertions(+), 80 deletions(-) diff --git a/bench_test.go b/bench_test.go index 6022f31..0b850ed 100644 --- a/bench_test.go +++ b/bench_test.go @@ -48,7 +48,7 @@ func BenchmarkCalc(b *testing.B) { } func BenchmarkLongLiteral(b *testing.B) { - dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) V#ALUES (#1#2#3, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) + dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) benchPath(b, dir, "sql") } @@ -62,14 +62,14 @@ func BenchmarkRepeat(b *testing.B) { // two independent fields. func BenchmarkBound(b *testing.B) { dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"#100 00"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"#5#7#3 00"}}]}`) + {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"1{digits(2)} {digits(2)}"}}, + {"format":"{locality}","locality":"Tranås","postal-code":{"format":"573 {digits(2)}"}}]}`) benchPath(b, dir, "addr") } func BenchmarkUnbound(b *testing.B) { dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}", - "postal-code":[{"format":"#100 00"},{"format":"#5#7#3 00"}], + "postal-code":[{"format":"1{digits(2)} {digits(2)}"},{"format":"573 {digits(2)}"}], "locality":["Stockholm","Tranås"]}`) benchPath(b, dir, "addr") } @@ -78,7 +78,7 @@ func BenchmarkUnbound(b *testing.B) { // depth question is about. func BenchmarkBoundDeep(b *testing.B) { dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":[ - {"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":{"format":"#100 00"}}}]}`) + {"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":{"format":"1{digits(2)} {digits(2)}"}}}]}`) benchPath(b, dir, "addr") } diff --git a/bound_test.go b/bound_test.go index 36ea30c..a209949 100644 --- a/bound_test.go +++ b/bound_test.go @@ -16,8 +16,8 @@ import ( // swedishPlaces is a two-variant sibling whose variants pair a locality with the // postal-code prefix that really belongs to it. const swedishPlaces = `{"format":"%s","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"#100 00"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"#5#7#3 00"}} + {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"1{digits(2)} {digits(2)}"}}, + {"format":"{locality}","locality":"Tranås","postal-code":{"format":"573 {digits(2)}"}} ]}` // agree reports whether a rendered "postcode locality" pair is a real pairing. diff --git a/builtins_test.go b/builtins_test.go index fcac338..13af539 100644 --- a/builtins_test.go +++ b/builtins_test.go @@ -83,9 +83,9 @@ func TestBuiltinBase64(t *testing.T) { // hand-computed check, like the luhn test. mod-11 emits X when it would be 10. func TestBuiltinChecksums(t *testing.T) { cases := map[string]string{ - `{"format":"#1#2#3#4#5#6#7#8{mod11()}"}`: "123456785", // weights 2..7 from the right - `{"format":"#6{mod11()}"}`: "6X", // remainder 10 -> X - `{"format":"#4#0#0#6#3#8#1#3#3#3#9#3{ean()}"}`: "4006381333931", // EAN-13 (= ISBN-13) check digit + `{"format":"12345678{mod11()}"}`: "123456785", // weights 2..7 from the right + `{"format":"6{mod11()}"}`: "6X", // remainder 10 -> X + `{"format":"400638133393{ean()}"}`: "4006381333931", // EAN-13 (= ISBN-13) check digit } f := engine(1) for tmpl, want := range cases { @@ -217,3 +217,16 @@ func mustPanic(t *testing.T, name string, call func()) { }() call() } + +func TestClassBuiltinArgs(t *testing.T) { + for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} { + if _, err := compile(parse(t, bad)); err == nil { + t.Errorf("compile(%s) = nil error, want the arg rejected", bad) + } + } + for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v", ok, err) + } + } +} diff --git a/edge_test.go b/edge_test.go index 223e31b..4103385 100644 --- a/edge_test.go +++ b/edge_test.go @@ -13,15 +13,14 @@ import ( // --- format string edge cases --- -func TestEscapeEdgeCases(t *testing.T) { +func TestHashIsLiteral(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":""}`: "", // empty format - `{"format":"#"}`: "#", // trailing escape is a literal # - `{"format":"##"}`: "#", // escaped hash - `{"format":"#0#1#A#a"}`: "01Aa", // escaped class chars stay literal - `{"format":"#{x#}"}`: "{x}", // escaping braces disables tokens - `{"format":"x}y"}`: "x}y", // an unmatched } is literal (x, y aren't classes) + `{"format":"","x":["v"]}`: "", + `{"format":"#","x":["v"]}`: "#", + `{"format":"##","x":["v"]}`: "##", + `{"format":"#0#1#A#a","x":["v"]}`: "#0#1#A#a", + `{"format":"#{x}","x":["v"]}`: "#v", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -33,7 +32,7 @@ func TestEscapeEdgeCases(t *testing.T) { func TestMultibyteFormat(t *testing.T) { // Scanning is rune-aware: multibyte literals coexist with class chars and // tokens without corrupting indices. - got := mustRender(t, engine(2), `{"format":"Öster{x}-0å","x":["väg"]}`) + got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":["väg"]}`) if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { t.Fatalf("multibyte format = %q", got) } @@ -224,11 +223,11 @@ func TestDeepDottedPath(t *testing.T) { } } -func TestDescendIntoLiteralErrors(t *testing.T) { +func TestDescendIntoStringErrors(t *testing.T) { f := engine(1) f.categories = map[string]node{"greeting": compiled(t, `["hej"]`)} - if _, err := f.Fake("greeting.extra"); err == nil { - t.Fatal("Fake(greeting.extra) = nil error, want descend-into-literal error") + if _, err := f.Fake("greeting.extra"); err == nil || !strings.Contains(err.Error(), `no field "extra"`) { + t.Fatalf("Fake(greeting.extra) = %v, want a no-field error", err) } } @@ -303,8 +302,8 @@ func TestMissingFieldNamesItself(t *testing.T) { func TestCategoryRootShapes(t *testing.T) { dir := writeData(t, map[string]string{ - "obj": `{"format":"00"}`, // object root - "lit": `"hello"`, // bare-string root + "obj": `{"format":"{digits(2)}"}`, // object root + "lit": `"hello"`, // bare-string root }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { diff --git a/template_stability_test.go b/template_stability_test.go index cf16d86..67dc7f1 100644 --- a/template_stability_test.go +++ b/template_stability_test.go @@ -9,10 +9,10 @@ func TestSeededOutputIsStable(t *testing.T) { dir := writeData(t, map[string]string{ "alt": `{"format":"{a|b}","a":["A"],"b":["B"]}`, "calc": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, - "classes": `{"format":"00-11-AA-aa"}`, - "escapes": `{"format":"#0#1#A#a##{x}","x":["!"]}`, + "classes": `{"format":"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"}`, + "escapes": `{"format":"01Aa#{x}","x":["!"]}`, "funcs": `{"format":"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"}`, - "nested": `{"format":"{outer}","outer":[{"format":"{inner}-00","inner":["i"]}]}`, + "nested": `{"format":"{outer}","outer":[{"format":"{inner}-{digits(2)}","inner":["i"]}]}`, "ref": `{"format":"see {..alt}"}`, "repeat": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`, "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":["012345678901234"],"e":["123456789012"],"m":["12345678"]}`, diff --git a/template_test.go b/template_test.go index ee56174..33887a8 100644 --- a/template_test.go +++ b/template_test.go @@ -35,19 +35,33 @@ func mustRender(t *testing.T, f *Generator, s string) string { return render(f.rand, compiled(t, s)) } -func TestLiteralStringIsVerbatim(t *testing.T) { - // A bare string is a literal, never formatted: 'a'/'A' must survive. - if got := mustRender(t, engine(1), `"Malmö"`); got != "Malmö" { - t.Fatalf("literal = %q, want Malmö", got) +func TestStringIsAFormat(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `"Malmö"`: "Malmö", + `"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0", + `"{{x}}"`: "{x}", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } + if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) { + t.Errorf(`render("{digits(3)}") = %q, want three digits`, got) + } + if _, err := compile(parse(t, `"{x}"`)); err == nil || !strings.Contains(err.Error(), `no field "x"`) { + t.Errorf(`compile("{x}") = %v, want a no-field error`, err) } } -func TestCharacterClasses(t *testing.T) { +func TestClassBuiltins(t *testing.T) { cases := map[string]*regexp.Regexp{ - `{"format":"0"}`: regexp.MustCompile(`^[0-9]$`), - `{"format":"1"}`: regexp.MustCompile(`^[1-9]$`), - `{"format":"A"}`: regexp.MustCompile(`^[A-Z]$`), - `{"format":"a"}`: regexp.MustCompile(`^[a-z]$`), + `"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`), + `"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`), + `"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`), + `"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`), + `"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`), + `"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`), } f := engine(7) for tmpl, re := range cases { @@ -59,10 +73,34 @@ func TestCharacterClasses(t *testing.T) { } } -func TestEscapeAndLiteralChars(t *testing.T) { - // '#' escapes the next char; non-class chars (7, x, -) are literal. - if got := mustRender(t, engine(1), `{"format":"#0#1#A#a## x7-z"}`); got != "01Aa# x7-z" { - t.Fatalf("escape = %q, want \"01Aa# x7-z\"", got) +func TestTextIsLiteral(t *testing.T) { + if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":["A"]}`); got != "100 Main St #1 A" { + t.Fatalf("text = %q, want it verbatim", got) + } +} + +func TestBraceEscapes(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `{"format":"{{","x":["v"]}`: "{", + `{"format":"}}","x":["v"]}`: "}", + `{"format":"{{x}}","x":["v"]}`: "{x}", + `{"format":"{{{x}}}","x":["v"]}`: "{v}", + `{"format":"a{{{{b}}}}","x":["v"]}`: "a{{b}}", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } + for src, want := range map[string]string{ + `{"format":"x}y"}`: "}}", + `{"format":"}"}`: "}}", + `{"format":"{a{b}","a":["Q"]}`: "'{'", + `{"format":"{x"}`: "unterminated", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) + } } } @@ -118,7 +156,7 @@ func TestRepeatRendersFormatNTimes(t *testing.T) { f, re := engine(7), regexp.MustCompile(`^[0-9]{4}$`) seen := map[string]bool{} for i := 0; i < 50; i++ { - got := mustRender(t, f, `{"format":"0","repeat":4}`) + got := mustRender(t, f, `{"format":"{digits(1)}","repeat":4}`) if !re.MatchString(got) { t.Fatalf("repeat-4 = %q, want 4 digits", got) } @@ -135,9 +173,9 @@ func TestFunctionTokenLuhn(t *testing.T) { // buffer, so a value is never re-rendered. Bodies are escaped to fix input. f := engine(1) cases := map[string]string{ - `{"format":"#8#1#1#2#1#8#9#8#7{luhn()}"}`: "8112189876", // personnummer body - `{"format":"#7#9#9#2#7#3#9#8#7#1{luhn()}"}`: "79927398713", // classic Luhn vector - `{"format":"#8#1#1#2#1#8-#9#8#7{luhn()}"}`: "811218-9876", // '-' skipped, kept + `{"format":"811218987{luhn()}"}`: "8112189876", // personnummer body + `{"format":"7992739871{luhn()}"}`: "79927398713", // classic Luhn vector + `{"format":"811218-987{luhn()}"}`: "811218-9876", // '-' skipped, kept `{"format":"{n}{luhn()}","n":["811218987"]}`: "8112189876", // over a rendered token } for tmpl, want := range cases { @@ -162,45 +200,51 @@ func TestCompileErrors(t *testing.T) { // Every structural problem is caught up front, at compile/New time, never // deferred to a random render that happens to hit the bad branch. for _, bad := range []string{ - `{"x":["Q"]}`, // object without "format" - `{"format":"{y}","x":1}`, // a field is a bare number - `[1, 2]`, // a choice of numbers - `5`, // unsupported node type - `[]`, // empty choice - `{"format":"{x"}`, // unterminated brace - `{"format":"{y}","x":["Q"]}`, // token names a missing field - `{"format":"{}"}`, // empty token name - `{"format":"{a|}","a":["Q"]}`, // empty alternation segment + `{"x":["Q"]}`, // object without "format" + `{"format":"{y}","x":1}`, // a field is a bare number + `[1, 2]`, // a choice of numbers + `5`, // unsupported node type + `[]`, // empty choice + `{"format":"{x"}`, // unterminated brace + `{"format":"x}y"}`, // a lone } must be written }} + `{"format":"{a{b}","a":["Q"]}`, // a brace inside a token + `"{x}"`, // a bare string has no fields to name + `{"format":"{digits(0)}"}`, // count must be positive + `{"format":"{upper(x)}"}`, // count must be an integer + `{"format":"{lower()}"}`, // wrong arity + `{"format":"{y}","x":["Q"]}`, // token names a missing field + `{"format":"{}"}`, // empty token name + `{"format":"{a|}","a":["Q"]}`, // empty alternation segment `[{"format":"A","weight":-1},{"format":"B"}]`, // negative weight `[{"format":"A","weight":0},{"format":"B","weight":0}]`, // weights sum to zero `[{"format":"A","weight":1e308},{"format":"B","weight":1e308}]`, // weights overflow to +Inf `[{"format":"A","weight":"heavy"}]`, // non-numeric weight - `{"format":"x","repeat":0}`, // repeat below 1 - `{"format":"x","repeat":-2}`, // negative repeat - `{"format":"x","repeat":1.5}`, // non-integer repeat - `{"format":"x","repeat":"two"}`, // non-numeric repeat - `{"format":"x","repeat":2,"separator":5}`, // non-string separator - `{"format":"{nope()}"}`, // unknown function - `{"format":"{luhn(x)}"}`, // function given args it takes none of - `{"format":"{luhn(}"}`, // malformed function token - `{"format":"{int(1)}"}`, // wrong arity - `{"format":"{int(a,b)}"}`, // non-integer args - `{"format":"{int(5,1)}"}`, // min > max - `{"format":"{hex(0)}"}`, // count must be positive - `{"format":"{nanoid(-1)}"}`, // negative count - `{"format":"{base64(0)}"}`, // count must be positive - `{"format":"{float(1,2)}"}`, // wrong arity - `{"format":"{float(1,2,-1)}"}`, // negative decimals - `{"format":"{iban(US)}"}`, // unsupported country - `{"format":"{seq(a,b)}"}`, // seq takes at most one name - `{"format":"{calc()}"}`, // calc needs an expression - `{"format":"{calc(1 +)}"}`, // dangling operator - `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis - `{"format":"{calc(1 2)}"}`, // two operands, no operator - `{"format":"{calc(price)}"}`, // operand names no field - `{"format":"{calc(1, 2, 3)}"}`, // too many args - `{"format":"{calc(1, x)}"}`, // decimals arg not an integer - `{"format":"{calc(1, -1)}"}`, // decimals negative + `{"format":"x","repeat":0}`, // repeat below 1 + `{"format":"x","repeat":-2}`, // negative repeat + `{"format":"x","repeat":1.5}`, // non-integer repeat + `{"format":"x","repeat":"two"}`, // non-numeric repeat + `{"format":"x","repeat":2,"separator":5}`, // non-string separator + `{"format":"{nope()}"}`, // unknown function + `{"format":"{luhn(x)}"}`, // function given args it takes none of + `{"format":"{luhn(}"}`, // malformed function token + `{"format":"{int(1)}"}`, // wrong arity + `{"format":"{int(a,b)}"}`, // non-integer args + `{"format":"{int(5,1)}"}`, // min > max + `{"format":"{hex(0)}"}`, // count must be positive + `{"format":"{nanoid(-1)}"}`, // negative count + `{"format":"{base64(0)}"}`, // count must be positive + `{"format":"{float(1,2)}"}`, // wrong arity + `{"format":"{float(1,2,-1)}"}`, // negative decimals + `{"format":"{iban(US)}"}`, // unsupported country + `{"format":"{seq(a,b)}"}`, // seq takes at most one name + `{"format":"{calc()}"}`, // calc needs an expression + `{"format":"{calc(1 +)}"}`, // dangling operator + `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis + `{"format":"{calc(1 2)}"}`, // two operands, no operator + `{"format":"{calc(price)}"}`, // operand names no field + `{"format":"{calc(1, 2, 3)}"}`, // too many args + `{"format":"{calc(1, x)}"}`, // decimals arg not an integer + `{"format":"{calc(1, -1)}"}`, // decimals negative } { if _, err := compile(parse(t, bad)); err == nil { t.Errorf("compile(%s) = nil error, want error", bad) @@ -235,8 +279,9 @@ func TestGrowIsALowerBound(t *testing.T) { for _, format := range []string{ "", "plain literal", - "00-11-AA-aa", - "#0#1#A#a## literal", + "{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}", + "01Aa# literal", + "{{}} {{{x}}}", "Ö dag åäö 日本語", "{x}{x}{x}", "{hex(8)}-{int(10,99)}-{nanoid(5)}", -- 2.52.0 From ff4b243148951b6722e7d4b0387ffc667fab1758 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:37:21 +0200 Subject: [PATCH 06/38] Format text is literal; class runs are {digits(n)} {upper(n)} {lower(n)}; a string is a format --- README.md | 49 +++++++++---------- builtins.go | 62 ++++++++++------------- calc.go | 10 ---- data/en_US/address.json | 12 ++--- data/en_US/color.json | 2 +- data/en_US/date.json | 12 ++--- data/en_US/ip.json | 2 +- data/en_US/phone.json | 8 +-- data/en_US/price.json | 14 +++--- data/en_US/ssn.json | 2 +- data/en_US/time.json | 2 +- data/en_US/version.json | 2 +- data/misc/creditcard.json | 6 +-- data/sv_SE/address.json | 14 +++--- data/sv_SE/color.json | 2 +- data/sv_SE/date.json | 12 ++--- data/sv_SE/ip.json | 2 +- data/sv_SE/phone.json | 12 ++--- data/sv_SE/price.json | 12 ++--- data/sv_SE/ssn.json | 2 +- data/sv_SE/time.json | 6 +-- data/sv_SE/version.json | 2 +- fejkdata.go | 2 - format-migration/convert.py | 72 +++++++++++++++++++++++++++ node.go | 60 +++++++++++++++-------- reference.go | 20 +++++--- render.go | 24 ++------- template.go | 98 ++++++++++++++++--------------------- template_test.go | 52 ++++++++++---------- 29 files changed, 308 insertions(+), 267 deletions(-) create mode 100755 format-migration/convert.py diff --git a/README.md b/README.md index e189c01..a65c822 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ own and render it — no code change. Save this as `mydata/sql.json`: ```json { - "format": "INSERT INTO users V#ALUES({sql-username});", + "format": "INSERT INTO users VALUES({sql-username});", "sql-username": { "format": "'{username}'", "repeat": 3, @@ -74,9 +74,7 @@ own and render it — no code change. Save this as `mydata/sql.json`: ``` `sql-username` renders `'{username}'` `repeat` times and joins the results with -the `),(` separator; the outer `V#ALUES(…)` wraps that into one valid row list. -(`#A` escapes the literal `A`, which a format string would otherwise read as a -letter token — see [Data format](#data-format).) +the `),(` separator; the outer `VALUES(…)` wraps that into one valid row list. ```sh fejkdata --seed 1 --data-path ./mydata sql @@ -160,7 +158,7 @@ when drawn from one goroutine. The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) plus a locale-neutral `misc` folder — is embedded in the library and the CLI, so both work with no data on disk. Layer your own directories over it; a category or -folder name must not use `.`, `|`, `(` or `}` (see [Data format](#data-format)), +folder name must not use `.`, `|`, `(`, `{` or `}` (see [Data format](#data-format)), and dot-prefixed entries are skipped, so a data directory can also be a checkout. A directory is just a namespace. Each JSON file is a category named after the @@ -206,7 +204,7 @@ Every value is a **node**, one of three shapes, nestable without limit: | Node | JSON | Meaning | |------|------|---------| -| literal | `"Malmö"` | emitted verbatim — never formatted | +| string | `"Malmö"` | a format with no fields: text, or tokens that need none (`"{digits(3)}"`, `"{..path}"`) | | choice | `["a", "b", …]` | one element, picked at random | | template | `{"format": "…", …}` | a format string plus the named sub-nodes it references | @@ -215,9 +213,9 @@ within a choice: ```json [ - { "format": "#070-000 00 00", "weight": 10 }, - { "format": "#01-000 00 00" }, - { "format": "#010-000 00 00" } + { "format": "070-{digits(3)} {digits(2)} {digits(2)}", "weight": 10 }, + { "format": "01-{digits(3)} {digits(2)} {digits(2)}" }, + { "format": "010-{digits(3)} {digits(2)} {digits(2)}" } ] ``` @@ -242,8 +240,8 @@ is a field**. So write `seperator` and you get a field by that name while the option stays unset. `New` rejects an option that cannot take effect — a `separator` without a `repeat` above 1, a `weight` outside a choice — and a name using a character the grammars reserve: `.` separates the segments of a path, `|` -the arms of a token, `(` opens a function call and `}` ends the token, so a name -carrying one is a name no format could ever spell. An empty name goes the same +the arms of a token, `(` opens a function call and `{` `}` delimit the token, so a +name carrying one is a name no format could ever spell. An empty name goes the same way — it is no path segment at all, so `List` never offers it. That holds for a category, a folder and a field alike. @@ -254,7 +252,7 @@ argument counts are rejected at `New`. This is what makes a generated Swedish personnummer valid — its last digit is a Luhn checksum over the nine before it: ```json -{ "format": "00{mmdd}-000{luhn()}", "mmdd": [ … ] } +{ "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": [ … ] } ``` renders e.g. `811218-987`, then `{luhn()}` appends `6` → `811218-9876`. Place it @@ -264,7 +262,7 @@ form prefixes the century outside the checksummed core: ```json { "format": "{century}{core}", "century": ["19", "20"], - "core": { "format": "00{mmdd}-000{luhn()}", "mmdd": [ … ] } } + "core": { "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": [ … ] } } ``` A function must be deterministic (no wall-clock), so a seeded generator stays @@ -289,6 +287,9 @@ fat-fingered `hex(2000000000)` can't try to allocate gigabytes at render. | `{ulid()}` | sample | ULID, 26-char Crockford base32 | | `{nanoid(n)}` | sample | URL-safe Nano ID, `n` chars | | `{hex(n)}` | sample | `n` lowercase hex digits | +| `{digits(n)}` | sample | `n` digits 0–9 | +| `{upper(n)}` | sample | `n` letters A–Z | +| `{lower(n)}` | sample | `n` letters a–z | | `{base64(n)}` | sample | `n` random bytes, base64 | | `{int(min,max)}` | sample | uniform integer in `[min, max]` | | `{float(min,max,dp)}` | sample | number in `[min, max]` with `dp` decimals | @@ -349,8 +350,8 @@ locality and the postal code that really covers it, stay together: ```json { "format": "{street} {number}\n{place.postal-code} {place.locality}", "place": [ - { "format": "{locality}", "locality": "Stockholm", "postal-code": { "format": "#100 00" }, "weight": 975 }, - { "format": "{locality}", "locality": "Tranås", "postal-code": { "format": "#5#7#3 00" }, "weight": 18 } + { "format": "{locality}", "locality": "Stockholm", "postal-code": "1{digits(2)} {digits(2)}", "weight": 975 }, + { "format": "{locality}", "locality": "Tranås", "postal-code": "573 {digits(2)}", "weight": 18 } ] } ``` @@ -411,31 +412,25 @@ carries "postal-code"; all carry [locality] The sub-fields stay addressable on their own — `Fake("address.place.locality")` renders, and `List` advertises it. -**Format string.** Every character is literal except: +**Format string.** Every character is literal except a `{…}` token: | Token | Expands to | |-------|-----------| -| `0` | digit 0–9 | -| `1` | digit 1–9 | -| `A` | letter A–Z | -| `a` | letter a–z | -| `#` | escape — the next char is literal (`#0` → `0`, `##` → `#`) | | `{name}` | render the sibling field `name` | | `{name.field}` | render `field` of one draw of the sibling `name` (see **Correlated fields**) | | `{name()}` | call a built-in function (see **Functions**) | | `{..path}` | render the node at a dot path from the data root (see **References**) | +| `{{`, `}}` | a literal `{` or `}` | `{a|b}` renders one of the sibling fields `a` or `b`, chosen at random; an arm may be a `{..path}` reference too (`{name|..en_US.person}`). The arms are picked evenly and must differ — `{a|a|b}` would skew the odds, which is what `weight` is for, so a repeated arm is a load error. -Inside a `format`, `0 1 A a` are **always** character classes — so a fixed `0`, -`1`, `A` or `a` must be escaped (`#1`, `#A`) or it becomes random. A format of -`100 Main St` renders e.g. `506 Mdin St` — the `1`, `0`, `0` and `a` were random, -the `M`, `in` and `St` were not. A half-fixed string is the trap: `555-0000` -keeps `555-` and randomises the last four digits. For a value with no -tokens at all, use a bare string node (`"100 Main St"`), emitted verbatim. +Text means what it says: `100 Main St` renders `100 Main St`. Random characters +come from the sample functions — `{digits(3)}`, `{upper(1)}`, `{lower(2)}`, +`{int(10,99)}` — so a phone pattern is `070-{digits(3)} {digits(2)} {digits(2)}`. +A lone `}` is a load error naming `}}`. **Putting it together** (`person.json`): diff --git a/builtins.go b/builtins.go index 5482665..eca7bba 100644 --- a/builtins.go +++ b/builtins.go @@ -16,27 +16,21 @@ const ( maxDecimals = 1024 ) -// builtins is the registry of {name(args)} functions. Derivations read the -// digits emitted so far in the current expansion (luhn, mod11, ean — place them -// after their payload); samples read only the rng (uuid, ulid, ...). All -// must stay pure over (rng, emitted, args) so seeded output is reproducible — a -// time-based id (uuid v7, ulid) draws its timestamp from the rng, not the wall -// clock. Add a builtin only for what data can't express: a random v4 UUID and a -// 24-hex ObjectID both ship as data, so the uuid builtin is v7. +// builtins is the registry of {name(args)} functions. Derivations read the digits +// emitted so far in the current expansion (place them after their payload); +// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp +// from the rng, not the wall clock, so seeded output stays reproducible. var builtins = map[string]builtin{ - "luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })}, - "mod11": {arity: 0, prep: derive(mod11Check)}, - "ean": {arity: 0, prep: derive(eanCheck)}, - "uuid": {arity: 0, prep: sample(uuidV7)}, - "ulid": {arity: 0, prep: sample(ulid)}, - "nanoid": {arity: 1, check: posIntArg, prep: func(a []string) callFn { - n := atoi(a[0]) - return func(s *session, _ string, _ []string) string { return nanoid(s, n) } - }}, - "hex": {arity: 1, check: posIntArg, prep: func(a []string) callFn { - n := atoi(a[0]) - return func(s *session, _ string, _ []string) string { return randHex(s, n) } - }}, + "luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })}, + "mod11": {arity: 0, prep: derive(mod11Check)}, + "ean": {arity: 0, prep: derive(eanCheck)}, + "uuid": {arity: 0, prep: sample(uuidV7)}, + "ulid": {arity: 0, prep: sample(ulid)}, + "nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)}, + "hex": {arity: 1, check: posIntArg, prep: chars(hexDigits)}, + "digits": {arity: 1, check: posIntArg, prep: chars("0123456789")}, + "upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")}, + "lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")}, "base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn { n := atoi(a[0]) return func(s *session, _ string, _ []string) string { @@ -74,9 +68,9 @@ var builtins = map[string]builtin{ }}, } -// derive and sample are the two argument-free builtin shapes the README names: a -// derivation reads the output emitted so far, a sample reads only the rng. Each -// lifts that one function into the prep every registry entry supplies. +// derive and sample are the two argument-free builtin shapes: a derivation reads +// the output emitted so far, a sample reads only the rng. chars is the shape of a +// sample of n characters drawn from an alphabet. func derive(f func(emitted string) string) func([]string) callFn { return func([]string) callFn { return func(_ *session, emitted string, _ []string) string { return f(emitted) } @@ -89,6 +83,13 @@ func sample(f func(rng) string) func([]string) callFn { } } +func chars(alphabet string) func([]string) callFn { + return func(a []string) callFn { + n := atoi(a[0]) + return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) } + } +} + const hexDigits = "0123456789abcdef" // atoi parses an arg a builtin's check already validated. It panics rather than @@ -119,10 +120,10 @@ func randBytes(r rng, n int) []byte { return b } -func randHex(r rng, n int) string { +func randChars(r rng, n int, alphabet string) string { b := make([]byte, n) for i := range b { - b[i] = hexDigits[r.IntN(16)] + b[i] = alphabet[r.IntN(len(alphabet))] } return string(b) } @@ -182,18 +183,9 @@ func seqArg(_ map[string]node, a []string) error { return nil } -// nanoidAlphabet is the 64-char URL-safe set Nano IDs use (order is irrelevant -// to the uniform pick). +// nanoidAlphabet is the 64-char URL-safe set Nano IDs use. const nanoidAlphabet = "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -func nanoid(r rng, n int) string { - b := make([]byte, n) - for i := range b { - b[i] = nanoidAlphabet[r.IntN(len(nanoidAlphabet))] - } - return string(b) -} - // uuidV7 builds an RFC 9562 v7 UUID. The 48-bit timestamp field is drawn from // the rng (not the clock) to stay reproducible, then the version (7) and variant // (10) bits are forced; the rest is random. diff --git a/calc.go b/calc.go index 038fc3e..f0c5e4b 100644 --- a/calc.go +++ b/calc.go @@ -8,16 +8,6 @@ import ( "unicode" ) -// calc is the {calc(expr[, dp])} token: an arithmetic expression over number -// literals and sibling-field names, with + - * /, unary minus and parentheses. It is -// a registry builtin like any other {name(args)} function — the one that names -// operands, which expand reads for it (once per expansion, so the value computed is -// the value the format showed) and hands over as strings. The value prints in -// minimal decimal form, or rounded to dp when given. An operand that isn't a number -// becomes NaN, which propagates and prints as "NaN" — visible, never a render error. -// The expression is parsed once at New into an AST every render shares and none -// mutates, so render cannot fail and must not carry per-render state. - // calcNode is a parsed expression node. It evaluates over the operand values expand // read, so the evaluator touches neither the rng nor the node tree. type calcNode interface { diff --git a/data/en_US/address.json b/data/en_US/address.json index 0fd122e..bf71c64 100644 --- a/data/en_US/address.json +++ b/data/en_US/address.json @@ -9,16 +9,16 @@ } ], "street-number": [ - { "format": "10" }, - { "format": "100", "weight": 2 }, - { "format": "1000", "weight": 0.5 }, - { "format": "10100", "weight": 0.3 } + { "format": "{int(10,99)}" }, + { "format": "{int(100,999)}", "weight": 2 }, + { "format": "{int(1000,9999)}", "weight": 0.5 }, + { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 } ], "locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"], "region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"], "postal-code": [ - { "format": "10000" }, - { "format": "10000-0000", "weight": 0.3 } + { "format": "{int(10000,99999)}" }, + { "format": "{int(10000,99999)}-{digits(4)}", "weight": 0.3 } ] } ] diff --git a/data/en_US/color.json b/data/en_US/color.json index d472ec4..c6cf983 100644 --- a/data/en_US/color.json +++ b/data/en_US/color.json @@ -1,6 +1,6 @@ [ { - "format": "##{x}{x}{x}{x}{x}{x}", + "format": "#{x}{x}{x}{x}{x}{x}", "x": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], "weight": 2 }, diff --git a/data/en_US/date.json b/data/en_US/date.json index afee0f8..05e6167 100644 --- a/data/en_US/date.json +++ b/data/en_US/date.json @@ -4,12 +4,12 @@ "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], "year": [ - { "format": "#1970" }, - { "format": "#1980" }, - { "format": "#1990", "weight": 2 }, - { "format": "2#0#00", "weight": 3 }, - { "format": "2#0#10", "weight": 3 }, - { "format": "2#020", "weight": 2 } + { "format": "197{digits(1)}" }, + { "format": "198{digits(1)}" }, + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } ] } ] diff --git a/data/en_US/ip.json b/data/en_US/ip.json index 2f071a7..e16c78a 100644 --- a/data/en_US/ip.json +++ b/data/en_US/ip.json @@ -2,7 +2,7 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "0", "weight": 3 }, { "format": "10", "weight": 4 }, { "format": "#100", "weight": 2 }] + "o": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}", "weight": 4 }, { "format": "1{digits(2)}", "weight": 2 }] }, { "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" diff --git a/data/en_US/phone.json b/data/en_US/phone.json index 4866991..20f65ab 100644 --- a/data/en_US/phone.json +++ b/data/en_US/phone.json @@ -3,13 +3,13 @@ "format": "({area}) {exch}-{line}", "weight": 2, "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "100" }], - "line": [{ "format": "0000" }] + "exch": [{ "format": "{int(100,999)}" }], + "line": [{ "format": "{digits(4)}" }] }, { "format": "{area}-{exch}-{line}", "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "100" }], - "line": [{ "format": "0000" }] + "exch": [{ "format": "{int(100,999)}" }], + "line": [{ "format": "{digits(4)}" }] } ] diff --git a/data/en_US/price.json b/data/en_US/price.json index 25197d2..84c3c87 100644 --- a/data/en_US/price.json +++ b/data/en_US/price.json @@ -2,15 +2,15 @@ { "format": "${amt}.{cents}", "amt": [ - { "format": "1", "weight": 2 }, - { "format": "10", "weight": 4 }, - { "format": "100", "weight": 3 }, - { "format": "1#,000", "weight": 1 }, - { "format": "10#,000", "weight": 0.4 }, - { "format": "100#,000", "weight": 0.1 } + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + { "format": "{int(1,9)},{digits(3)}", "weight": 1 }, + { "format": "{int(10,99)},{digits(3)}", "weight": 0.4 }, + { "format": "{int(100,999)},{digits(3)}", "weight": 0.1 } ], "cents": [ - { "format": "00", "weight": 3 }, + { "format": "{digits(2)}", "weight": 3 }, "49", "95", "99" diff --git a/data/en_US/ssn.json b/data/en_US/ssn.json index d7ff95d..b114ca6 100644 --- a/data/en_US/ssn.json +++ b/data/en_US/ssn.json @@ -1,3 +1,3 @@ [ - { "format": "100-00-0000" } + { "format": "{int(100,999)}-{digits(2)}-{digits(4)}" } ] diff --git a/data/en_US/time.json b/data/en_US/time.json index 0a04213..09fd32d 100644 --- a/data/en_US/time.json +++ b/data/en_US/time.json @@ -2,7 +2,7 @@ { "format": "{hour}:{minute} {ampm}", "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], - "minute": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], + "minute": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], "ampm": ["AM", "PM"] } ] diff --git a/data/en_US/version.json b/data/en_US/version.json index 65bc1cc..f0806d4 100644 --- a/data/en_US/version.json +++ b/data/en_US/version.json @@ -1,7 +1,7 @@ [ { "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "0", "weight": 3 }, { "format": "10" }], + "n": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}" }], "pre": ["", { "format": "v", "weight": 0.4 }], "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] } diff --git a/data/misc/creditcard.json b/data/misc/creditcard.json index a9d7447..f2d1517 100644 --- a/data/misc/creditcard.json +++ b/data/misc/creditcard.json @@ -1,5 +1,5 @@ [ - { "format": "4{d}{luhn()}", "d": { "format": "0", "repeat": 14 }, "weight": 4 }, - { "format": "5{m}{d}{luhn()}", "m": ["1", "2", "3", "4", "5"], "d": { "format": "0", "repeat": 13 }, "weight": 3 }, - { "format": "3{m}{d}{luhn()}", "m": ["4", "7"], "d": { "format": "0", "repeat": 12 }, "weight": 1 } + { "format": "4{d}{luhn()}", "d": { "format": "{digits(1)}", "repeat": 14 }, "weight": 4 }, + { "format": "5{m}{d}{luhn()}", "m": ["1", "2", "3", "4", "5"], "d": { "format": "{digits(1)}", "repeat": 13 }, "weight": 3 }, + { "format": "3{m}{d}{luhn()}", "m": ["4", "7"], "d": { "format": "{digits(1)}", "repeat": 12 }, "weight": 1 } ] diff --git a/data/sv_SE/address.json b/data/sv_SE/address.json index 94de7d1..81766cb 100644 --- a/data/sv_SE/address.json +++ b/data/sv_SE/address.json @@ -10,15 +10,15 @@ ["Avenyn", "Birger Jarlsgatan", "Promenaden", "Staby", "Sveavägen", "Vintjärn"] ], "street-number": [ - { "format": "1" }, - { "format": "10" }, - { "format": "100", "weight": 0.2 }, - { "format": "1A", "weight": 0.2 }, - { "format": "10A", "weight": 0.1 }, - { "format": "100A", "weight": 0.05 } + { "format": "{int(1,9)}" }, + { "format": "{int(10,99)}" }, + { "format": "{int(100,999)}", "weight": 0.2 }, + { "format": "{int(1,9)}{upper(1)}", "weight": 0.2 }, + { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 }, + { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 } ], "postal-code": [ - { "format": "100 10" } + { "format": "{int(100,999)} {int(10,99)}" } ], "locality": ["Alingsås", "Alvesta", "Ängelholm", "Arboga", "Arvika", "Avesta", "Boden", "Bollnäs", "Borås", "Borlänge", "Enköping", "Eskilstuna", "Eslöv", "Fagersta", "Falkenberg", "Falköping", "Falun", "Finspång", "Gällivare", "Gävle", "Göteborg", "Halmstad", "Haparanda", "Härnösand", "Hässleholm", "Helsingborg", "Huddinge", "Hudiksvall", "Jönköping", "Kalmar", "Karlshamn", "Karlskoga", "Karlskrona", "Karlstad", "Katrineholm", "Kiruna", "Köping", "Kramfors", "Kristianstad", "Kristinehamn", "Landskrona", "Lidingö", "Lidköping", "Lindesberg", "Linköping", "Ljungby", "Ludvika", "Luleå", "Lund", "Lycksele", "Malmö", "Mariestad", "Mjölby", "Mölndal", "Mora", "Motala", "Nacka", "Nässjö", "Norrköping", "Norrtälje", "Nyköping", "Nynäshamn", "Örebro", "Örnsköldsvik", "Oskarshamn", "Östersund", "Piteå", "Rabbalshede", "Ronneby", "Säffle", "Sandviken", "Sävsjö", "Sigtuna", "Skara", "Skellefteå", "Skövde", "Söderhamn", "Södertälje", "Sollentuna", "Solna", "Sölvesborg", "Stockholm", "Strängnäs", "Sundbyberg", "Sundsvall", "Täby", "Tierp", "Tranås", "Trelleborg", "Trollhättan", "Uddevalla", "Ulricehamn", "Umeå", "Upplands Väsby", "Uppsala", "Vänersborg", "Varberg", "Värnamo", "Västerås", "Västervik", "Växjö", "Vetlanda", "Vimmerby", "Visby", "Ystad"] } diff --git a/data/sv_SE/color.json b/data/sv_SE/color.json index 58cbd03..52f4747 100644 --- a/data/sv_SE/color.json +++ b/data/sv_SE/color.json @@ -1,6 +1,6 @@ [ { - "format": "##{x}{x}{x}{x}{x}{x}", + "format": "#{x}{x}{x}{x}{x}{x}", "x": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], "weight": 2 }, diff --git a/data/sv_SE/date.json b/data/sv_SE/date.json index f19ce7b..18de4f5 100644 --- a/data/sv_SE/date.json +++ b/data/sv_SE/date.json @@ -4,12 +4,12 @@ "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], "year": [ - { "format": "#1970" }, - { "format": "#1980" }, - { "format": "#1990", "weight": 2 }, - { "format": "2#0#00", "weight": 3 }, - { "format": "2#0#10", "weight": 3 }, - { "format": "2#020", "weight": 2 } + { "format": "197{digits(1)}" }, + { "format": "198{digits(1)}" }, + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } ] } ] diff --git a/data/sv_SE/ip.json b/data/sv_SE/ip.json index 2f071a7..e16c78a 100644 --- a/data/sv_SE/ip.json +++ b/data/sv_SE/ip.json @@ -2,7 +2,7 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "0", "weight": 3 }, { "format": "10", "weight": 4 }, { "format": "#100", "weight": 2 }] + "o": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}", "weight": 4 }, { "format": "1{digits(2)}", "weight": 2 }] }, { "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" diff --git a/data/sv_SE/phone.json b/data/sv_SE/phone.json index 69a4c4c..0bef075 100644 --- a/data/sv_SE/phone.json +++ b/data/sv_SE/phone.json @@ -3,15 +3,15 @@ "format": "{prefix}-{a} {b} {c}", "weight": 10, "prefix": ["070", "072", "073", "076", "079"], - "a": [{ "format": "000" }], - "b": [{ "format": "00" }], - "c": [{ "format": "00" }] + "a": [{ "format": "{digits(3)}" }], + "b": [{ "format": "{digits(2)}" }], + "c": [{ "format": "{digits(2)}" }] }, { "format": "{prefix}-{a} {b} {c}", "prefix": ["08", "011", "013", "018", "019", "021", "023", "026", "031", "033", "035", "036", "040", "042", "044", "046", "054", "060", "063", "090"], - "a": [{ "format": "000" }], - "b": [{ "format": "00" }], - "c": [{ "format": "00" }] + "a": [{ "format": "{digits(3)}" }], + "b": [{ "format": "{digits(2)}" }], + "c": [{ "format": "{digits(2)}" }] } ] diff --git a/data/sv_SE/price.json b/data/sv_SE/price.json index d82c69e..92a33db 100644 --- a/data/sv_SE/price.json +++ b/data/sv_SE/price.json @@ -2,12 +2,12 @@ { "format": "{amt}{ore} kr", "amt": [ - { "format": "1", "weight": 2 }, - { "format": "10", "weight": 4 }, - { "format": "100", "weight": 3 }, - { "format": "1 000", "weight": 1 }, - { "format": "10 000", "weight": 0.3 } + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + { "format": "{int(1,9)} {digits(3)}", "weight": 1 }, + { "format": "{int(10,99)} {digits(3)}", "weight": 0.3 } ], - "ore": ["", { "format": ",{c}", "c": [{ "format": "00" }], "weight": 0.4 }] + "ore": ["", { "format": ",{c}", "c": [{ "format": "{digits(2)}" }], "weight": 0.4 }] } ] diff --git a/data/sv_SE/ssn.json b/data/sv_SE/ssn.json index 3dbc854..07ed245 100644 --- a/data/sv_SE/ssn.json +++ b/data/sv_SE/ssn.json @@ -1,6 +1,6 @@ [ { - "format": "00{mmdd}-000{luhn()}", + "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": [ { "format": "{m}{d}", diff --git a/data/sv_SE/time.json b/data/sv_SE/time.json index 856e733..2084e19 100644 --- a/data/sv_SE/time.json +++ b/data/sv_SE/time.json @@ -1,8 +1,8 @@ [ { "format": "{hour}:{minute}{sec}", - "hour": [{ "format": "#00", "weight": 4 }, { "format": "#10", "weight": 4 }, { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 }], - "minute": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], - "sec": ["", { "format": ":{s}", "s": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], "weight": 0.4 }] + "hour": [{ "format": "0{digits(1)}", "weight": 4 }, { "format": "1{digits(1)}", "weight": 4 }, { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 }], + "minute": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], + "sec": ["", { "format": ":{s}", "s": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], "weight": 0.4 }] } ] diff --git a/data/sv_SE/version.json b/data/sv_SE/version.json index 65bc1cc..f0806d4 100644 --- a/data/sv_SE/version.json +++ b/data/sv_SE/version.json @@ -1,7 +1,7 @@ [ { "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "0", "weight": 3 }, { "format": "10" }], + "n": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}" }], "pre": ["", { "format": "v", "weight": 0.4 }], "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] } diff --git a/fejkdata.go b/fejkdata.go index def1fe6..b0fecb8 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -156,8 +156,6 @@ func paths(n node) []string { out = append(out, p) } return out - case literal: - return []string{""} } return nil } diff --git a/format-migration/convert.py b/format-migration/convert.py new file mode 100755 index 0000000..073d3ef --- /dev/null +++ b/format-migration/convert.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Rewrite data files from the class-char format grammar to the literal-text one. + + convert.py data/sv_SE/person.json ... + +Old: 0 1 A a are character classes, # escapes. New: text is literal; a run of +0/A/a becomes {digits(n)}/{upper(n)}/{lower(n)}, 1 followed by n zeros becomes +{int(10^n, 10^(n+1)-1)}, a literal brace becomes {{ or }}. +""" +import re +import sys + +RUN = {"0": "{{digits({})}}", "A": "{{upper({})}}", "a": "{{lower({})}}"} + + +def literal(c): + return {"{": "{{", "}": "}}"}.get(c, c) + + +def convert_format(f): + out, i, n = [], 0, len(f) + while i < n: + c = f[i] + if c == "#": + i += 1 + if i < n: + out.append(literal(f[i])) + i += 1 + else: + out.append("#") + elif c == "{": + j = f.find("}", i) + if j < 0: + out.append(f[i:]) + break + out.append(f[i:j + 1]) + i = j + 1 + elif c in RUN: + k = 0 + while i < n and f[i] == c: + k += 1 + i += 1 + out.append(RUN[c].format(k)) + elif c == "1": + i += 1 + k = 0 + while i < n and f[i] == "0": + k += 1 + i += 1 + out.append("{{int({},{})}}".format(10 ** k, 10 ** (k + 1) - 1)) + else: + out.append(literal(c)) + i += 1 + return "".join(out) + + +FORMAT_VALUE = re.compile(r'("format":\s*")((?:[^"\\]|\\.)*)(")') + + +def convert_text(text): + return FORMAT_VALUE.sub(lambda m: m.group(1) + convert_format(m.group(2)) + m.group(3), text) + + +if __name__ == "__main__": + for path in sys.argv[1:]: + with open(path) as fh: + before = fh.read() + after = convert_text(before) + if after != before: + with open(path, "w") as fh: + fh.write(after) + print("converted", path) diff --git a/node.go b/node.go index 4e81294..b9cda30 100644 --- a/node.go +++ b/node.go @@ -7,16 +7,11 @@ import ( "strings" ) -// node is a compiled template element: literal, choice, or template. Compiling -// JSON into these once (see compile) means rendering never re-inspects the raw -// JSON or re-sums weights. +// node is a compiled template element: a choice or a template. Compiling JSON +// into these once (see compile) means rendering never re-inspects the raw JSON or +// re-sums weights. type node interface{ isNode() } -// literal is emitted verbatim, never formatted. -type literal string - -func (literal) isNode() {} - // group is a namespace of named children, built from a directory of JSON files // and subdirectories. It has no value of its own: descend into a named child by // dot path; rendering one is an error (see Fake). @@ -36,16 +31,19 @@ type choice struct { func (*choice) isNode() {} -// template renders a format string, substituting {tokens} from fields. repeat -// (default 1) renders that format that many times and joins the results with -// separator (default ""), each render an independent pick. +// template renders a format string, substituting {tokens} from fields. A bare +// JSON string is a template with no fields. repeat (default 1) renders that format +// that many times and joins the results with separator (default ""), each render +// an independent pick. type template struct { format string fields map[string]node repeat int separator string - ops []op // format compiled once (see compileOps); what expand walks - grow int // minimum output size, to size the render buffer + ops []op // format compiled once (see compileOps); what expand walks + grow int // minimum output size, to size the render buffer + fixed bool // no op varies, so every render is lit + lit string // the whole output when fixed // bound maps each field the format addresses by dotted path to one path token // reading it, which is the half of an overlap the fences name. nil when the // format takes no path. @@ -72,7 +70,7 @@ func compile(v any) (node, error) { func compileItem(v any) (node, error) { switch v := v.(type) { case string: - return literal(v), nil + return compileString(v) case []any: return compileChoice(v) case map[string]any: @@ -82,6 +80,29 @@ func compileItem(v any) (node, error) { } } +func compileString(s string) (node, error) { + if err := checkTokens(s, nil); err != nil { + return nil, err + } + t := &template{format: s, repeat: 1} + t.compileFormat() + return t, nil +} + +// compileFormat compiles the format into ops once every field is in place. +func (t *template) compileFormat() { + t.ops, t.grow, t.bound, t.held = compileOps(t.format) + t.fixed = true + for _, o := range t.ops { + if o.kind != 'l' { + t.fixed = false + } + } + if t.fixed && len(t.ops) == 1 { + t.lit = t.ops[0].lit + } +} + func compileChoice(items []any) (node, error) { if len(items) == 0 { return nil, fmt.Errorf("empty choice") @@ -163,7 +184,7 @@ func compileTemplate(m map[string]any) (node, error) { if err := checkTokens(format, t.fields); err != nil { return nil, err } - t.ops, t.grow, t.bound, t.held = compileOps(format) + t.compileFormat() if err := checkNoOverlap(format, t.bound); err != nil { return nil, err } @@ -205,8 +226,6 @@ func checkPath(n node, tail []string, level string) error { return nil } return checkPath(n.items[0], tail, level) - case literal: - return fmt.Errorf("a plain string has no field %q", tail[0]) default: return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) } @@ -255,9 +274,10 @@ func weightOf(raw any) (float64, error) { // reservedInName is what a category, folder or field name may not contain: a dot // separates the segments of a path, '|' the arms of a token, '(' opens a function -// call and '}' ends the token. A name carrying one is reachable by no format, so it -// is rejected where it is authored rather than at the token that cannot reach it. -const reservedInName = ".|(}" +// call and braces delimit the token. A name carrying one is reachable by no format, +// so it is rejected where it is authored rather than at the token that cannot +// reach it. +const reservedInName = ".|({}" // reservedList spells reservedInName for an error message, so the two cannot drift. var reservedList = strings.Join(strings.Split(reservedInName, ""), " ") diff --git a/reference.go b/reference.go index e449855..97d108e 100644 --- a/reference.go +++ b/reference.go @@ -90,11 +90,10 @@ func checkBoundLevelsHeld(root map[string]node) error { } // cover collects what one held draw of a level answers for: the level and -// everything contained in it, since a path may read any of it. Literals are left -// out — one fixed string cannot disagree with itself, and a literal is a value, so -// two that spell the same text are indistinguishable. +// everything contained in it, since a path may read any of it. A fixed string is +// left out — it cannot disagree with itself. func cover(n node, into map[node]bool) { - if _, fixed := n.(literal); fixed { + if isFixed(n) { return } into[n] = true @@ -111,10 +110,9 @@ func cover(n node, into map[node]bool) { // The walk stops at a {..path} edge, which is where the operand's own value ends // and a shared source begins: two names referencing one category are two draws, the // same rule {word} {word} follows. cover stops there too, by way of named, so both -// halves of the fence end at the same boundary. A literal is left out for the -// reason cover leaves one out. +// halves of the fence end at the same boundary. func operandDraw(n node, into map[node]bool) { - if _, fixed := n.(literal); fixed { + if isFixed(n) { return } if into[n] { @@ -129,6 +127,12 @@ func operandDraw(n node, into map[node]bool) { } } +// isFixed is a string that varies nothing: fixed text with no fields to read into. +func isFixed(n node) bool { + t, ok := n.(*template) + return ok && t.fixed && len(t.fields) == 0 +} + // renders reports whether rendering n can reach anything in want, following the // same edges expand does. seen keeps a node shared by several routes from being // walked twice; checkNoCycles has already proved the graph is a DAG, so the walk @@ -294,7 +298,7 @@ func (e renderEdge) reached() string { // renderEdges lists the children rendering n recurses into, mirroring expand: a // choice's items, and a template's field/reference tokens plus its calc operands. -// A literal or group renders nothing, so it has no edges. +// A group renders nothing, so it has no edges. func renderEdges(n node) []renderEdge { switch n := n.(type) { case *choice: diff --git a/render.go b/render.go index 13ca4bd..4139ab7 100644 --- a/render.go +++ b/render.go @@ -59,8 +59,6 @@ func descend(s *session, n node, segments []string) (node, error) { } } return descend(s, pick(s, n), segments) - case literal: - return nil, fmt.Errorf("a plain string has no field %q", segments[0]) default: return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[0]) } @@ -85,12 +83,13 @@ func unreachableInChoice(c *choice, want string) error { // front, so rendering a compiled tree cannot fail. func render(s *session, n node) string { switch n := n.(type) { - case literal: - return string(n) case *choice: return render(s, pick(s, n)) case *template: if n.repeat == 1 { + if n.fixed { + return n.lit + } return expand(s, n) } var b strings.Builder @@ -119,21 +118,6 @@ func pick(r rng, c *choice) node { return c.items[i] } -// classChar randomises one character-class rune: '0' digit 0-9, '1' digit 1-9, -// 'A' letter A-Z, 'a' letter a-z. -func classChar(s *session, c rune) byte { - switch c { - case '0': - return byte('0' + s.IntN(10)) - case '1': - return byte('1' + s.IntN(9)) - case 'A': - return byte('A' + s.IntN(26)) - default: // 'a' - return byte('a' + s.IntN(26)) - } -} - // expand renders a template's compiled ops. compile validated every token, so this // cannot fail. func expand(s *session, t *template) string { @@ -153,8 +137,6 @@ func expand(s *session, t *template) string { switch o.kind { case 'l': b.WriteString(o.lit) - case 'c': - b.WriteByte(classChar(s, o.r)) case 'f': b.WriteString(readField(s, t, held, o.arms[s.IntN(len(o.arms))])) case 'b': diff --git a/template.go b/template.go index 2ca0fb6..76aed2b 100644 --- a/template.go +++ b/template.go @@ -6,41 +6,33 @@ import ( "strings" ) -// This file is the {token} grammar of a format string: how it is scanned -// (eachToken), how a function token is parsed (funcCall) and validated -// (checkFunc, checkTokens), and the builtin contract its functions implement. -// render.go evaluates these tokens; node.go compiles the surrounding JSON; -// reference.go binds {..path} tokens across the tree. - -// ftoken is one unit of a scanned format string: a literal rune to emit, a class -// char to randomise ('0' '1' 'A' 'a'), or the body of a {…} token. +// ftoken is one unit of a scanned format string: a literal rune or the body of a +// {…} token. type ftoken struct { - kind byte // 'l' literal rune, 'c' class char, 'b' brace body - r rune // for kinds 'l' and 'c' + kind byte // 'l' literal rune, 'b' brace body + r rune body string } // eachToken scans a format string once and calls fn for each unit, the single -// source of truth for how '#' escapes and {…} braces are read — compileOps, -// checkTokens, fieldTokens and refTokens all drive off it so the grammar can't -// drift between the validator and the renderer. '#' escapes the next char to a -// literal ("#0" -> '0', "##" -> '#'); a '{' must reach a '}' (else an error); an -// unmatched '}' is an ordinary literal. The error stops the scan early. +// source of truth for how braces are read: "{{" and "}}" are literal braces, a "{" +// opens a token that must reach its "}", and a lone "}" is an error. func eachToken(format string, fn func(ftoken) error) error { rs := []rune(format) for i := 0; i < len(rs); i++ { var t ftoken switch c := rs[i]; c { - case '#': - t.kind, t.r = 'l', '#' - if i++; i < len(rs) { - t.r = rs[i] - } - case '0', '1', 'A', 'a': - t.kind, t.r = 'c', c case '{': + if i+1 < len(rs) && rs[i+1] == '{' { + t.kind, t.r = 'l', '{' + i++ + break + } end := i + 1 for end < len(rs) && rs[end] != '}' { + if rs[end] == '{' { + return fmt.Errorf("'{' inside a token in %q; a literal brace is written {{", format) + } end++ } if end >= len(rs) { @@ -48,6 +40,13 @@ func eachToken(format string, fn func(ftoken) error) error { } t.kind, t.body = 'b', string(rs[i+1:end]) i = end + case '}': + if i+1 < len(rs) && rs[i+1] == '}' { + t.kind, t.r = 'l', '}' + i++ + break + } + return fmt.Errorf("lone '}' in %q; a literal brace is written }}", format) default: t.kind, t.r = 'l', c } @@ -61,13 +60,10 @@ func eachToken(format string, fn func(ftoken) error) error { // builtin is a format-string function invoked as {name(args)}. It receives the // session (its rng, and the {seq()} counters), the output emitted so far in the // current expansion (for derivations such as a checksum over preceding digits), and -// the values of the operands it named (only calc names any). -// Almost all are pure over (rng, emitted, args) — no wall-clock, no crypto/rand — -// so seeding stays reproducible; a time-based id derives its time from the rng. -// seq is the one exception: it advances per-session counter state, which is itself -// deterministic (1, 2, 3 …). arity is the exact arg count, or -1 for variadic -// (then check does all the validation). The optional check validates args at -// compile time (their values, beyond the count). The registry lives in builtins.go. +// the values of the operands it named (only calc names any). All must stay pure +// over (rng, emitted, args) so seeded output is reproducible; seq advances +// per-session counter state, which is itself deterministic. arity is the exact arg +// count, or -1 for variadic (then check does all the validation). type builtin struct { arity int // prep parses validated args once, at compile time, into the closure expand calls. @@ -165,11 +161,9 @@ func checkTokens(format string, fields map[string]node) error { }) } -// checkNoRepeatedArm rejects {a|a|b}. An alternation picks its arms evenly, so a -// repeated one skews the odds — a third spelling of what weight is for, and one an -// author is far likelier to have typed by accident than meant. The error names the -// spelling that does skew a pick. It runs over the arms as written, so a repeated -// reference arm ({..a|..a}) is caught alongside a repeated sibling. +// checkNoRepeatedArm rejects {a|a|b}: an alternation picks its arms evenly, so a +// repeated one is a second spelling of weight. The error names the spelling that +// does skew a pick. func checkNoRepeatedArm(body string, names []string) error { if len(names) < 2 { return nil @@ -228,10 +222,9 @@ func splitArm(name string) arm { } // checkNoOverlap rejects a format that both renders a level and reads a path into -// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The two spell one -// draw two ways: the path reads the level's held draw, while rendering the level -// expands it afresh, so their values disagree. One spelling, so there is nothing -// to get wrong. Names are compared in sorted order, so which pair is reported +// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The path reads the +// level's held draw while rendering the level expands it afresh, so their values +// would disagree. Names are compared in sorted order, so which pair is reported // does not depend on where the tokens sit. func checkNoOverlap(format string, bound map[string]string) error { names := boundReaders(format, bound) @@ -311,13 +304,12 @@ func splitArms(body string) []arm { // values of the operands it named, which expand read for it. type callFn func(s *session, emitted string, operands []string) string -// op is one compiled unit of a format string: a literal run, a class char, a field -// alternation, or a builtin already bound to its args. compile builds these so -// render never re-scans the format. +// op is one compiled unit of a format string: a literal run, a field alternation, +// or a builtin already bound to its args. compile builds these so render never +// re-scans the format. type op struct { - kind byte // 'l' literal run, 'c' class char, 'f' field alternation, 'b' builtin + kind byte // 'l' literal run, 'f' field alternation, 'b' builtin lit string // kind 'l' - r rune // kind 'c' arms []arm // kind 'f': the '|' alternatives, split into key and path once call callFn // operands names the sibling fields a {calc()} reads, in the order calcVars @@ -325,14 +317,14 @@ type op struct { operands []string } -// compileOps turns a format string into ops, and returns the smallest output it can -// produce (literals plus one byte per class char) to size the render buffer, plus -// two sets. bound is the levels the format addresses by dotted path, each mapped to -// the first path reading it, which is what the overlap fences name. held is every -// name drawn once per expansion — those levels, plus the siblings a {calc()} reads, -// so an operand shown is the operand computed. Both are nil when the format needs -// neither, so data that uses neither carries no render-time cost. Call checkTokens -// first: it is what proves the scan and every token are valid. +// compileOps turns a format string into ops, and returns the size of its literal +// text to size the render buffer, plus two sets. bound is the levels the format +// addresses by dotted path, each mapped to the first path reading it, which is what +// the overlap fences name. held is every name drawn once per expansion — those +// levels, plus the siblings a {calc()} reads, so an operand shown is the operand +// computed. Both are nil when the format needs neither, so data that uses neither +// carries no render-time cost. Call checkTokens first: it is what proves the scan +// and every token are valid. func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { var ops []op var lit strings.Builder @@ -356,10 +348,6 @@ func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { switch t.kind { case 'l': lit.WriteRune(t.r) - case 'c': - flush() - grow++ - ops = append(ops, op{kind: 'c', r: t.r}) case 'b': flush() if name, args, ok := funcCall(t.body); ok { diff --git a/template_test.go b/template_test.go index 33887a8..cdd6420 100644 --- a/template_test.go +++ b/template_test.go @@ -219,32 +219,32 @@ func TestCompileErrors(t *testing.T) { `[{"format":"A","weight":0},{"format":"B","weight":0}]`, // weights sum to zero `[{"format":"A","weight":1e308},{"format":"B","weight":1e308}]`, // weights overflow to +Inf `[{"format":"A","weight":"heavy"}]`, // non-numeric weight - `{"format":"x","repeat":0}`, // repeat below 1 - `{"format":"x","repeat":-2}`, // negative repeat - `{"format":"x","repeat":1.5}`, // non-integer repeat - `{"format":"x","repeat":"two"}`, // non-numeric repeat - `{"format":"x","repeat":2,"separator":5}`, // non-string separator - `{"format":"{nope()}"}`, // unknown function - `{"format":"{luhn(x)}"}`, // function given args it takes none of - `{"format":"{luhn(}"}`, // malformed function token - `{"format":"{int(1)}"}`, // wrong arity - `{"format":"{int(a,b)}"}`, // non-integer args - `{"format":"{int(5,1)}"}`, // min > max - `{"format":"{hex(0)}"}`, // count must be positive - `{"format":"{nanoid(-1)}"}`, // negative count - `{"format":"{base64(0)}"}`, // count must be positive - `{"format":"{float(1,2)}"}`, // wrong arity - `{"format":"{float(1,2,-1)}"}`, // negative decimals - `{"format":"{iban(US)}"}`, // unsupported country - `{"format":"{seq(a,b)}"}`, // seq takes at most one name - `{"format":"{calc()}"}`, // calc needs an expression - `{"format":"{calc(1 +)}"}`, // dangling operator - `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis - `{"format":"{calc(1 2)}"}`, // two operands, no operator - `{"format":"{calc(price)}"}`, // operand names no field - `{"format":"{calc(1, 2, 3)}"}`, // too many args - `{"format":"{calc(1, x)}"}`, // decimals arg not an integer - `{"format":"{calc(1, -1)}"}`, // decimals negative + `{"format":"x","repeat":0}`, // repeat below 1 + `{"format":"x","repeat":-2}`, // negative repeat + `{"format":"x","repeat":1.5}`, // non-integer repeat + `{"format":"x","repeat":"two"}`, // non-numeric repeat + `{"format":"x","repeat":2,"separator":5}`, // non-string separator + `{"format":"{nope()}"}`, // unknown function + `{"format":"{luhn(x)}"}`, // function given args it takes none of + `{"format":"{luhn(}"}`, // malformed function token + `{"format":"{int(1)}"}`, // wrong arity + `{"format":"{int(a,b)}"}`, // non-integer args + `{"format":"{int(5,1)}"}`, // min > max + `{"format":"{hex(0)}"}`, // count must be positive + `{"format":"{nanoid(-1)}"}`, // negative count + `{"format":"{base64(0)}"}`, // count must be positive + `{"format":"{float(1,2)}"}`, // wrong arity + `{"format":"{float(1,2,-1)}"}`, // negative decimals + `{"format":"{iban(US)}"}`, // unsupported country + `{"format":"{seq(a,b)}"}`, // seq takes at most one name + `{"format":"{calc()}"}`, // calc needs an expression + `{"format":"{calc(1 +)}"}`, // dangling operator + `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis + `{"format":"{calc(1 2)}"}`, // two operands, no operator + `{"format":"{calc(price)}"}`, // operand names no field + `{"format":"{calc(1, 2, 3)}"}`, // too many args + `{"format":"{calc(1, x)}"}`, // decimals arg not an integer + `{"format":"{calc(1, -1)}"}`, // decimals negative } { if _, err := compile(parse(t, bad)); err == nil { t.Errorf("compile(%s) = nil error, want error", bad) -- 2.52.0 From de8c1d13e937d3d6d3035ffa4792a7b10741545b Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:37:21 +0200 Subject: [PATCH 07/38] Tests for one spelling per shape: no one-item choice, repeated item or inert object --- bench_test.go | 13 ++-- bound_test.go | 104 ++++++++++++++----------------- builtins_test.go | 54 ++++++++--------- calc_test.go | 36 +++++------ edge_test.go | 84 ++++++++++++------------- loading_test.go | 32 +++++----- one_spelling_test.go | 48 +++++++++++++++ reference_test.go | 74 +++++++++++------------ shipped_data_test.go | 14 ++--- template_stability_test.go | 16 ++--- template_test.go | 121 +++++++++++++++++++------------------ 11 files changed, 315 insertions(+), 281 deletions(-) create mode 100644 one_spelling_test.go diff --git a/bench_test.go b/bench_test.go index 0b850ed..d3bc76f 100644 --- a/bench_test.go +++ b/bench_test.go @@ -43,7 +43,7 @@ func tmpData(b *testing.B, name, body string) string { } func BenchmarkCalc(b *testing.B) { - dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`) + dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`) benchPath(b, dir, "inv") } @@ -61,24 +61,19 @@ func BenchmarkRepeat(b *testing.B) { // what a correlated pair costs against BenchmarkUnbound, the same output drawn from // two independent fields. func BenchmarkBound(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"1{digits(2)} {digits(2)}"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"573 {digits(2)}"}}]}`) + dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`) benchPath(b, dir, "addr") } func BenchmarkUnbound(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}", - "postal-code":[{"format":"1{digits(2)} {digits(2)}"},{"format":"573 {digits(2)}"}], - "locality":["Stockholm","Tranås"]}`) + dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`) benchPath(b, dir, "addr") } // BenchmarkBoundDeep reads two paths through an intermediate level, the shape the // depth question is about. func BenchmarkBoundDeep(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":[ - {"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":{"format":"1{digits(2)} {digits(2)}"}}}]}`) + dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`) benchPath(b, dir, "addr") } diff --git a/bound_test.go b/bound_test.go index a209949..a266a34 100644 --- a/bound_test.go +++ b/bound_test.go @@ -15,10 +15,7 @@ import ( // swedishPlaces is a two-variant sibling whose variants pair a locality with the // postal-code prefix that really belongs to it. -const swedishPlaces = `{"format":"%s","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"1{digits(2)} {digits(2)}"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"573 {digits(2)}"}} -]}` +const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}` // agree reports whether a rendered "postcode locality" pair is a real pairing. var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`) @@ -48,10 +45,10 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { // other rendered — so the pair is a load error rather than a shape whose two // halves can disagree. rejected := map[string]string{ - "bare head beside a path": `{"format":"{p} + {p.first}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, - "prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":[{"format":"{addr}","addr":[` + - `{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}]}`, - "either order": `{"format":"{p.first} + {p}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + "bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + "prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` + + `{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`, + "either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, } for name, file := range rejected { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) @@ -65,7 +62,7 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { } // The same path twice is one spelling, so it stays legal. if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{p.first} + {p.first}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + "cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`, }))); err != nil { t.Errorf("New = %v, want one path read twice accepted", err) } @@ -84,7 +81,7 @@ func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) { } // Arithmetic over fields no path names is untouched. if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`, + "cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`, }))); err != nil { t.Errorf("New = %v, want plain arithmetic accepted", err) } @@ -95,13 +92,13 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { // afresh beside the path that reads its held draw — the same overlap by // another spelling. rejected := map[string]string{ - "reference names the head": `{"format":"{p.first}|{..cat.p}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + "reference names the head": `{"format":"{p.first}|{..cat.p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, "reference names the leaf": `{"format":"{p.addr}|{..cat.p.addr}","p":{"format":"x","addr":["A","B","C","D"]}}`, // The reference need not sit in the format that binds: any field it renders // reaches the level just the same, however deep. "reference from a sibling field": `{"format":"{p.first}|{inner}","p":[` + `{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` + - `"inner":{"format":"{..cat.p}"}}`, + `"inner":"{..cat.p}"}`, } for name, file := range rejected { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) @@ -111,7 +108,7 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { } // A reference to anything this format does not bind is untouched. if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{p.first} {..surname}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + "cat": `{"format":"{p.first} {..surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`, "surname": `["Eriksson","Lindqvist"]`, }))); err != nil { t.Errorf("New = %v, want a reference outside the bound level accepted", err) @@ -123,7 +120,7 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { // cycle walk has to follow it there. A head whose own format names nothing // would otherwise hide the cycle until render, where it is fatal. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"static","x":{"format":"{..a}"}}}`, + "a": `{"format":"{p.x}","p":{"format":"static","x":"{..a}"}}`, }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) @@ -134,7 +131,7 @@ func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) { // {p.a} renders q, so it is a route to the level {q.x} holds — even though p's // own format names nothing. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":{"format":"{..thing.q}"}},` + + "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{..thing.q}"},` + `"q":{"format":"{x}","x":["1","2"]}}`, }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { @@ -161,7 +158,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` + - `"q":{"format":"{..cat.net}"}}`, + `"q":"{..cat.net}"}`, }, `{q} renders "net"`, }, @@ -169,8 +166,8 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { // so wrapping the operand in one changes nothing. "a reference to an operand wrapped in a choice": { map[string]string{ - "cat": `{"format":"{calc(n * 2, 2)} {q}","n":[{"format":"{v}","v":["1","2"]}],` + - `"q":{"format":"{..cat.n}"}}`, + "cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` + + `"q":"{..cat.n}"}`, }, `{q} renders "n"`, }, @@ -179,7 +176,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "an operand reaching another operand": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` + - `"b":{"format":"{..cat.a}"}}`, + `"b":"{..cat.a}"}`, }, `calc operand "b" renders "a"`, }, @@ -196,7 +193,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference into the operand one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` + - `"q":{"format":"{..cat.net.v}"}}`, + `"q":"{..cat.net.v}"}`, }, `{q} renders "net"`, }, @@ -206,7 +203,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a violation on the second of two held heads": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` + - `"b":{"format":"{y}","y":["3","4"]},"w":{"format":"{..cat.b}"}}`, + `"b":{"format":"{y}","y":["3","4"]},"w":"{..cat.b}"}`, }, `{w} renders "b"`, }, @@ -233,20 +230,20 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference to a sibling the operand never renders": { "cat": `{"format":"{calc(net * 2, 2)} {unit}",` + `"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` + - `"unit":{"format":"{..cat.net.spare}"}}`, + `"unit":"{..cat.net.spare}"}`, }, // Two operands drawing from one source are two names, so two draws: each is // held under its own name and shown once, and neither can disagree. "two operands sharing one source": { "die": `["1","2","3","4","5","6"]`, - "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":{"format":"{..die}"},` + - `"d2":{"format":"{..die}"}}`, + "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{..die}",` + + `"d2":"{..die}"}`, }, // Likewise a reference drawing from what the operand draws from: {..common} // and net are two names, not two spellings of one field. "a reference to what an operand renders through": { "common": `["1","2"]`, - "cat": `{"format":"{calc(net * 2, 2)} {..common}","net":{"format":"{..common}"}}`, + "cat": `{"format":"{calc(net * 2, 2)} {..common}","net":"{..common}"}`, }, // A fixed string cannot disagree with itself, so it needs no fence. "a literal operand named twice": { @@ -272,11 +269,11 @@ func TestAPathReachesEveryVariantItMightDraw(t *testing.T) { // to a held level disagrees silently. rejected := map[string]struct{ file, want string }{ "a cycle in a later variant": { - `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":{"format":"{..cat}"}}]}`, + `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{..cat}"}]}`, "reference cycle", }, "a second route in a later variant": { - `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":{"format":"{..cat.q}"}}],` + + `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{..cat.q}"}],` + `"q":{"format":"{y}","y":["1","2"]}}`, "reads a path into", }, @@ -309,10 +306,10 @@ func TestALevelAPathNeverRendersIsAccepted(t *testing.T) { func TestADeepDiamondChainLoads(t *testing.T) { // A diamond chain is 2^n routes through n nodes. The search past a bound level // must walk each node once, not once per route, or a deep chain never loads. - files := map[string]string{"l0": `["x"]`} + files := map[string]string{"l0": `"x"`} for i := 1; i <= 30; i++ { files[fmt.Sprintf("l%d", i)] = fmt.Sprintf( - `{"format":"{a}{b}","a":{"format":"{..l%d}"},"b":{"format":"{..l%d}"}}`, i-1, i-1) + `{"format":"{a}{b}","a":"{..l%d}","b":"{..l%d}"}`, i-1, i-1) } files["thing"] = `{"format":"{p.first} {..l30}","p":{"format":"x","first":["A","B"]}}` if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { @@ -325,9 +322,9 @@ func TestASharedNodeIsWalkedOnce(t *testing.T) { // The search past a bound level must take that in its stride rather than walk // the shared node once per route. f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{p.first}|{q}","p":[{"format":"{first}","first":["Anna","Bo"]}],` + - `"q":{"format":"{a}{b}","a":{"format":"{..shared}"},"b":{"format":"{..shared}"}}}`, - "shared": `["x"]`, + "cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` + + `"q":{"format":"{a}{b}","a":"{..shared}","b":"{..shared}"}}`, + "shared": `"x"`, })), WithSeed(1)) if err != nil { t.Fatalf("New = %v, want a shared node accepted", err) @@ -342,7 +339,7 @@ func TestTheEarlierReaderIsNamed(t *testing.T) { // first is the one reported, so the error points at the same place a reader // looking at the format would start. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":[{"format":"{first}","first":["1","2"]}]}`, + "cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`, }))) if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) { t.Fatalf("New = %v, want the calc operand named, being written first", err) @@ -368,7 +365,7 @@ func TestNestedChoiceDrawsOneVariant(t *testing.T) { f := engine(21) seen := map[string]bool{} for i := 0; i < 200; i++ { - seen[mustRender(t, f, `{"format":"{p.x}","p":[[{"format":"{x}","x":"1"}],[{"format":"{x}","x":"2"}]]}`)] = true + seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true } if !seen["1"] || !seen["2"] || len(seen) != 2 { t.Fatalf("200 draws produced %v, want 1 and 2", seen) @@ -379,7 +376,7 @@ func TestRepeatingLevelIsNamedInTheError(t *testing.T) { // The level carrying the repeat is the one to fix, so the error names it // rather than the head the path started from. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":["z"]}}}`, + "cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`, }))) if err == nil || !strings.Contains(err.Error(), `"p.a"`) { t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err) @@ -391,7 +388,7 @@ func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) { // reach inside one — otherwise the direct spelling is a load error and the same // mistake behind a choice silently drops the repeat. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":["x"]},{"format":"{a}","a":["y"]}]}`, + "cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`, }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want the repeat behind a choice rejected", err) @@ -402,7 +399,7 @@ func TestPathIntoARepeatingLevelIsRejected(t *testing.T) { // A path reads one level's draw, so it can never apply that level's repeat. // The engine rejects options that cannot take effect, and this is one. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":["z"]}}`, + "cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`, }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want a path into a repeating level rejected", err) @@ -511,7 +508,7 @@ func TestDottedTokenErrors(t *testing.T) { want string }{ "unknown head": { - `{"format":"{nope.x}","place":[{"format":"{v}","v":"A"}]}`, + `{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`, `no field "nope"`, }, "unknown tail": { @@ -519,7 +516,7 @@ func TestDottedTokenErrors(t *testing.T) { `"nope"`, }, "tail only some variants carry": { - `{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},{"format":"x"}]}`, + `{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`, `locality`, }, "path into a literal": { @@ -527,7 +524,7 @@ func TestDottedTokenErrors(t *testing.T) { `"y"`, }, "dotted field key": { - `{"format":"[{a.b}]","a.b":["V"]}`, + `{"format":"[{a.b}]","a.b":"V"}`, `contains "."`, }, } @@ -548,8 +545,7 @@ func TestSamePathReadTwiceReadsOneValue(t *testing.T) { // what a name shown in a display form and again in an address needs. f := engine(7) for i := 0; i < 300; i++ { - got := mustRender(t, f, `{"format":"{p.first}|{p.first}", - "p":[{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}]}`) + got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`) parts := strings.Split(got, "|") if parts[0] != parts[1] { t.Fatalf("draw %d = %q, want one value read twice", i, got) @@ -571,9 +567,7 @@ func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) { // deepPlaces nests the correlated facts a level down: the row holds an addr, and // the addr holds the city and the zip that belongs to it. -const deepPlaces = `{"format":"%s","p":[{"format":"{addr}","addr":[ - {"format":"{city}","city":"Stockholm","zip":"11111"}, - {"format":"{city}","city":"Kiruna","zip":"98100"}]}]}` +const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}` func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) { // A choice below the head is drawn once too, so two paths sharing a prefix @@ -595,13 +589,7 @@ func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) { func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) { // The rule does not run out at two: every level a path passes through is held. f := engine(12) - tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":[{"format":"{geo}","geo":[ - {"format":"{town}","region":"Norrbotten","town":[ - {"format":"{name}","name":"Kiruna","zip":"98100"}, - {"format":"{name}","name":"Luleå","zip":"97200"}]}, - {"format":"{town}","region":"Skåne","town":[ - {"format":"{name}","name":"Malmö","zip":"21100"}, - {"format":"{name}","name":"Lund","zip":"22100"}]}]}]}` + tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}` want := map[string]bool{ "Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true, "Malmö/21100 Skåne": true, "Lund/22100 Skåne": true, @@ -641,9 +629,7 @@ func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) { // Two paths share only what they actually share: a common prefix is one draw, // and each keeps its own draw past the point they part. f := engine(14) - tmpl := `{"format":"{p.a.v}{p.b.v}","p":[{"format":"x", - "a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}], - "b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}]}` + tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}` seen := map[string]bool{} for i := 0; i < 400; i++ { seen[mustRender(t, f, tmpl)] = true @@ -660,9 +646,9 @@ func TestEmptyPathSegmentIsRejected(t *testing.T) { // nothing is reported as that rather than as a missing field. An empty name is // rejected where it is authored, so no data can make these resolve. rejected := map[string]string{ - "trailing dot": `{"format":"[{a.}]","a":{"format":"x"}}`, - "leading dot": `{"format":"[{.b}]","a":{"format":"{b}","b":["V"]}}`, - "double dot": `{"format":"[{a..b}]","a":{"format":"x"}}`, + "trailing dot": `{"format":"[{a.}]","a":"x"}`, + "leading dot": `{"format":"[{.b}]","a":{"format":"{b}","b":"V"}}`, + "double dot": `{"format":"[{a..b}]","a":"x"}`, } for name, file := range rejected { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) @@ -681,7 +667,7 @@ func TestCycleThroughAPathTokenIsRejected(t *testing.T) { // must be caught at New. Reaching render would be fatal: the recursion never // terminates, and a stack overflow cannot be recovered. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"{x}","x":{"format":"{..a}"}}}`, + "a": `{"format":"{p.x}","p":{"format":"{x}","x":"{..a}"}}`, }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) diff --git a/builtins_test.go b/builtins_test.go index 13af539..7a63509 100644 --- a/builtins_test.go +++ b/builtins_test.go @@ -13,10 +13,10 @@ func TestBuiltinIDGenerators(t *testing.T) { tmpl string re *regexp.Regexp }{ - {"uuid v7", `{"format":"{uuid()}"}`, regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)}, - {"ulid", `{"format":"{ulid()}"}`, regexp.MustCompile(`^[0-7][0-9A-HJKMNP-TV-Z]{25}$`)}, - {"nanoid", `{"format":"{nanoid(21)}"}`, regexp.MustCompile(`^[A-Za-z0-9_-]{21}$`)}, - {"hex", `{"format":"{hex(16)}"}`, regexp.MustCompile(`^[0-9a-f]{16}$`)}, + {"uuid v7", `"{uuid()}"`, regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)}, + {"ulid", `"{ulid()}"`, regexp.MustCompile(`^[0-7][0-9A-HJKMNP-TV-Z]{25}$`)}, + {"nanoid", `"{nanoid(21)}"`, regexp.MustCompile(`^[A-Za-z0-9_-]{21}$`)}, + {"hex", `"{hex(16)}"`, regexp.MustCompile(`^[0-9a-f]{16}$`)}, } f := engine(1) for _, c := range cases { @@ -38,9 +38,9 @@ func TestBuiltinIDGenerators(t *testing.T) { // sample draws only from the seeded rng, so same seed -> same value. func TestBuiltinSamplesReproducible(t *testing.T) { for _, tmpl := range []string{ - `{"format":"{uuid()}"}`, `{"format":"{ulid()}"}`, - `{"format":"{nanoid(12)}"}`, `{"format":"{int(1,1000000)}"}`, - `{"format":"{float(0,1,6)}"}`, `{"format":"{base64(12)}"}`, `{"format":"{iban(SE)}"}`, + `"{uuid()}"`, `"{ulid()}"`, + `"{nanoid(12)}"`, `"{int(1,1000000)}"`, + `"{float(0,1,6)}"`, `"{base64(12)}"`, `"{iban(SE)}"`, } { if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b { t.Fatalf("%s not reproducible: %q != %q", tmpl, a, b) @@ -52,7 +52,7 @@ func TestBuiltinIntInclusiveRange(t *testing.T) { f := engine(1) lo, hi := false, false for i := 0; i < 1000; i++ { - n, err := strconv.Atoi(mustRender(t, f, `{"format":"{int(3,7)}"}`)) + n, err := strconv.Atoi(mustRender(t, f, `"{int(3,7)}"`)) if err != nil || n < 3 || n > 7 { t.Fatalf("int(3,7) = %d (err %v), out of range", n, err) } @@ -66,14 +66,14 @@ func TestBuiltinIntInclusiveRange(t *testing.T) { func TestBuiltinFloat(t *testing.T) { f, re := engine(1), regexp.MustCompile(`^[12]\.\d{3}$`) for i := 0; i < 200; i++ { - if got := mustRender(t, f, `{"format":"{float(1,2,3)}"}`); !re.MatchString(got) { + if got := mustRender(t, f, `"{float(1,2,3)}"`); !re.MatchString(got) { t.Fatalf("float(1,2,3) = %q, want d.ddd in [1,2]", got) } } } func TestBuiltinBase64(t *testing.T) { - got := mustRender(t, engine(1), `{"format":"{base64(9)}"}`) + got := mustRender(t, engine(1), `"{base64(9)}"`) if b, err := base64.StdEncoding.DecodeString(got); err != nil || len(b) != 9 { t.Fatalf("base64(9) = %q decodes to %d bytes (err %v), want 9", got, len(b), err) } @@ -83,9 +83,9 @@ func TestBuiltinBase64(t *testing.T) { // hand-computed check, like the luhn test. mod-11 emits X when it would be 10. func TestBuiltinChecksums(t *testing.T) { cases := map[string]string{ - `{"format":"12345678{mod11()}"}`: "123456785", // weights 2..7 from the right - `{"format":"6{mod11()}"}`: "6X", // remainder 10 -> X - `{"format":"400638133393{ean()}"}`: "4006381333931", // EAN-13 (= ISBN-13) check digit + `"12345678{mod11()}"`: "123456785", // weights 2..7 from the right + `"6{mod11()}"`: "6X", // remainder 10 -> X + `"400638133393{ean()}"`: "4006381333931", // EAN-13 (= ISBN-13) check digit } f := engine(1) for tmpl, want := range cases { @@ -101,14 +101,14 @@ func TestBuiltinChecksums(t *testing.T) { func TestBuiltinSeqPerSession(t *testing.T) { f := engine(1) for i := 1; i <= 5; i++ { - if got := mustRender(t, f, `{"format":"{seq()}"}`); got != strconv.Itoa(i) { + if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) { t.Fatalf("seq call %d = %q, want %d", i, got, i) } } - if got := mustRender(t, f, `{"format":"{seq(orders)}"}`); got != "1" { + if got := mustRender(t, f, `"{seq(orders)}"`); got != "1" { t.Fatalf("named seq(orders) = %q, want its own count from 1", got) } - if got := mustRender(t, f, `{"format":"{seq()}"}`); got != "6" { + if got := mustRender(t, f, `"{seq()}"`); got != "6" { t.Fatalf("default seq after the named one = %q, want 6 (counters are independent)", got) } // repeat drives one counter forward across its renders... @@ -116,7 +116,7 @@ func TestBuiltinSeqPerSession(t *testing.T) { t.Fatalf("repeat seq = %q, want 1,2,3", got) } // ...and a fresh session restarts from 1. - if got := mustRender(t, engine(2), `{"format":"{seq()}"}`); got != "1" { + if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" { t.Fatalf("new session seq = %q, want 1", got) } } @@ -126,14 +126,14 @@ func TestBuiltinSeqPerSession(t *testing.T) { // at compile, not detonate when the template is drawn. func TestBuiltinArgLimits(t *testing.T) { bad := []string{ - `{"format":"{int(-9223372036854775808,9223372036854775807)}"}`, // span overflows int -> IntN panic - `{"format":"{hex(2000000000)}"}`, // ~2 GB string - `{"format":"{nanoid(2000000000)}"}`, - `{"format":"{base64(2000000000)}"}`, - `{"format":"{float(-1e308,1e308,2)}"}`, // span is +Inf - `{"format":"{float(0,1,100000)}"}`, // decimals -> huge string - `{"format":"{calc(1+1,100000)}"}`, // calc decimals -> huge string - `{"format":"{x}","x":["1"],"repeat":2000000000}`, // repeat -> multi-GB join + `"{int(-9223372036854775808,9223372036854775807)}"`, // span overflows int -> IntN panic + `"{hex(2000000000)}"`, // ~2 GB string + `"{nanoid(2000000000)}"`, + `"{base64(2000000000)}"`, + `"{float(-1e308,1e308,2)}"`, // span is +Inf + `"{float(0,1,100000)}"`, // decimals -> huge string + `"{calc(1+1,100000)}"`, // calc decimals -> huge string + `{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join } for _, tmpl := range bad { if _, err := compile(parse(t, tmpl)); err == nil { @@ -141,7 +141,7 @@ func TestBuiltinArgLimits(t *testing.T) { } } // Ordinary values still compile. - if _, err := compile(parse(t, `{"format":"{hex(16)} {int(-5,5)} {float(0,1,3)} {calc(1+1,2)}"}`)); err != nil { + if _, err := compile(parse(t, `"{hex(16)} {int(-5,5)} {float(0,1,3)} {calc(1+1,2)}"`)); err != nil { t.Errorf("compile of normal args failed: %v", err) } } @@ -150,7 +150,7 @@ func TestBuiltinIBAN(t *testing.T) { wantLen := map[string]int{"SE": 24, "DE": 22, "NO": 15} f := engine(1) for cc, n := range wantLen { - tmpl := `{"format":"{iban(` + cc + `)}"}` + tmpl := `"{iban(` + cc + `)}"` for i := 0; i < 100; i++ { got := mustRender(t, f, tmpl) if got[:2] != cc || len(got) != n { diff --git a/calc_test.go b/calc_test.go index 1b9a3a2..b9d6072 100644 --- a/calc_test.go +++ b/calc_test.go @@ -12,14 +12,14 @@ import ( func TestCalcArithmetic(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(2 + 3)}"}`: "5", - `{"format":"{calc(2 * 3 + 4)}"}`: "10", // * binds tighter than + - `{"format":"{calc(2 + 3 * 4)}"}`: "14", - `{"format":"{calc((2 + 3) * 4)}"}`: "20", // parentheses override - `{"format":"{calc(10 / 4)}"}`: "2.5", - `{"format":"{calc(-2 + 5)}"}`: "3", // unary minus - `{"format":"{calc(2 - -3)}"}`: "5", - `{"format":"{calc(1.5 * 2)}"}`: "3", // whole result drops the decimals + `"{calc(2 + 3)}"`: "5", + `"{calc(2 * 3 + 4)}"`: "10", // * binds tighter than + + `"{calc(2 + 3 * 4)}"`: "14", + `"{calc((2 + 3) * 4)}"`: "20", // parentheses override + `"{calc(10 / 4)}"`: "2.5", + `"{calc(-2 + 5)}"`: "3", // unary minus + `"{calc(2 - -3)}"`: "5", + `"{calc(1.5 * 2)}"`: "3", // whole result drops the decimals } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -33,9 +33,9 @@ func TestCalcArithmetic(t *testing.T) { func TestCalcAuto(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(10 / 3)}"}`: "3.3333333333333335", - `{"format":"{calc(6 / 2)}"}`: "3", - `{"format":"{calc(1 / 4)}"}`: "0.25", + `"{calc(10 / 3)}"`: "3.3333333333333335", + `"{calc(6 / 2)}"`: "3", + `"{calc(1 / 4)}"`: "0.25", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -49,9 +49,9 @@ func TestCalcAuto(t *testing.T) { func TestCalcDecimals(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(10 / 3, 2)}"}`: "3.33", - `{"format":"{calc(10 / 3, 0)}"}`: "3", - `{"format":"{calc(2 * 3, 2)}"}`: "6.00", + `"{calc(10 / 3, 2)}"`: "3.33", + `"{calc(10 / 3, 0)}"`: "3", + `"{calc(2 * 3, 2)}"`: "6.00", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -63,11 +63,11 @@ func TestCalcDecimals(t *testing.T) { // TestCalcFields pins that bare names resolve to sibling fields, rendered then // parsed as numbers. func TestCalcFields(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{calc(price * qty, 2)}","price":["19.99"],"qty":["3"]}`); got != "59.97" { + if got := mustRender(t, engine(1), `{"format":"{calc(price * qty, 2)}","price":"19.99","qty":"3"}`); got != "59.97" { t.Fatalf("calc over fields = %q, want 59.97", got) } // A field that is itself a template renders before parsing. - if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":[{"format":"{n}","n":["10"]}],"b":["3"]}`); got != "7" { + if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" { t.Fatalf("calc(a - b) = %q, want 7", got) } } @@ -75,14 +75,14 @@ func TestCalcFields(t *testing.T) { // TestCalcNonNumericIsNaN pins the never-fail rule: a field that doesn't render // to a number becomes NaN, which propagates and prints visibly. func TestCalcNonNumericIsNaN(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{calc(x * 2)}","x":["abc"]}`); got != "NaN" { + if got := mustRender(t, engine(1), `{"format":"{calc(x * 2)}","x":"abc"}`); got != "NaN" { t.Fatalf("calc over non-numeric field = %q, want NaN", got) } } // TestCalcReproducible pins that a calc over a random operand stays seed-stable. func TestCalcReproducible(t *testing.T) { - tmpl := `{"format":"{calc(q * 2 + 1)}","q":[{"format":"{int(1,1000000)}"}]}` + tmpl := `{"format":"{calc(q * 2 + 1)}","q":"{int(1,1000000)}"}` if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b { t.Fatalf("calc not reproducible: %q != %q", a, b) } diff --git a/edge_test.go b/edge_test.go index 4103385..eee05c8 100644 --- a/edge_test.go +++ b/edge_test.go @@ -16,11 +16,11 @@ import ( func TestHashIsLiteral(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"","x":["v"]}`: "", - `{"format":"#","x":["v"]}`: "#", - `{"format":"##","x":["v"]}`: "##", - `{"format":"#0#1#A#a","x":["v"]}`: "#0#1#A#a", - `{"format":"#{x}","x":["v"]}`: "#v", + `{"format":"","x":"v"}`: "", + `{"format":"#","x":"v"}`: "#", + `{"format":"##","x":"v"}`: "##", + `{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a", + `{"format":"#{x}","x":"v"}`: "#v", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -32,7 +32,7 @@ func TestHashIsLiteral(t *testing.T) { func TestMultibyteFormat(t *testing.T) { // Scanning is rune-aware: multibyte literals coexist with class chars and // tokens without corrupting indices. - got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":["väg"]}`) + got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`) if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { t.Fatalf("multibyte format = %q", got) } @@ -42,7 +42,7 @@ func TestAlternationThreeWay(t *testing.T) { f := engine(4) seen := map[string]bool{} for i := 0; i < 200; i++ { - seen[mustRender(t, f, `{"format":"{a|b|c}","a":["A"],"b":["B"],"c":["C"]}`)] = true + seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true } if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 { t.Fatalf("3-way alternation produced %v, want A, B and C", seen) @@ -69,11 +69,11 @@ func TestNewErrors(t *testing.T) { want string }{ "separator without repeat": { - map[string]string{"a": `{"format":"{x}","x":["1"],"separator":","}`}, + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, "has no effect without a repeat above 1", }, "separator with an explicit repeat of 1": { - map[string]string{"a": `{"format":"{x}","x":["1"],"repeat":1,"separator":","}`}, + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, "has no effect without a repeat above 1", }, "weight outside a choice": { @@ -85,38 +85,38 @@ func TestNewErrors(t *testing.T) { "weight only skews a choice's items", }, "weight used as a field": { - map[string]string{"a": `{"format":"{name} {weight}kg","name":["Anvil"],"weight":["7"]}`}, + map[string]string{"a": `{"format":"{name} {weight}kg","name":"Anvil","weight":["7"]}`}, "can never be a field", }, "an option name used as a token": { - map[string]string{"a": `{"format":"{weight}"}`}, + map[string]string{"a": `"{weight}"`}, `"weight" is an option and can never be a field`, }, "category name with a dot": { - map[string]string{"a.b": `["1"]`}, + map[string]string{"a.b": `"1"`}, `category "a.b" contains "."`, }, "folder name with a dot": { - map[string]string{"a.b/cat": `["1"]`}, + map[string]string{"a.b/cat": `"1"`}, `/a.b: folder "a.b" contains "."`, }, // The token grammar reserves three more characters. A name carrying one // still resolves by dot path, but no format can name it, so it is rejected // where it is authored rather than at the token that cannot reach it. "field name with a pipe": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b|c":["2"]}`}, + map[string]string{"a": `{"format":"{x}","x":"1","b|c":"2"}`}, `field "b|c" contains "|"`, }, "field name with a paren": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b(c":["2"]}`}, + map[string]string{"a": `{"format":"{x}","x":"1","b(c":"2"}`}, `field "b(c" contains "("`, }, "field name with a closing brace": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b}c":["2"]}`}, + map[string]string{"a": `{"format":"{x}","x":"1","b}c":"2"}`}, `field "b}c" contains "}"`, }, "category name with a pipe": { - map[string]string{"a|b": `["1"]`}, + map[string]string{"a|b": `"1"`}, `category "a|b" contains "|"`, }, // An empty name is not a path segment, so List never offered it — while a @@ -127,42 +127,42 @@ func TestNewErrors(t *testing.T) { `field "" is empty`, }, "folder name with a paren": { - map[string]string{"a(b/cat": `["1"]`}, + map[string]string{"a(b/cat": `"1"`}, `folder "a(b" contains "("`, }, // A repeated arm skews an alternation, which weight is the spelling for. "repeated alternation arm": { - map[string]string{"a": `{"format":"{x|x}","x":["1"]}`}, + map[string]string{"a": `{"format":"{x|x}","x":"1"}`}, `arm "x" is repeated`, }, "repeated arm among others": { - map[string]string{"a": `{"format":"{x|y|x}","x":["1"],"y":["2"]}`}, + map[string]string{"a": `{"format":"{x|y|x}","x":"1","y":"2"}`}, `arm "x" is repeated`, }, "repeated path arm": { - map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":["1"]}}`}, + map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":"1"}}`}, `arm "p.v" is repeated`, }, // A reference arm is the only kind that reaches the repeat check by passing // the per-arm checks rather than falling through them. "repeated reference arm": { - map[string]string{"a": `["x"]`, "b": `{"format":"{..a|..a}"}`}, + map[string]string{"a": `"x"`, "b": `"{..a|..a}"`}, `arm "..a" is repeated`, }, // An arm that is broken on its own terms is reported as that, not as a // repeat: the repeat is a consequence of the real mistake. "repeated arm with no path": { - map[string]string{"a": `{"format":"{..|..}"}`}, + map[string]string{"a": `"{..|..}"`}, "reference has no path", }, // No field can be named "", so the token is told that rather than sent to // name one — the fix "no field" points at is itself a load error. "repeated empty arm": { - map[string]string{"a": `{"format":"{|}"}`}, + map[string]string{"a": `"{|}"`}, "a name is never empty", }, "bare empty token": { - map[string]string{"a": `{"format":"[{}]","x":["1"]}`}, + map[string]string{"a": `{"format":"[{}]","x":"1"}`}, "a name is never empty", }, } @@ -182,16 +182,16 @@ func TestNewErrors(t *testing.T) { path string want string }{ - "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":["Anvil"],"Weight":["7"]}`}, "a", "Anvil 7kg"}, - "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":["PDF"]}`}, "a", "PDF"}, - "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":["1"]}`}, "a.x-y", "1"}, - "category named Format": {map[string]string{"Format": `["1"]`}, "Format", "1"}, - "folder named Repeat": {map[string]string{"Repeat/cat": `["1"]`}, "Repeat.cat", "1"}, - "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":["2"]}`}, "a", "2"}, - "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":["1"]}`}, "a", "111"}, + "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":"Anvil","Weight":"7"}`}, "a", "Anvil 7kg"}, + "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":"PDF"}`}, "a", "PDF"}, + "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":"1"}`}, "a.x-y", "1"}, + "category named Format": {map[string]string{"Format": `"1"`}, "Format", "1"}, + "folder named Repeat": {map[string]string{"Repeat/cat": `"1"`}, "Repeat.cat", "1"}, + "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":"2"}`}, "a", "2"}, + "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":"1"}`}, "a", "111"}, // One name in two separate tokens is two independent draws, not a repeated // arm; only a repeat within one alternation is rejected. - "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":["1"]}`}, "a", "11"}, + "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":"1"}`}, "a", "11"}, } for name, c := range accepted { f, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) @@ -212,7 +212,7 @@ func TestDeepDottedPath(t *testing.T) { // on the path are single-variant, so it resolves deterministically. f := engine(1) f.categories = map[string]node{ - "deep": compiled(t, `[{"format":"{a}","a":[{"format":"{b}","b":[{"format":"{c}","c":[{"format":"{d}","d":["leaf"]}]}]}]}]`), + "deep": compiled(t, `{"format":"{a}","a":{"format":"{b}","b":{"format":"{c}","c":{"format":"{d}","d":"leaf"}}}}`), } if got, err := f.Fake("deep.a.b.c.d"); err != nil || got != "leaf" { t.Fatalf("Fake(deep.a.b.c.d) = %q, %v, want leaf", got, err) @@ -225,7 +225,7 @@ func TestDeepDottedPath(t *testing.T) { func TestDescendIntoStringErrors(t *testing.T) { f := engine(1) - f.categories = map[string]node{"greeting": compiled(t, `["hej"]`)} + f.categories = map[string]node{"greeting": compiled(t, `"hej"`)} if _, err := f.Fake("greeting.extra"); err == nil || !strings.Contains(err.Error(), `no field "extra"`) { t.Fatalf("Fake(greeting.extra) = %v, want a no-field error", err) } @@ -235,9 +235,9 @@ func TestDescendIntoStringErrors(t *testing.T) { // one call and failing on the next: every variant must carry the rest of the path. func TestPathThroughChoice(t *testing.T) { dir := writeData(t, map[string]string{ - "every": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"]}]`, - "notall": `[{"format":"{f}","f":["1"]},["plain"]]`, - "some": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"],"extra":["x"]}]`, + "every": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, + "notall": `[{"format":"{f}","f":"1"},"plain"]`, + "some": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2","extra":"x"}]`, }) f := newGenerator(t, dir, WithSeed(1)) for i := 0; i < 200; i++ { @@ -269,7 +269,7 @@ func TestPathThroughChoice(t *testing.T) { // wherever it appears. func TestPathKeyIsUnambiguous(t *testing.T) { dir := writeData(t, map[string]string{ - "cat": `[{"format":"{a.b}","a.b":["1"]},{"format":"{a}","a":{"format":"{b}","b":["2"]}}]`, + "cat": `[{"format":"{a.b}","a.b":"1"},{"format":"{a}","a":{"format":"{b}","b":"2"}}]`, }) _, err := New(WithoutShippedData(), WithDataPath(dir)) if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { @@ -277,7 +277,7 @@ func TestPathKeyIsUnambiguous(t *testing.T) { } // The same data without the dotted key is fine, and the path resolves. f := newGenerator(t, writeData(t, map[string]string{ - "cat": `{"format":"{a.b}","a":{"format":"{b}","b":["2"]}}`, + "cat": `{"format":"{a.b}","a":{"format":"{b}","b":"2"}}`, }), WithSeed(1)) if !slices.Contains(f.List(), "cat.a.b") { t.Error("List() omits cat.a.b, which the data carries") @@ -302,8 +302,8 @@ func TestMissingFieldNamesItself(t *testing.T) { func TestCategoryRootShapes(t *testing.T) { dir := writeData(t, map[string]string{ - "obj": `{"format":"{digits(2)}"}`, // object root - "lit": `"hello"`, // bare-string root + "obj": `"{digits(2)}"`, // object root + "lit": `"hello"`, // bare-string root }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { diff --git a/loading_test.go b/loading_test.go index d8890ca..f7aadfc 100644 --- a/loading_test.go +++ b/loading_test.go @@ -13,13 +13,13 @@ import ( // folder segments — descending transparently through single-variant choices. func TestList(t *testing.T) { dir := writeData(t, map[string]string{ - "person": `{"format":"{first} {last}","first":["A"],"last":["B"]}`, + "person": `{"format":"{first} {last}","first":"A","last":"B"}`, "word": `["x", "y"]`, - "geo/city": `["Z"]`, + "geo/city": `"Z"`, // A bound {..path} reference is a render edge, not an addressable field. - "greeting": `{"format":"hej {..person.first} and {own}","own":["x"]}`, + "greeting": `{"format":"hej {..person.first} and {own}","own":"x"}`, // Only the fields every variant carries are addressable, so "extra" is not. - "coin": `[{"format":"{code}","code":["A"],"name":["Aa"]},{"format":"{code}","code":["B"],"name":["Bb"],"extra":["x"]}]`, + "coin": `[{"format":"{code}","code":"A","name":"Aa"},{"format":"{code}","code":"B","name":"Bb","extra":"x"}]`, }) got := newGenerator(t, dir, WithSeed(1)).List() want := []string{"coin", "coin.code", "coin.name", "geo.city", "greeting", "greeting.own", "person", "person.first", "person.last", "word"} @@ -66,8 +66,8 @@ func TestNewEmptyDirErrors(t *testing.T) { // namespace, so data//person.json is reachable as ".person". func TestFoldersBecomeDotPaths(t *testing.T) { dir := writeData(t, map[string]string{ - "sv_SE/greeting": `["hej"]`, - "en_US/greeting": `["hi"]`, + "sv_SE/greeting": `"hej"`, + "en_US/greeting": `"hi"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "sv_SE.greeting"); got != "hej" { @@ -82,7 +82,7 @@ func TestFoldersBecomeDotPaths(t *testing.T) { // a/b/c, then deep JSON fields inside the file at the end of that path. func TestNestedFoldersAndJSON(t *testing.T) { dir := writeData(t, map[string]string{ - "a/b/c/thing": `{"format":"{x}","x":{"format":"{y}","y":{"format":"{z}","z":["leaf"]}}}`, + "a/b/c/thing": `{"format":"{x}","x":{"format":"{y}","y":{"format":"{z}","z":"leaf"}}}`, }) f := newGenerator(t, dir, WithSeed(1)) // Folders a.b.c, file thing, then JSON fields x.y.z — one continuous path. @@ -96,7 +96,7 @@ func TestNestedFoldersAndJSON(t *testing.T) { } func TestRenderingAFolderErrors(t *testing.T) { - dir := writeData(t, map[string]string{"sv_SE/greeting": `["hej"]`}) + dir := writeData(t, map[string]string{"sv_SE/greeting": `"hej"`}) f := newGenerator(t, dir, WithSeed(1)) if _, err := f.Fake("sv_SE"); err == nil { t.Fatal("Fake(folder) = nil error, want a not-a-value error") @@ -106,8 +106,8 @@ func TestRenderingAFolderErrors(t *testing.T) { // TestMultiPathLastWins loads two dirs; on a name clash the later dir wins, and // non-clashing entries from both are reachable (data combines). func TestMultiPathLastWins(t *testing.T) { - a := writeData(t, map[string]string{"greeting": `["from-a"]`, "only-a": `["a"]`}) - b := writeData(t, map[string]string{"greeting": `["from-b"]`, "only-b": `["b"]`}) + a := writeData(t, map[string]string{"greeting": `"from-a"`, "only-a": `"a"`}) + b := writeData(t, map[string]string{"greeting": `"from-b"`, "only-b": `"b"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "greeting"); got != "from-b" { t.Fatalf("greeting = %q, want from-b (last loaded wins)", got) @@ -124,8 +124,8 @@ func TestMultiPathLastWins(t *testing.T) { // children rather than replacing wholesale: each dir adds a file to sv_SE, and // a per-file clash inside still resolves last-wins. func TestMultiPathMergesFolders(t *testing.T) { - a := writeData(t, map[string]string{"sv_SE/person": `["from-a"]`, "sv_SE/shared": `["a"]`}) - b := writeData(t, map[string]string{"sv_SE/company": `["from-b"]`, "sv_SE/shared": `["b"]`}) + a := writeData(t, map[string]string{"sv_SE/person": `"from-a"`, "sv_SE/shared": `"a"`}) + b := writeData(t, map[string]string{"sv_SE/company": `"from-b"`, "sv_SE/shared": `"b"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "sv_SE.person"); got != "from-a" { t.Fatalf("sv_SE.person = %q, want from-a (folder merged, not replaced)", got) @@ -163,10 +163,10 @@ func TestListedPathsAllRender(t *testing.T) { // carrying no JSON never contributed a namespace, so neither may fail the load. func TestHiddenEntriesAreSkipped(t *testing.T) { dir := writeData(t, map[string]string{ - "cat": `["V"]`, - ".git/HEAD": `["ignored"]`, - ".hidden": `["ignored"]`, - "empty.folder/doc": `["ignored"]`, + "cat": `"V"`, + ".git/HEAD": `"ignored"`, + ".hidden": `"ignored"`, + "empty.folder/doc": `"ignored"`, }) if err := os.Rename(filepath.Join(dir, "empty.folder", "doc.json"), filepath.Join(dir, "empty.folder", "doc.txt")); err != nil { t.Fatal(err) diff --git a/one_spelling_test.go b/one_spelling_test.go new file mode 100644 index 0000000..e2d0fe1 --- /dev/null +++ b/one_spelling_test.go @@ -0,0 +1,48 @@ +package fejkdata + +import ( + "strings" + "testing" +) + +func TestOneItemChoiceIsRejected(t *testing.T) { + for _, src := range []string{`["x"]`, `[{"format":"{d}","d":"x"}]`, `{"format":"{w}","w":["only"]}`} { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), "one-item choice") { + t.Errorf("compile(%s) = %v, want the one-item choice rejected", src, err) + } + } +} + +func TestRepeatedChoiceItemIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `["a", "a", "b"]`: `{ "format": "a", "weight": 2 }`, + `[{"format":"{x}","x":"1"},{"format":"{x}","x":"1"}]`: "repeats item", + `{"format":"{w}","w":["", "", "x"]}`: `{ "format": "", "weight": 2 }`, + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error naming %s", src, err, want) + } + } + if _, err := compile(parse(t, `[{"format":"a","weight":2}, "b"]`)); err != nil { + t.Errorf("compile(weighted a, b) = %v", err) + } +} + +func TestInertObjectIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"Malmö"}`: `write "Malmö"`, + `{"format":"{digits(3)}"}`: `write "{digits(3)}"`, + `[{"format":"a","weight":1},"b"]`: "weight 1", + `{"format":"{x}","x":"v","repeat":1}`: "repeat 1", + `{"format":"{x}","x":"v","separator":","}`: "separator", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) + } + } + for _, ok := range []string{`[{"format":"a","weight":2},"b"]`, `{"format":"ab","repeat":2}`, `{"format":"{x}","x":"v"}`, `"{digits(3)}"`} { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v", ok, err) + } + } +} diff --git a/reference_test.go b/reference_test.go index 3ed55ef..7ac7b52 100644 --- a/reference_test.go +++ b/reference_test.go @@ -6,8 +6,8 @@ import "testing" // pulls a value from another via a {..path} reference resolved from the data root. func TestRootReferenceAcrossFolders(t *testing.T) { dir := writeData(t, map[string]string{ - "en_US/person": `["Pat Smith"]`, - "sv_SE/greeting": `{"format":"Hej, {..en_US.person}!"}`, + "en_US/person": `"Pat Smith"`, + "sv_SE/greeting": `"Hej, {..en_US.person}!"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "sv_SE.greeting"); got != "Hej, Pat Smith!" { @@ -19,8 +19,8 @@ func TestRootReferenceAcrossFolders(t *testing.T) { // a single-variant choice and then a template field (..who.last). func TestReferenceIntoAField(t *testing.T) { dir := writeData(t, map[string]string{ - "who": `[{"format":"{first} {last}","first":["Ada"],"last":["Byron"]}]`, - "card": `{"format":"signed {..who.last}"}`, + "who": `{"format":"{first} {last}","first":"Ada","last":"Byron"}`, + "card": `"signed {..who.last}"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "card"); got != "signed Byron" { @@ -32,8 +32,8 @@ func TestReferenceIntoAField(t *testing.T) { // alternation, so a field and a cross-file value share one slot. func TestReferenceInAlternation(t *testing.T) { dir := writeData(t, map[string]string{ - "far": `["X"]`, - "near": `{"format":"{here|..far}","here":["H"]}`, + "far": `"X"`, + "near": `{"format":"{here|..far}","here":"H"}`, }) f := newGenerator(t, dir, WithSeed(2)) seen := map[string]bool{} @@ -48,8 +48,8 @@ func TestReferenceInAlternation(t *testing.T) { // TestReferenceCombinesLoadedPaths is the point of references over the merge // model: data layered from two dirs can point at each other through the root. func TestReferenceCombinesLoadedPaths(t *testing.T) { - a := writeData(t, map[string]string{"en_US/word": `["river"]`}) - b := writeData(t, map[string]string{"mine/slug": `{"format":"the-{..en_US.word}"}`}) + a := writeData(t, map[string]string{"en_US/word": `"river"`}) + b := writeData(t, map[string]string{"mine/slug": `"the-{..en_US.word}"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "mine.slug"); got != "the-river" { t.Fatalf("slug = %q, want the-river", got) @@ -60,9 +60,9 @@ func TestReferenceCombinesLoadedPaths(t *testing.T) { // linking order cannot matter. func TestReferenceChain(t *testing.T) { dir := writeData(t, map[string]string{ - "a": `{"format":"{..b}"}`, - "b": `{"format":"{..c}"}`, - "c": `["deep"]`, + "a": `"{..b}"`, + "b": `"{..c}"`, + "c": `"deep"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "a"); got != "deep" { @@ -74,37 +74,37 @@ func TestReferenceChain(t *testing.T) { // fails at load, never at a random render. func TestReferenceErrors(t *testing.T) { cases := map[string]map[string]string{ - "missing target": {"card": `{"format":"{..nope.gone}"}`}, - "folder target": {"en_US/word": `["w"]`, "card": `{"format":"{..en_US}"}`}, + "missing target": {"card": `"{..nope.gone}"`}, + "folder target": {"en_US/word": `"w"`, "card": `"{..en_US}"`}, "multi-variant on the path": { - "who": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"]}]`, - "card": `{"format":"{..who.f}"}`, + "who": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, + "card": `"{..who.f}"`, }, - "empty reference path": {"card": `{"format":"{..}"}`}, + "empty reference path": {"card": `"{..}"`}, // A reference that leads back to its own value never terminates at render, // so New must reject the cycle up front (direct, mutual, or chained). - "direct cycle": {"a": `{"format":"x{..a}"}`}, - "mutual cycle": {"a": `{"format":"{..b}"}`, "b": `{"format":"{..a}"}`}, - "chain cycle": {"a": `{"format":"{..b}"}`, "b": `{"format":"{..c}"}`, "c": `{"format":"{..a}"}`}, + "direct cycle": {"a": `"x{..a}"`}, + "mutual cycle": {"a": `"{..b}"`, "b": `"{..a}"`}, + "chain cycle": {"a": `"{..b}"`, "b": `"{..c}"`, "c": `"{..a}"`}, // calc renders its operands, so a cycle through one must be caught too. - "calc operand cycle": {"x": `{"format":"{calc(y)}","y":[{"format":"{..x}"}]}`}, + "calc operand cycle": {"x": `{"format":"{calc(y)}","y":"{..x}"}`}, // A field its parent's format never renders is still reachable by dot path, // so a cycle hiding in one must fail at New rather than at render. - "cycle in an unrendered field": {"cat": `{"format":"hi","x":{"format":"{..cat.x}"}}`}, + "cycle in an unrendered field": {"cat": `{"format":"hi","x":"{..cat.x}"}`}, "mutual cycle between unrendered fields": { - "cat": `{"format":"hi","x":{"format":"{..cat.y}"},"y":{"format":"{..cat.x}"}}`, + "cat": `{"format":"hi","x":"{..cat.y}","y":"{..cat.x}"}`, }, "cycle in an unrendered field of a choice arm": { - "cat": `[{"format":"hi","x":{"format":"{..cat.x}"}}]`, + "cat": `{"format":"hi","x":"{..cat.x}"}`, }, // The shipped layout puts categories in folders, so a cycle one level down // is the common case, not an edge case. - "cycle in a subfolder": {"sv_SE/a": `{"format":"x{..sv_SE.a}"}`}, - "mutual cycle within a subfolder": {"sv_SE/a": `{"format":"{..sv_SE.b}"}`, "sv_SE/b": `{"format":"{..sv_SE.a}"}`}, - "mutual cycle across two folders": {"en_US/a": `{"format":"{..sv_SE.b}"}`, "sv_SE/b": `{"format":"{..en_US.a}"}`}, + "cycle in a subfolder": {"sv_SE/a": `"x{..sv_SE.a}"`}, + "mutual cycle within a subfolder": {"sv_SE/a": `"{..sv_SE.b}"`, "sv_SE/b": `"{..sv_SE.a}"`}, + "mutual cycle across two folders": {"en_US/a": `"{..sv_SE.b}"`, "sv_SE/b": `"{..en_US.a}"`}, // ".." is reserved for bound references, so an authored key using it would // name a node nothing can reach and nothing would validate. - "field key using the reference prefix": {"cat": `{"format":"hi","..x":{"format":"{..nope}"}}`}, + "field key using the reference prefix": {"cat": `{"format":"hi","..x":"{..nope}"}`}, } for name, files := range cases { if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { @@ -119,10 +119,10 @@ func TestReferenceErrors(t *testing.T) { // starts with "." — it is skipped, not rejected. func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { f := newGenerator(t, writeData(t, map[string]string{ - "sv_SE/ok": `["fine"]`, - "sv_SE/..bad": `{"format":"{..nope}"}`, - "sv_SE/..y/ct": `{"format":"{..nope}"}`, - ".git/config": `["not data"]`, + "sv_SE/ok": `"fine"`, + "sv_SE/..bad": `"{..nope}"`, + "sv_SE/..y/ct": `"{..nope}"`, + ".git/config": `"not data"`, }), WithSeed(1)) if got := f.List(); len(got) != 1 || got[0] != "sv_SE.ok" { t.Fatalf("List() = %v, want only sv_SE.ok", got) @@ -133,7 +133,7 @@ func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { // over-rejecting: a field the format never renders may point back at its own // category, which terminates, and stays renderable by path. func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { - dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":{"format":"see {..cat}"}}`}) + dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":"see {..cat}"}`}) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "cat"); got != "hi" { t.Fatalf("cat = %q, want hi", got) @@ -149,9 +149,9 @@ func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { func TestNewErrorIsDeterministic(t *testing.T) { cases := map[string]map[string]string{ "three bad references": { - "a": `{"format":"{..nope.one}"}`, - "b": `{"format":"{..nope.two}"}`, - "c": `{"format":"{..nope.three}"}`, + "a": `"{..nope.one}"`, + "b": `"{..nope.two}"`, + "c": `"{..nope.three}"`, }, "two bad fields in one template": { "cat": `{"format":"hi","aaa":{"no":1},"zzz":{"no":2}}`, @@ -188,12 +188,12 @@ func TestNewErrorPathIsCanonical(t *testing.T) { }{ { "cycle inside a choice arm", - map[string]string{"cat": `[{"format":"hi","x":{"format":"{..cat.x}"}}]`}, + map[string]string{"cat": `{"format":"hi","x":"{..cat.x}"}`}, "fejkdata: reference cycle: cat.x -> ..cat.x", }, { "bad reference reached through another reference", - map[string]string{"a": `{"format":"{..b}"}`, "b": `{"format":"{..nope}"}`}, + map[string]string{"a": `"{..b}"`, "b": `"{..nope}"`}, `fejkdata: b: reference {..nope}: no entry "nope"`, }, } diff --git a/shipped_data_test.go b/shipped_data_test.go index 660fccd..c1d97e3 100644 --- a/shipped_data_test.go +++ b/shipped_data_test.go @@ -26,7 +26,7 @@ func TestNewDefaultsToShippedData(t *testing.T) { } func TestWithDataPathLayersOverShipped(t *testing.T) { - dir := writeData(t, map[string]string{"sv_SE/word": `["only-mine"]`, "greeting": `["hej"]`}) + dir := writeData(t, map[string]string{"sv_SE/word": `"only-mine"`, "greeting": `"hej"`}) f, err := New(WithDataPath(dir), WithSeed(1)) if err != nil { t.Fatalf("New = %v", err) @@ -43,7 +43,7 @@ func TestWithDataPathLayersOverShipped(t *testing.T) { } func TestUserDataMayReferenceShipped(t *testing.T) { - dir := writeData(t, map[string]string{"greeting": `{"format":"Hej {..sv_SE.person}!"}`}) + dir := writeData(t, map[string]string{"greeting": `"Hej {..sv_SE.person}!"`}) f, err := New(WithDataPath(dir), WithSeed(1)) if err != nil { t.Fatalf("New = %v", err) @@ -61,7 +61,7 @@ func TestWithoutShippedDataNeedsASource(t *testing.T) { } func TestWithoutShippedDataListsOnlyOwn(t *testing.T) { - dir := writeData(t, map[string]string{"greeting": `["hej"]`}) + dir := writeData(t, map[string]string{"greeting": `"hej"`}) f := newGenerator(t, dir, WithSeed(1)) if got := f.List(); !reflect.DeepEqual(got, []string{"greeting"}) { t.Errorf("List() = %v, want only the loaded dir", got) @@ -70,10 +70,10 @@ func TestWithoutShippedDataListsOnlyOwn(t *testing.T) { func TestWithDataFS(t *testing.T) { fsys := fstest.MapFS{ - "greeting.json": {Data: []byte(`["hej"]`)}, - "nested/x.json": {Data: []byte(`{"format":"{y}","y":["z"]}`)}, - ".hidden.json": {Data: []byte(`["ignored"]`)}, - "broken/no.json": {Data: []byte(`["x"]`)}, + "greeting.json": {Data: []byte(`"hej"`)}, + "nested/x.json": {Data: []byte(`{"format":"{y}","y":"z"}`)}, + ".hidden.json": {Data: []byte(`"ignored"`)}, + "broken/no.json": {Data: []byte(`"x"`)}, } f, err := New(WithoutShippedData(), WithDataFS(fsys), WithSeed(1)) if err != nil { diff --git a/template_stability_test.go b/template_stability_test.go index 67dc7f1..4e410ed 100644 --- a/template_stability_test.go +++ b/template_stability_test.go @@ -7,16 +7,16 @@ import "testing" // cannot quietly shift the rng stream that reproducibility depends on. func TestSeededOutputIsStable(t *testing.T) { dir := writeData(t, map[string]string{ - "alt": `{"format":"{a|b}","a":["A"],"b":["B"]}`, + "alt": `{"format":"{a|b}","a":"A","b":"B"}`, "calc": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, - "classes": `{"format":"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"}`, - "escapes": `{"format":"01Aa#{x}","x":["!"]}`, - "funcs": `{"format":"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"}`, - "nested": `{"format":"{outer}","outer":[{"format":"{inner}-{digits(2)}","inner":["i"]}]}`, - "ref": `{"format":"see {..alt}"}`, + "classes": `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`, + "escapes": `{"format":"01Aa#{x}","x":"!"}`, + "funcs": `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`, + "nested": `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`, + "ref": `"see {..alt}"`, "repeat": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`, - "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":["012345678901234"],"e":["123456789012"],"m":["12345678"]}`, - "weights": `[{"format":"big","weight":9},{"format":"tiny","weight":1}]`, + "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`, + "weights": `[{"format":"big","weight":9},"tiny"]`, }) want := map[string][]string{ "alt": {"A", "B", "A", "A"}, diff --git a/template_test.go b/template_test.go index cdd6420..1f8a59a 100644 --- a/template_test.go +++ b/template_test.go @@ -74,7 +74,7 @@ func TestClassBuiltins(t *testing.T) { } func TestTextIsLiteral(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":["A"]}`); got != "100 Main St #1 A" { + if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" { t.Fatalf("text = %q, want it verbatim", got) } } @@ -82,21 +82,21 @@ func TestTextIsLiteral(t *testing.T) { func TestBraceEscapes(t *testing.T) { f := engine(1) for src, want := range map[string]string{ - `{"format":"{{","x":["v"]}`: "{", - `{"format":"}}","x":["v"]}`: "}", - `{"format":"{{x}}","x":["v"]}`: "{x}", - `{"format":"{{{x}}}","x":["v"]}`: "{v}", - `{"format":"a{{{{b}}}}","x":["v"]}`: "a{{b}}", + `{"format":"{{","x":"v"}`: "{", + `{"format":"}}","x":"v"}`: "}", + `{"format":"{{x}}","x":"v"}`: "{x}", + `{"format":"{{{x}}}","x":"v"}`: "{v}", + `{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}", } { if got := mustRender(t, f, src); got != want { t.Errorf("render(%s) = %q, want %q", src, got, want) } } for src, want := range map[string]string{ - `{"format":"x}y"}`: "}}", - `{"format":"}"}`: "}}", - `{"format":"{a{b}","a":["Q"]}`: "'{'", - `{"format":"{x"}`: "unterminated", + `"x}y"`: "}}", + `"}"`: "}}", + `{"format":"{a{b}","a":"Q"}`: "'{'", + `"{x"`: "unterminated", } { if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) @@ -105,7 +105,7 @@ func TestBraceEscapes(t *testing.T) { } func TestTokenSubstitution(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{x}sson","x":["Erik"]}`); got != "Eriksson" { + if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" { t.Fatalf("token = %q, want Eriksson", got) } } @@ -114,7 +114,7 @@ func TestAlternationPicksOneField(t *testing.T) { f := engine(3) seen := map[string]bool{} for i := 0; i < 100; i++ { - seen[mustRender(t, f, `{"format":"{a|b}","a":["A"],"b":["B"]}`)] = true + seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true } if !seen["A"] || !seen["B"] || len(seen) != 2 { t.Fatalf("alternation produced %v, want both A and B", seen) @@ -134,7 +134,7 @@ func TestWeightSkewsDistribution(t *testing.T) { f := engine(9) heavy := 0 for i := 0; i < 1000; i++ { - if mustRender(t, f, `[{"format":"H","weight":10},{"format":"L"}]`) == "H" { + if mustRender(t, f, `[{"format":"H","weight":10},"L"]`) == "H" { heavy++ } } @@ -173,10 +173,10 @@ func TestFunctionTokenLuhn(t *testing.T) { // buffer, so a value is never re-rendered. Bodies are escaped to fix input. f := engine(1) cases := map[string]string{ - `{"format":"811218987{luhn()}"}`: "8112189876", // personnummer body - `{"format":"7992739871{luhn()}"}`: "79927398713", // classic Luhn vector - `{"format":"811218-987{luhn()}"}`: "811218-9876", // '-' skipped, kept - `{"format":"{n}{luhn()}","n":["811218987"]}`: "8112189876", // over a rendered token + `"811218987{luhn()}"`: "8112189876", // personnummer body + `"7992739871{luhn()}"`: "79927398713", // classic Luhn vector + `"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept + `{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -189,7 +189,7 @@ func TestRecursionHasNoDepthLimit(t *testing.T) { // Build {format:{a}, a:[{format:{a}, a:[ ... "deep" ]]}} 50 levels deep. tmpl := `"deep"` for i := 0; i < 50; i++ { - tmpl = `{"format":"{a}","a":[` + tmpl + `]}` + tmpl = `{"format":"{a}","a":` + tmpl + `}` } if got := mustRender(t, engine(1), tmpl); got != "deep" { t.Fatalf("deep recursion = %q, want deep", got) @@ -200,51 +200,56 @@ func TestCompileErrors(t *testing.T) { // Every structural problem is caught up front, at compile/New time, never // deferred to a random render that happens to hit the bad branch. for _, bad := range []string{ - `{"x":["Q"]}`, // object without "format" - `{"format":"{y}","x":1}`, // a field is a bare number - `[1, 2]`, // a choice of numbers - `5`, // unsupported node type - `[]`, // empty choice - `{"format":"{x"}`, // unterminated brace - `{"format":"x}y"}`, // a lone } must be written }} - `{"format":"{a{b}","a":["Q"]}`, // a brace inside a token - `"{x}"`, // a bare string has no fields to name - `{"format":"{digits(0)}"}`, // count must be positive - `{"format":"{upper(x)}"}`, // count must be an integer - `{"format":"{lower()}"}`, // wrong arity - `{"format":"{y}","x":["Q"]}`, // token names a missing field - `{"format":"{}"}`, // empty token name - `{"format":"{a|}","a":["Q"]}`, // empty alternation segment - `[{"format":"A","weight":-1},{"format":"B"}]`, // negative weight + `{"x":"Q"}`, // object without "format" + `{"format":"{y}","x":1}`, // a field is a bare number + `[1, 2]`, // a choice of numbers + `5`, // unsupported node type + `[]`, // empty choice + `"{x"`, // unterminated brace + `"x}y"`, // a lone } must be written }} + `["x"]`, // a one-item choice is its item + `["a","a"]`, // a repeated item is a weight + `{"format":"x"}`, // an object holding only a format is a string + `[{"format":"a","weight":1},"b"]`, // weight 1 is the default + `{"format":"{x}","x":"v","repeat":1}`, // repeat 1 is the default + `{"format":"{a{b}","a":"Q"}`, // a brace inside a token + `"{x}"`, // a bare string has no fields to name + `"{digits(0)}"`, // count must be positive + `"{upper(x)}"`, // count must be an integer + `"{lower()}"`, // wrong arity + `{"format":"{y}","x":"Q"}`, // token names a missing field + `"{}"`, // empty token name + `{"format":"{a|}","a":"Q"}`, // empty alternation segment + `[{"format":"A","weight":-1},"B"]`, // negative weight `[{"format":"A","weight":0},{"format":"B","weight":0}]`, // weights sum to zero `[{"format":"A","weight":1e308},{"format":"B","weight":1e308}]`, // weights overflow to +Inf - `[{"format":"A","weight":"heavy"}]`, // non-numeric weight + `{"format":"A","weight":"heavy"}`, // non-numeric weight `{"format":"x","repeat":0}`, // repeat below 1 `{"format":"x","repeat":-2}`, // negative repeat `{"format":"x","repeat":1.5}`, // non-integer repeat `{"format":"x","repeat":"two"}`, // non-numeric repeat `{"format":"x","repeat":2,"separator":5}`, // non-string separator - `{"format":"{nope()}"}`, // unknown function - `{"format":"{luhn(x)}"}`, // function given args it takes none of - `{"format":"{luhn(}"}`, // malformed function token - `{"format":"{int(1)}"}`, // wrong arity - `{"format":"{int(a,b)}"}`, // non-integer args - `{"format":"{int(5,1)}"}`, // min > max - `{"format":"{hex(0)}"}`, // count must be positive - `{"format":"{nanoid(-1)}"}`, // negative count - `{"format":"{base64(0)}"}`, // count must be positive - `{"format":"{float(1,2)}"}`, // wrong arity - `{"format":"{float(1,2,-1)}"}`, // negative decimals - `{"format":"{iban(US)}"}`, // unsupported country - `{"format":"{seq(a,b)}"}`, // seq takes at most one name - `{"format":"{calc()}"}`, // calc needs an expression - `{"format":"{calc(1 +)}"}`, // dangling operator - `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis - `{"format":"{calc(1 2)}"}`, // two operands, no operator - `{"format":"{calc(price)}"}`, // operand names no field - `{"format":"{calc(1, 2, 3)}"}`, // too many args - `{"format":"{calc(1, x)}"}`, // decimals arg not an integer - `{"format":"{calc(1, -1)}"}`, // decimals negative + `"{nope()}"`, // unknown function + `"{luhn(x)}"`, // function given args it takes none of + `"{luhn(}"`, // malformed function token + `"{int(1)}"`, // wrong arity + `"{int(a,b)}"`, // non-integer args + `"{int(5,1)}"`, // min > max + `"{hex(0)}"`, // count must be positive + `"{nanoid(-1)}"`, // negative count + `"{base64(0)}"`, // count must be positive + `"{float(1,2)}"`, // wrong arity + `"{float(1,2,-1)}"`, // negative decimals + `"{iban(US)}"`, // unsupported country + `"{seq(a,b)}"`, // seq takes at most one name + `"{calc()}"`, // calc needs an expression + `"{calc(1 +)}"`, // dangling operator + `"{calc((1 + 2)}"`, // unbalanced parenthesis + `"{calc(1 2)}"`, // two operands, no operator + `"{calc(price)}"`, // operand names no field + `"{calc(1, 2, 3)}"`, // too many args + `"{calc(1, x)}"`, // decimals arg not an integer + `"{calc(1, -1)}"`, // decimals negative } { if _, err := compile(parse(t, bad)); err == nil { t.Errorf("compile(%s) = nil error, want error", bad) @@ -255,7 +260,7 @@ func TestCompileErrors(t *testing.T) { func TestFakePathNavigation(t *testing.T) { f := engine(1) f.categories = map[string]node{ - "addr": compiled(t, `[{"format":"{street}","street":["Main"]}]`), + "addr": compiled(t, `{"format":"{street}","street":"Main"}`), } if got, err := f.Fake("addr"); err != nil || !strings.Contains(got, "Main") { t.Fatalf("Fake(addr) = %q, %v", got, err) @@ -288,7 +293,7 @@ func TestGrowIsALowerBound(t *testing.T) { "9{d}{luhn()}", "{a|b} and {a|b}", } { - src := `{"format":` + quote(format) + `,"x":["1"],"a":["A"],"b":["B"],"d":["012345678901234"]}` + src := `{"format":` + quote(format) + `,"x":"1","a":"A","b":"B","d":"012345678901234"}` tmpl, ok := compiled(t, src).(*template) if !ok { t.Fatalf("format %q did not compile to a template", format) -- 2.52.0 From 054f99c94f1e136d3c7eeb80744265c3ac5481aa Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:38:59 +0200 Subject: [PATCH 08/38] One spelling per shape: reject a one-item choice, a repeated item and an inert object; reshape the data --- .gitignore | 1 + README.md | 49 +++++++++++---------- data/en_US/address.json | 41 ++++++++--------- data/en_US/company.json | 26 +++++------ data/en_US/date.json | 28 ++++++------ data/en_US/email.json | 55 +++++++++++------------ data/en_US/ip.json | 10 +++-- data/en_US/person.json | 23 +++------- data/en_US/phone.json | 8 ++-- data/en_US/price.json | 31 +++++-------- data/en_US/sentence.json | 4 +- data/en_US/ssn.json | 4 +- data/en_US/time.json | 14 +++--- data/en_US/url.json | 29 ++++++++----- data/en_US/username.json | 11 +++-- data/en_US/version.json | 14 +++--- data/misc/car.json | 16 +++---- data/misc/coordinate.json | 8 +--- data/misc/country.json | 40 ++++++++--------- data/misc/creditcard.json | 19 ++++++-- data/misc/currency.json | 32 +++++++------- data/misc/emoji.json | 7 +-- data/misc/httpstatus.json | 32 +++++++------- data/misc/language.json | 32 +++++++------- data/misc/mac.json | 4 +- data/misc/mimetype.json | 30 ++++++------- data/misc/objectid.json | 4 +- data/misc/timezone.json | 30 +------------ data/misc/useragent.json | 11 +---- data/misc/uuid.json | 7 +-- data/sv_SE/address.json | 46 +++++++++----------- data/sv_SE/company.json | 26 +++++------ data/sv_SE/date.json | 28 ++++++------ data/sv_SE/email.json | 55 +++++++++++------------ data/sv_SE/ip.json | 10 +++-- data/sv_SE/person.json | 73 ++++++++++++++----------------- data/sv_SE/phone.json | 12 ++--- data/sv_SE/price.json | 24 +++++----- data/sv_SE/sentence.json | 2 +- data/sv_SE/ssn.json | 47 ++++++++++---------- data/sv_SE/time.json | 25 +++++++---- data/sv_SE/url.json | 29 ++++++++----- data/sv_SE/username.json | 11 +++-- data/sv_SE/version.json | 14 +++--- fejkdata.go | 12 ++--- format-migration/reshape.py | 87 +++++++++++++++++++++++++++++++++++++ node.go | 70 +++++++++++++++++++++-------- reference.go | 16 +++---- render.go | 6 +-- 49 files changed, 620 insertions(+), 593 deletions(-) create mode 100755 format-migration/reshape.py diff --git a/.gitignore b/.gitignore index 71fd48f..28f4f54 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .claude *.out +__pycache__/ todo.md \ No newline at end of file diff --git a/README.md b/README.md index a65c822..5c647bb 100644 --- a/README.md +++ b/README.md @@ -219,10 +219,11 @@ within a choice: ] ``` -Only template (object) nodes carry `weight` — a bare string or nested array in a -choice always counts as `1`. Weights are checked when you create the generator: a -negative, non-numeric, or all-zero set is rejected at `New`, so a typo fails -fast instead of silently skewing output. +A bare string or nested array in a choice counts as `1`; to weight a string, +write it as `{ "format": "AB", "weight": 3 }`. A repeated item is a load error +naming that spelling, and so is a `weight` of `1`. Weights are checked when you +create the generator: a negative, non-numeric, or all-zero set is rejected at +`New`, so a typo fails fast instead of silently skewing output. **Repeat.** A template node may carry a `repeat` (default `1`) to render its `format` that many times — each render an independent pick — joined by @@ -232,14 +233,16 @@ fast instead of silently skewing output. { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -This yields e.g. `bar foo baz`. `repeat` must be a positive integer and +This yields e.g. `bar foo baz`. `repeat` must be an integer above `1` and `separator` a string, both checked at `New`. `format`, `weight`, `repeat` and `separator` are the only options; **any other key is a field**. So write `seperator` and you get a field by that name while the option stays unset. `New` rejects an option that cannot take effect — a -`separator` without a `repeat` above 1, a `weight` outside a choice — and a name -using a character the grammars reserve: `.` separates the segments of a path, `|` +`separator` without a `repeat`, a `weight` outside a choice, a `weight` or +`repeat` of `1` — an object holding only a `format` (that is a string; write it), +a one-item choice (that is its item; write it), and a name using a character the +grammars reserve: `.` separates the segments of a path, `|` the arms of a token, `(` opens a function call and `{` `}` delimit the token, so a name carrying one is a name no format could ever spell. An empty name goes the same way — it is no path segment at all, so `List` never offers it. That holds for a @@ -337,7 +340,7 @@ data dirs: ``` renders e.g. `Hej, Pat Smith!`. References are bound when you create the generator, so -a path that is unknown, names a folder, or steps through a multi-variant choice +a path that is unknown, names a folder, or steps through a choice fails at `New`. A reference that leads back to its own value (directly, mutually, or through a chain) is a cycle that would never finish rendering, so it too is rejected at `New`. @@ -401,7 +404,7 @@ The binding lasts for one expansion, so each `repeat` iteration draws again and nested template keeps its own. A field no dotted token addresses is unaffected: `{word} {word}` still draws twice. -`New` checks a path the way `Fake` resolves one: every variant of a multi-variant +`New` checks a path the way `Fake` resolves one: every variant of a choice must carry the whole path, so a row missing a field is named at load: ```text @@ -435,21 +438,19 @@ A lone `}` is a load error naming `}}`. **Putting it together** (`person.json`): ```json -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Anna", "Astrid", "Elin"], - "malefirst": ["Anders", "Erik", "Gustav"], - "last": [ - { "format": "{first}sson", "first": ["Ander", "Erik", "Karl"] }, - ["Berg", "von Flemming"] - ], - "prefix": [ - "", - { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 } - ] - } -] +{ + "format": "{prefix}{femalefirst|malefirst} {last}", + "femalefirst": ["Anna", "Astrid", "Elin"], + "malefirst": ["Anders", "Erik", "Gustav"], + "last": [ + { "format": "{first}sson", "first": ["Ander", "Erik", "Karl"] }, + ["Berg", "von Flemming"] + ], + "prefix": [ + "", + { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 } + ] +} ``` This yields e.g. `Anna Eriksson`, `Erik Berg`, or rarely `dr Astrid von Flemming`. diff --git a/data/en_US/address.json b/data/en_US/address.json index bf71c64..d48eebe 100644 --- a/data/en_US/address.json +++ b/data/en_US/address.json @@ -1,24 +1,17 @@ -[ - { - "format": "{street-number} {street}\n{locality}, {region} {postal-code}", - "street": [ - { - "format": "{name} {suffix}", - "name": ["Adams", "Ashby", "Aspen", "Bay", "Birch", "Bridge", "Cedar", "Chestnut", "Church", "Clark", "Cypress", "Dogwood", "Elm", "Forest", "Franklin", "Garden", "Grove", "Hawthorn", "Hickory", "Highland", "Jackson", "Jefferson", "Juniper", "Lake", "Laurel", "Liberty", "Lincoln", "Madison", "Magnolia", "Maple", "Market", "Meadow", "Mill", "Oak", "Park", "Pine", "Poplar", "Prospect", "Ridge", "River", "Spruce", "Sunset", "Sycamore", "Union", "Walnut", "Washington", "Willow", "Wilson"], - "suffix": ["Avenue", "Boulevard", "Circle", "Court", "Drive", "Lane", "Place", "Road", "Street", "Terrace", "Trail", "Way"] - } - ], - "street-number": [ - { "format": "{int(10,99)}" }, - { "format": "{int(100,999)}", "weight": 2 }, - { "format": "{int(1000,9999)}", "weight": 0.5 }, - { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 } - ], - "locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"], - "region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"], - "postal-code": [ - { "format": "{int(10000,99999)}" }, - { "format": "{int(10000,99999)}-{digits(4)}", "weight": 0.3 } - ] - } -] +{ + "format": "{street-number} {street}\n{locality}, {region} {postal-code}", + "street": { + "format": "{name} {suffix}", + "name": ["Adams", "Ashby", "Aspen", "Bay", "Birch", "Bridge", "Cedar", "Chestnut", "Church", "Clark", "Cypress", "Dogwood", "Elm", "Forest", "Franklin", "Garden", "Grove", "Hawthorn", "Hickory", "Highland", "Jackson", "Jefferson", "Juniper", "Lake", "Laurel", "Liberty", "Lincoln", "Madison", "Magnolia", "Maple", "Market", "Meadow", "Mill", "Oak", "Park", "Pine", "Poplar", "Prospect", "Ridge", "River", "Spruce", "Sunset", "Sycamore", "Union", "Walnut", "Washington", "Willow", "Wilson"], + "suffix": ["Avenue", "Boulevard", "Circle", "Court", "Drive", "Lane", "Place", "Road", "Street", "Terrace", "Trail", "Way"] + }, + "street-number": [ + "{int(10,99)}", + { "format": "{int(100,999)}", "weight": 2 }, + { "format": "{int(1000,9999)}", "weight": 0.5 }, + { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 } + ], + "locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"], + "region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"], + "postal-code": ["{int(10000,99999)}", { "format": "{int(10000,99999)}-{digits(4)}", "weight": 0.3 }] +} diff --git a/data/en_US/company.json b/data/en_US/company.json index ab045a4..5b27b76 100644 --- a/data/en_US/company.json +++ b/data/en_US/company.json @@ -1,14 +1,12 @@ -[ - { - "format": "{base} {suffix}", - "base": [ - { - "format": "{a}{b}", - "a": ["Acme", "Apex", "Atlas", "Aurora", "Blue", "Bright", "Cedar", "Crest", "Delta", "Eagle", "Ever", "Fair", "Falcon", "Iron", "Maple", "North", "Onyx", "Pioneer", "Polar", "Prime", "Quartz", "River", "Silver", "Solar", "Summit", "Sun", "Swift", "True", "Vertex", "West"], - "b": ["bridge", "core", "field", "flow", "forge", "gate", "grid", "hub", "line", "logic", "mark", "nest", "path", "point", "scape", "soft", "span", "spark", "stone", "tech", "ware", "wave", "works", "yard"] - }, - ["Atlas Industries", "Beacon Labs", "Cedar & Vine", "Compass Group", "Evergreen Trading", "Harbor Logistics", "Hudson Partners", "Liberty Holdings", "Meridian Capital", "Northgate Studios", "Pinnacle Systems", "Riverside", "Sequoia Foods", "Smith & Sons", "Stonebridge Ventures", "Sunrise Bakery", "Wexford Holdings"] - ], - "suffix": ["Co.", "Corp.", "Group", "Inc.", "Inc.", "LLC", "LLC", "Ltd."] - } -] +{ + "format": "{base} {suffix}", + "base": [ + { + "format": "{a}{b}", + "a": ["Acme", "Apex", "Atlas", "Aurora", "Blue", "Bright", "Cedar", "Crest", "Delta", "Eagle", "Ever", "Fair", "Falcon", "Iron", "Maple", "North", "Onyx", "Pioneer", "Polar", "Prime", "Quartz", "River", "Silver", "Solar", "Summit", "Sun", "Swift", "True", "Vertex", "West"], + "b": ["bridge", "core", "field", "flow", "forge", "gate", "grid", "hub", "line", "logic", "mark", "nest", "path", "point", "scape", "soft", "span", "spark", "stone", "tech", "ware", "wave", "works", "yard"] + }, + ["Atlas Industries", "Beacon Labs", "Cedar & Vine", "Compass Group", "Evergreen Trading", "Harbor Logistics", "Hudson Partners", "Liberty Holdings", "Meridian Capital", "Northgate Studios", "Pinnacle Systems", "Riverside", "Sequoia Foods", "Smith & Sons", "Stonebridge Ventures", "Sunrise Bakery", "Wexford Holdings"] + ], + "suffix": ["Co.", "Corp.", "Group", { "format": "Inc.", "weight": 2 }, { "format": "LLC", "weight": 2 }, "Ltd."] +} diff --git a/data/en_US/date.json b/data/en_US/date.json index 05e6167..cf3b880 100644 --- a/data/en_US/date.json +++ b/data/en_US/date.json @@ -1,15 +1,13 @@ -[ - { - "format": "{month}/{day}/{year}", - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], - "year": [ - { "format": "197{digits(1)}" }, - { "format": "198{digits(1)}" }, - { "format": "199{digits(1)}", "weight": 2 }, - { "format": "200{digits(1)}", "weight": 3 }, - { "format": "201{digits(1)}", "weight": 3 }, - { "format": "202{digits(1)}", "weight": 2 } - ] - } -] +{ + "format": "{month}/{day}/{year}", + "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], + "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], + "year": [ + "197{digits(1)}", + "198{digits(1)}", + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } + ] +} diff --git a/data/en_US/email.json b/data/en_US/email.json index efd9e8f..a24823b 100644 --- a/data/en_US/email.json +++ b/data/en_US/email.json @@ -1,29 +1,26 @@ -[ - { - "format": "{local}@{domain}", - "local": [ - { - "format": "{first}{sep}{last}{n}", - "weight": 3, - "first": ["alex", "amelia", "andrew", "anna", "ava", "benjamin", "carter", "charlotte", "chloe", "chris", "daniel", "david", "dylan", "elijah", "ella", "emily", "emma", "ethan", "evelyn", "gabriel", "grace", "hannah", "harper", "henry", "isaac", "isabella", "jack", "jacob", "james", "john", "joseph", "julia", "leah", "liam", "linda", "lily", "logan", "lucas", "luke", "mary", "mason", "matthew", "maya", "mia", "michael", "natalie", "noah", "nora", "oliver", "olivia", "owen", "patricia", "robert", "ryan", "samuel", "sarah", "sofia", "sophia", "thomas", "victoria", "william", "zoe"], - "last": ["adams", "allen", "anderson", "bailey", "baker", "bennett", "brooks", "brown", "campbell", "carter", "clark", "collins", "cook", "cooper", "davis", "edwards", "evans", "fisher", "flores", "foster", "garcia", "gonzalez", "gray", "green", "hall", "harris", "hayes", "hernandez", "hill", "howard", "hughes", "jackson", "jenkins", "johnson", "jones", "kelly", "king", "lee", "lewis", "long", "lopez", "martin", "martinez", "miller", "mitchell", "moore", "morgan", "morris", "murphy", "nelson", "parker", "perez", "perry", "phillips", "powell", "price", "reed", "reyes", "rivera", "roberts", "robinson", "rodriguez", "rogers", "ross", "russell", "sanchez", "sanders", "scott", "smith", "stewart", "sullivan", "taylor", "thomas", "thompson", "torres", "turner", "walker", "ward", "watson", "white", "williams", "wilson", "wood", "wright", "young"], - "sep": [".", ".", "_", "-", ""], - "n": ["", "", "", "1", "7", "21", "42", "88", "99", "2024"] - }, - { - "format": "{adj}{noun}{n}", - "weight": 2, - "adj": ["blue", "cool", "dark", "epic", "fast", "fluffy", "frosty", "funky", "fuzzy", "golden", "grumpy", "happy", "hyper", "jazzy", "lazy", "lucky", "mega", "mighty", "neon", "ninja", "quiet", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "spicy", "super", "swift", "turbo", "wild", "witty", "zany"], - "noun": ["badger", "banana", "comet", "dragon", "falcon", "ferret", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "ninja", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], - "n": ["", "", "7", "13", "14", "42", "77", "99", "123", "420", "2024"] - }, - { - "format": "{w}{n}", - "weight": 1, - "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] - } - ], - "domain": ["example.com", "example.org", "example.net", "example.io", "example.co", "example.dev", "example.app", "mail.example.com", "mail.example.org", "webmail.example.net", "inbox.example.com", "post.example.org", "demo.example.net", "test.example.org", "dev.example.io"] - } -] +{ + "format": "{local}@{domain}", + "local": [ + { + "format": "{first}{sep}{last}{n}", + "weight": 3, + "first": ["alex", "amelia", "andrew", "anna", "ava", "benjamin", "carter", "charlotte", "chloe", "chris", "daniel", "david", "dylan", "elijah", "ella", "emily", "emma", "ethan", "evelyn", "gabriel", "grace", "hannah", "harper", "henry", "isaac", "isabella", "jack", "jacob", "james", "john", "joseph", "julia", "leah", "liam", "linda", "lily", "logan", "lucas", "luke", "mary", "mason", "matthew", "maya", "mia", "michael", "natalie", "noah", "nora", "oliver", "olivia", "owen", "patricia", "robert", "ryan", "samuel", "sarah", "sofia", "sophia", "thomas", "victoria", "william", "zoe"], + "last": ["adams", "allen", "anderson", "bailey", "baker", "bennett", "brooks", "brown", "campbell", "carter", "clark", "collins", "cook", "cooper", "davis", "edwards", "evans", "fisher", "flores", "foster", "garcia", "gonzalez", "gray", "green", "hall", "harris", "hayes", "hernandez", "hill", "howard", "hughes", "jackson", "jenkins", "johnson", "jones", "kelly", "king", "lee", "lewis", "long", "lopez", "martin", "martinez", "miller", "mitchell", "moore", "morgan", "morris", "murphy", "nelson", "parker", "perez", "perry", "phillips", "powell", "price", "reed", "reyes", "rivera", "roberts", "robinson", "rodriguez", "rogers", "ross", "russell", "sanchez", "sanders", "scott", "smith", "stewart", "sullivan", "taylor", "thomas", "thompson", "torres", "turner", "walker", "ward", "watson", "white", "williams", "wilson", "wood", "wright", "young"], + "sep": [{ "format": ".", "weight": 2 }, "_", "-", ""], + "n": [{ "format": "", "weight": 3 }, "1", "7", "21", "42", "88", "99", "2024"] + }, + { + "format": "{adj}{noun}{n}", + "weight": 2, + "adj": ["blue", "cool", "dark", "epic", "fast", "fluffy", "frosty", "funky", "fuzzy", "golden", "grumpy", "happy", "hyper", "jazzy", "lazy", "lucky", "mega", "mighty", "neon", "ninja", "quiet", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "spicy", "super", "swift", "turbo", "wild", "witty", "zany"], + "noun": ["badger", "banana", "comet", "dragon", "falcon", "ferret", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "ninja", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "14", "42", "77", "99", "123", "420", "2024"] + }, + { + "format": "{w}{n}", + "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + } + ], + "domain": ["example.com", "example.org", "example.net", "example.io", "example.co", "example.dev", "example.app", "mail.example.com", "mail.example.org", "webmail.example.net", "inbox.example.com", "post.example.org", "demo.example.net", "test.example.org", "dev.example.io"] +} diff --git a/data/en_US/ip.json b/data/en_US/ip.json index e16c78a..c9bf575 100644 --- a/data/en_US/ip.json +++ b/data/en_US/ip.json @@ -2,9 +2,11 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}", "weight": 4 }, { "format": "1{digits(2)}", "weight": 2 }] + "o": [ + { "format": "{digits(1)}", "weight": 3 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "1{digits(2)}", "weight": 2 } + ] }, - { - "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" - } + "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" ] diff --git a/data/en_US/person.json b/data/en_US/person.json index c6ff20f..3377a48 100644 --- a/data/en_US/person.json +++ b/data/en_US/person.json @@ -1,16 +1,7 @@ -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"], - "malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"], - "last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"], - "prefix": [ - "", - { - "format": "{title} ", - "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], - "weight": 0.1 - } - ] - } -] +{ + "format": "{prefix}{femalefirst|malefirst} {last}", + "femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"], + "malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"], + "last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"], + "prefix": ["", { "format": "{title} ", "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], "weight": 0.1 }] +} diff --git a/data/en_US/phone.json b/data/en_US/phone.json index 20f65ab..b47298b 100644 --- a/data/en_US/phone.json +++ b/data/en_US/phone.json @@ -3,13 +3,13 @@ "format": "({area}) {exch}-{line}", "weight": 2, "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "{int(100,999)}" }], - "line": [{ "format": "{digits(4)}" }] + "exch": "{int(100,999)}", + "line": "{digits(4)}" }, { "format": "{area}-{exch}-{line}", "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "{int(100,999)}" }], - "line": [{ "format": "{digits(4)}" }] + "exch": "{int(100,999)}", + "line": "{digits(4)}" } ] diff --git a/data/en_US/price.json b/data/en_US/price.json index 84c3c87..2719fa0 100644 --- a/data/en_US/price.json +++ b/data/en_US/price.json @@ -1,19 +1,12 @@ -[ - { - "format": "${amt}.{cents}", - "amt": [ - { "format": "{int(1,9)}", "weight": 2 }, - { "format": "{int(10,99)}", "weight": 4 }, - { "format": "{int(100,999)}", "weight": 3 }, - { "format": "{int(1,9)},{digits(3)}", "weight": 1 }, - { "format": "{int(10,99)},{digits(3)}", "weight": 0.4 }, - { "format": "{int(100,999)},{digits(3)}", "weight": 0.1 } - ], - "cents": [ - { "format": "{digits(2)}", "weight": 3 }, - "49", - "95", - "99" - ] - } -] +{ + "format": "${amt}.{cents}", + "amt": [ + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + "{int(1,9)},{digits(3)}", + { "format": "{int(10,99)},{digits(3)}", "weight": 0.4 }, + { "format": "{int(100,999)},{digits(3)}", "weight": 0.1 } + ], + "cents": [{ "format": "{digits(2)}", "weight": 3 }, "49", "95", "99"] +} diff --git a/data/en_US/sentence.json b/data/en_US/sentence.json index d844cf6..2f02dfe 100644 --- a/data/en_US/sentence.json +++ b/data/en_US/sentence.json @@ -7,7 +7,7 @@ "noun": ["anchor", "archer", "beacon", "bridge", "captain", "cavern", "cottage", "ember", "engine", "falcon", "ferry", "forest", "fox", "garden", "glacier", "harbor", "hermit", "hollow", "island", "lantern", "lighthouse", "mariner", "meadow", "mountain", "orchard", "otter", "pilgrim", "prairie", "quarry", "raven", "ridge", "river", "sailor", "scholar", "signal", "sparrow", "stranger", "summit", "thicket", "tower", "traveler", "valley", "voyager", "wanderer", "willow", "woodland"], "verb": ["abandons", "anchors", "awakens", "befriends", "builds", "calms", "carries", "chases", "conceals", "crosses", "embraces", "follows", "gathers", "greets", "guards", "guides", "haunts", "heralds", "honors", "kindles", "leads", "mends", "mirrors", "outlasts", "ponders", "reveals", "salutes", "shapes", "shelters", "shields", "summons", "tends", "traces", "weathers"], "prep": ["above", "across", "against", "alongside", "amid", "around", "beneath", "beside", "between", "beyond", "near", "over", "past", "through", "toward", "under", "within"], - "end": [".", ".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 4 }, "!"] }, { "format": "{lead} {noun} {adv} {verb} {prep} {adj} {noun}{end}", @@ -18,7 +18,7 @@ "verb": ["builds", "carries", "chases", "crosses", "follows", "gathers", "guards", "guides", "leads", "mirrors", "outlasts", "reveals", "shapes", "shelters", "summons", "tends", "traces", "weathers"], "adj": ["ancient", "bright", "distant", "fierce", "fragile", "gentle", "golden", "hidden", "lonely", "quiet", "restless", "rugged", "silent", "stormy", "weary", "wild"], "prep": ["across", "against", "alongside", "amid", "beneath", "between", "beyond", "near", "over", "past", "through", "toward", "within"], - "end": [".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 3 }, "!"] }, { "format": "{q} {adj} {noun} {verb} {prep} {adj} {noun}?", diff --git a/data/en_US/ssn.json b/data/en_US/ssn.json index b114ca6..501335d 100644 --- a/data/en_US/ssn.json +++ b/data/en_US/ssn.json @@ -1,3 +1 @@ -[ - { "format": "{int(100,999)}-{digits(2)}-{digits(4)}" } -] +"{int(100,999)}-{digits(2)}-{digits(4)}" diff --git a/data/en_US/time.json b/data/en_US/time.json index 09fd32d..1c69ac9 100644 --- a/data/en_US/time.json +++ b/data/en_US/time.json @@ -1,8 +1,6 @@ -[ - { - "format": "{hour}:{minute} {ampm}", - "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], - "minute": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], - "ampm": ["AM", "PM"] - } -] +{ + "format": "{hour}:{minute} {ampm}", + "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + "minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "ampm": ["AM", "PM"] +} diff --git a/data/en_US/url.json b/data/en_US/url.json index d033d72..29a784b 100644 --- a/data/en_US/url.json +++ b/data/en_US/url.json @@ -1,12 +1,17 @@ -[ - { - "format": "https://{host}{path}", - "host": ["example.com", "www.example.com", "example.org", "www.example.org", "example.net", "example.io", "example.co", "blog.example.com", "blog.example.net", "shop.example.com", "store.example.org", "app.example.io", "api.example.com", "docs.example.org", "news.example.net", "mail.example.com"], - "path": [ - "", - "", - { "format": "/{seg}", "seg": ["about", "account", "blog", "careers", "cart", "categories", "checkout", "contact", "dashboard", "downloads", "events", "faq", "features", "help", "home", "items", "login", "news", "order", "pricing", "privacy", "products", "profile", "register", "search", "settings", "signup", "support", "terms", "welcome"] }, - { "format": "/{seg}/{sub}", "weight": 0.5, "seg": ["blog", "docs", "help", "products", "category", "user"], "sub": ["getting-started", "overview", "guide", "2024", "latest", "popular", "archive", "details"] } - ] - } -] +{ + "format": "https://{host}{path}", + "host": ["example.com", "www.example.com", "example.org", "www.example.org", "example.net", "example.io", "example.co", "blog.example.com", "blog.example.net", "shop.example.com", "store.example.org", "app.example.io", "api.example.com", "docs.example.org", "news.example.net", "mail.example.com"], + "path": [ + { "format": "", "weight": 2 }, + { + "format": "/{seg}", + "seg": ["about", "account", "blog", "careers", "cart", "categories", "checkout", "contact", "dashboard", "downloads", "events", "faq", "features", "help", "home", "items", "login", "news", "order", "pricing", "privacy", "products", "profile", "register", "search", "settings", "signup", "support", "terms", "welcome"] + }, + { + "format": "/{seg}/{sub}", + "weight": 0.5, + "seg": ["blog", "docs", "help", "products", "category", "user"], + "sub": ["getting-started", "overview", "guide", "2024", "latest", "popular", "archive", "details"] + } + ] +} diff --git a/data/en_US/username.json b/data/en_US/username.json index 9aa8743..5624d8a 100644 --- a/data/en_US/username.json +++ b/data/en_US/username.json @@ -4,21 +4,20 @@ "weight": 2, "first": ["alex", "ash", "avery", "bailey", "blake", "cameron", "casey", "charlie", "chris", "dakota", "drew", "elliot", "emerson", "finley", "frankie", "harley", "hayden", "jamie", "jordan", "kai", "kendall", "lane", "logan", "morgan", "parker", "quinn", "reese", "riley", "rowan", "sam", "sawyer", "skyler", "spencer", "taylor"], "last": ["adams", "brooks", "carter", "cole", "evans", "fisher", "fox", "grant", "gray", "hayes", "hunt", "lane", "lowe", "marsh", "mason", "morris", "north", "page", "quinn", "reed", "rhodes", "shaw", "stone", "vale", "wells", "west", "wolfe"], - "sep": ["", "", "", ".", "_"], - "n": ["", "", "", "", "1", "7", "12", "23", "42", "77", "99", "2024"] + "sep": [{ "format": "", "weight": 3 }, ".", "_"], + "n": [{ "format": "", "weight": 4 }, "1", "7", "12", "23", "42", "77", "99", "2024"] }, { "format": "{adj}{sep}{noun}{n}", "weight": 2, "adj": ["bluish", "brave", "chill", "cosmic", "cool", "dizzy", "electric", "epic", "fluffy", "frosty", "funky", "fuzzy", "golden", "groovy", "grumpy", "happy", "hyper", "jazzy", "lucky", "mega", "mighty", "neon", "nimble", "quirky", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "sparkly", "spicy", "sunny", "super", "swift", "turbo", "wild", "witty", "zany"], "noun": ["badger", "biscuit", "cactus", "comet", "donut", "dragon", "falcon", "ferret", "gizmo", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], - "sep": ["", "", "", ".", "_"], - "n": ["", "", "7", "13", "42", "77", "99", "123", "420", "2024"] + "sep": [{ "format": "", "weight": 3 }, ".", "_"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "42", "77", "99", "123", "420", "2024"] }, { "format": "{w}{n}", - "weight": 1, "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] } ] diff --git a/data/en_US/version.json b/data/en_US/version.json index f0806d4..43be399 100644 --- a/data/en_US/version.json +++ b/data/en_US/version.json @@ -1,8 +1,6 @@ -[ - { - "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}" }], - "pre": ["", { "format": "v", "weight": 0.4 }], - "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] - } -] +{ + "format": "{pre}{n}.{n}.{n}{suffix}", + "n": [{ "format": "{digits(1)}", "weight": 3 }, "{int(10,99)}"], + "pre": ["", { "format": "v", "weight": 0.4 }], + "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] +} diff --git a/data/misc/car.json b/data/misc/car.json index 208e638..91886c6 100644 --- a/data/misc/car.json +++ b/data/misc/car.json @@ -1,10 +1,10 @@ [ - { "format": "{maker} {model}", "maker": ["BMW"], "model": ["3 Series", "5 Series", "X3", "X5"] }, - { "format": "{maker} {model}", "maker": ["Ford"], "model": ["Fiesta", "Focus", "Mustang", "Explorer"] }, - { "format": "{maker} {model}", "maker": ["Honda"], "model": ["Civic", "Accord", "CR-V", "Jazz"] }, - { "format": "{maker} {model}", "maker": ["Mercedes-Benz"], "model": ["A-Class", "C-Class", "E-Class", "GLC"] }, - { "format": "{maker} {model}", "maker": ["Tesla"], "model": ["Model 3", "Model S", "Model X", "Model Y"] }, - { "format": "{maker} {model}", "maker": ["Toyota"], "model": ["Corolla", "Camry", "RAV4", "Yaris"] }, - { "format": "{maker} {model}", "maker": ["Volkswagen"], "model": ["Golf", "Passat", "Polo", "Tiguan"] }, - { "format": "{maker} {model}", "maker": ["Volvo"], "model": ["XC40", "XC60", "XC90", "V60"] } + { "format": "{maker} {model}", "maker": "BMW", "model": ["3 Series", "5 Series", "X3", "X5"] }, + { "format": "{maker} {model}", "maker": "Ford", "model": ["Fiesta", "Focus", "Mustang", "Explorer"] }, + { "format": "{maker} {model}", "maker": "Honda", "model": ["Civic", "Accord", "CR-V", "Jazz"] }, + { "format": "{maker} {model}", "maker": "Mercedes-Benz", "model": ["A-Class", "C-Class", "E-Class", "GLC"] }, + { "format": "{maker} {model}", "maker": "Tesla", "model": ["Model 3", "Model S", "Model X", "Model Y"] }, + { "format": "{maker} {model}", "maker": "Toyota", "model": ["Corolla", "Camry", "RAV4", "Yaris"] }, + { "format": "{maker} {model}", "maker": "Volkswagen", "model": ["Golf", "Passat", "Polo", "Tiguan"] }, + { "format": "{maker} {model}", "maker": "Volvo", "model": ["XC40", "XC60", "XC90", "V60"] } ] diff --git a/data/misc/coordinate.json b/data/misc/coordinate.json index b00f222..d2a35a7 100644 --- a/data/misc/coordinate.json +++ b/data/misc/coordinate.json @@ -1,7 +1 @@ -[ - { - "format": "{lat}, {lon}", - "lat": { "format": "{float(-90,90,6)}" }, - "lon": { "format": "{float(-180,180,6)}" } - } -] +{ "format": "{lat}, {lon}", "lat": "{float(-90,90,6)}", "lon": "{float(-180,180,6)}" } diff --git a/data/misc/country.json b/data/misc/country.json index cf6ae99..dc58f22 100644 --- a/data/misc/country.json +++ b/data/misc/country.json @@ -1,22 +1,22 @@ [ - { "format": "{name}", "name": ["Australia"], "alpha2": ["AU"], "alpha3": ["AUS"] }, - { "format": "{name}", "name": ["Brazil"], "alpha2": ["BR"], "alpha3": ["BRA"] }, - { "format": "{name}", "name": ["Canada"], "alpha2": ["CA"], "alpha3": ["CAN"] }, - { "format": "{name}", "name": ["China"], "alpha2": ["CN"], "alpha3": ["CHN"] }, - { "format": "{name}", "name": ["Denmark"], "alpha2": ["DK"], "alpha3": ["DNK"] }, - { "format": "{name}", "name": ["Finland"], "alpha2": ["FI"], "alpha3": ["FIN"] }, - { "format": "{name}", "name": ["France"], "alpha2": ["FR"], "alpha3": ["FRA"] }, - { "format": "{name}", "name": ["Germany"], "alpha2": ["DE"], "alpha3": ["DEU"] }, - { "format": "{name}", "name": ["India"], "alpha2": ["IN"], "alpha3": ["IND"] }, - { "format": "{name}", "name": ["Italy"], "alpha2": ["IT"], "alpha3": ["ITA"] }, - { "format": "{name}", "name": ["Japan"], "alpha2": ["JP"], "alpha3": ["JPN"] }, - { "format": "{name}", "name": ["Mexico"], "alpha2": ["MX"], "alpha3": ["MEX"] }, - { "format": "{name}", "name": ["Netherlands"], "alpha2": ["NL"], "alpha3": ["NLD"] }, - { "format": "{name}", "name": ["Norway"], "alpha2": ["NO"], "alpha3": ["NOR"] }, - { "format": "{name}", "name": ["Poland"], "alpha2": ["PL"], "alpha3": ["POL"] }, - { "format": "{name}", "name": ["Spain"], "alpha2": ["ES"], "alpha3": ["ESP"] }, - { "format": "{name}", "name": ["Sweden"], "alpha2": ["SE"], "alpha3": ["SWE"] }, - { "format": "{name}", "name": ["Switzerland"], "alpha2": ["CH"], "alpha3": ["CHE"] }, - { "format": "{name}", "name": ["United Kingdom"], "alpha2": ["GB"], "alpha3": ["GBR"] }, - { "format": "{name}", "name": ["United States"], "alpha2": ["US"], "alpha3": ["USA"] } + { "format": "{name}", "name": "Australia", "alpha2": "AU", "alpha3": "AUS" }, + { "format": "{name}", "name": "Brazil", "alpha2": "BR", "alpha3": "BRA" }, + { "format": "{name}", "name": "Canada", "alpha2": "CA", "alpha3": "CAN" }, + { "format": "{name}", "name": "China", "alpha2": "CN", "alpha3": "CHN" }, + { "format": "{name}", "name": "Denmark", "alpha2": "DK", "alpha3": "DNK" }, + { "format": "{name}", "name": "Finland", "alpha2": "FI", "alpha3": "FIN" }, + { "format": "{name}", "name": "France", "alpha2": "FR", "alpha3": "FRA" }, + { "format": "{name}", "name": "Germany", "alpha2": "DE", "alpha3": "DEU" }, + { "format": "{name}", "name": "India", "alpha2": "IN", "alpha3": "IND" }, + { "format": "{name}", "name": "Italy", "alpha2": "IT", "alpha3": "ITA" }, + { "format": "{name}", "name": "Japan", "alpha2": "JP", "alpha3": "JPN" }, + { "format": "{name}", "name": "Mexico", "alpha2": "MX", "alpha3": "MEX" }, + { "format": "{name}", "name": "Netherlands", "alpha2": "NL", "alpha3": "NLD" }, + { "format": "{name}", "name": "Norway", "alpha2": "NO", "alpha3": "NOR" }, + { "format": "{name}", "name": "Poland", "alpha2": "PL", "alpha3": "POL" }, + { "format": "{name}", "name": "Spain", "alpha2": "ES", "alpha3": "ESP" }, + { "format": "{name}", "name": "Sweden", "alpha2": "SE", "alpha3": "SWE" }, + { "format": "{name}", "name": "Switzerland", "alpha2": "CH", "alpha3": "CHE" }, + { "format": "{name}", "name": "United Kingdom", "alpha2": "GB", "alpha3": "GBR" }, + { "format": "{name}", "name": "United States", "alpha2": "US", "alpha3": "USA" } ] diff --git a/data/misc/creditcard.json b/data/misc/creditcard.json index f2d1517..0bdce70 100644 --- a/data/misc/creditcard.json +++ b/data/misc/creditcard.json @@ -1,5 +1,18 @@ [ - { "format": "4{d}{luhn()}", "d": { "format": "{digits(1)}", "repeat": 14 }, "weight": 4 }, - { "format": "5{m}{d}{luhn()}", "m": ["1", "2", "3", "4", "5"], "d": { "format": "{digits(1)}", "repeat": 13 }, "weight": 3 }, - { "format": "3{m}{d}{luhn()}", "m": ["4", "7"], "d": { "format": "{digits(1)}", "repeat": 12 }, "weight": 1 } + { + "format": "4{d}{luhn()}", + "d": { "format": "{digits(1)}", "repeat": 14 }, + "weight": 4 + }, + { + "format": "5{m}{d}{luhn()}", + "m": ["1", "2", "3", "4", "5"], + "d": { "format": "{digits(1)}", "repeat": 13 }, + "weight": 3 + }, + { + "format": "3{m}{d}{luhn()}", + "m": ["4", "7"], + "d": { "format": "{digits(1)}", "repeat": 12 } + } ] diff --git a/data/misc/currency.json b/data/misc/currency.json index 7e79b8a..6427331 100644 --- a/data/misc/currency.json +++ b/data/misc/currency.json @@ -1,18 +1,18 @@ [ - { "format": "{code}", "code": ["AUD"], "name": ["Australian Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["BRL"], "name": ["Brazilian Real"], "symbol": ["R$"] }, - { "format": "{code}", "code": ["CAD"], "name": ["Canadian Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["CHF"], "name": ["Swiss Franc"], "symbol": ["CHF"] }, - { "format": "{code}", "code": ["CNY"], "name": ["Chinese Yuan"], "symbol": ["¥"] }, - { "format": "{code}", "code": ["DKK"], "name": ["Danish Krone"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["EUR"], "name": ["Euro"], "symbol": ["€"] }, - { "format": "{code}", "code": ["GBP"], "name": ["Pound Sterling"], "symbol": ["£"] }, - { "format": "{code}", "code": ["INR"], "name": ["Indian Rupee"], "symbol": ["₹"] }, - { "format": "{code}", "code": ["JPY"], "name": ["Japanese Yen"], "symbol": ["¥"] }, - { "format": "{code}", "code": ["MXN"], "name": ["Mexican Peso"], "symbol": ["$"] }, - { "format": "{code}", "code": ["NOK"], "name": ["Norwegian Krone"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["PLN"], "name": ["Polish Zloty"], "symbol": ["zł"] }, - { "format": "{code}", "code": ["SEK"], "name": ["Swedish Krona"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["USD"], "name": ["US Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["ZAR"], "name": ["South African Rand"], "symbol": ["R"] } + { "format": "{code}", "code": "AUD", "name": "Australian Dollar", "symbol": "$" }, + { "format": "{code}", "code": "BRL", "name": "Brazilian Real", "symbol": "R$" }, + { "format": "{code}", "code": "CAD", "name": "Canadian Dollar", "symbol": "$" }, + { "format": "{code}", "code": "CHF", "name": "Swiss Franc", "symbol": "CHF" }, + { "format": "{code}", "code": "CNY", "name": "Chinese Yuan", "symbol": "¥" }, + { "format": "{code}", "code": "DKK", "name": "Danish Krone", "symbol": "kr" }, + { "format": "{code}", "code": "EUR", "name": "Euro", "symbol": "€" }, + { "format": "{code}", "code": "GBP", "name": "Pound Sterling", "symbol": "£" }, + { "format": "{code}", "code": "INR", "name": "Indian Rupee", "symbol": "₹" }, + { "format": "{code}", "code": "JPY", "name": "Japanese Yen", "symbol": "¥" }, + { "format": "{code}", "code": "MXN", "name": "Mexican Peso", "symbol": "$" }, + { "format": "{code}", "code": "NOK", "name": "Norwegian Krone", "symbol": "kr" }, + { "format": "{code}", "code": "PLN", "name": "Polish Zloty", "symbol": "zł" }, + { "format": "{code}", "code": "SEK", "name": "Swedish Krona", "symbol": "kr" }, + { "format": "{code}", "code": "USD", "name": "US Dollar", "symbol": "$" }, + { "format": "{code}", "code": "ZAR", "name": "South African Rand", "symbol": "R" } ] diff --git a/data/misc/emoji.json b/data/misc/emoji.json index 193f29a..6b3d096 100644 --- a/data/misc/emoji.json +++ b/data/misc/emoji.json @@ -1,6 +1 @@ -[ - "😀", "😂", "😍", "🤔", "😎", "😭", "😡", "👍", "👎", "🙏", - "👏", "🙌", "💪", "🔥", "✨", "🎉", "❤️", "💔", "💯", "👀", - "🚀", "⭐", "🌈", "☀️", "🌙", "⚡", "❄️", "🍕", "🍔", "🍺", - "☕", "🎂", "🐶", "🐱", "🦊", "🐢", "🦄", "🌸", "🌍", "💡" -] +["😀", "😂", "😍", "🤔", "😎", "😭", "😡", "👍", "👎", "🙏", "👏", "🙌", "💪", "🔥", "✨", "🎉", "❤️", "💔", "💯", "👀", "🚀", "⭐", "🌈", "☀️", "🌙", "⚡", "❄️", "🍕", "🍔", "🍺", "☕", "🎂", "🐶", "🐱", "🦊", "🐢", "🦄", "🌸", "🌍", "💡"] diff --git a/data/misc/httpstatus.json b/data/misc/httpstatus.json index b27e5ad..0a9af07 100644 --- a/data/misc/httpstatus.json +++ b/data/misc/httpstatus.json @@ -1,18 +1,18 @@ [ - { "format": "{code} {reason}", "code": ["200"], "reason": ["OK"] }, - { "format": "{code} {reason}", "code": ["201"], "reason": ["Created"] }, - { "format": "{code} {reason}", "code": ["204"], "reason": ["No Content"] }, - { "format": "{code} {reason}", "code": ["301"], "reason": ["Moved Permanently"] }, - { "format": "{code} {reason}", "code": ["302"], "reason": ["Found"] }, - { "format": "{code} {reason}", "code": ["304"], "reason": ["Not Modified"] }, - { "format": "{code} {reason}", "code": ["400"], "reason": ["Bad Request"] }, - { "format": "{code} {reason}", "code": ["401"], "reason": ["Unauthorized"] }, - { "format": "{code} {reason}", "code": ["403"], "reason": ["Forbidden"] }, - { "format": "{code} {reason}", "code": ["404"], "reason": ["Not Found"] }, - { "format": "{code} {reason}", "code": ["409"], "reason": ["Conflict"] }, - { "format": "{code} {reason}", "code": ["422"], "reason": ["Unprocessable Entity"] }, - { "format": "{code} {reason}", "code": ["429"], "reason": ["Too Many Requests"] }, - { "format": "{code} {reason}", "code": ["500"], "reason": ["Internal Server Error"] }, - { "format": "{code} {reason}", "code": ["502"], "reason": ["Bad Gateway"] }, - { "format": "{code} {reason}", "code": ["503"], "reason": ["Service Unavailable"] } + { "format": "{code} {reason}", "code": "200", "reason": "OK" }, + { "format": "{code} {reason}", "code": "201", "reason": "Created" }, + { "format": "{code} {reason}", "code": "204", "reason": "No Content" }, + { "format": "{code} {reason}", "code": "301", "reason": "Moved Permanently" }, + { "format": "{code} {reason}", "code": "302", "reason": "Found" }, + { "format": "{code} {reason}", "code": "304", "reason": "Not Modified" }, + { "format": "{code} {reason}", "code": "400", "reason": "Bad Request" }, + { "format": "{code} {reason}", "code": "401", "reason": "Unauthorized" }, + { "format": "{code} {reason}", "code": "403", "reason": "Forbidden" }, + { "format": "{code} {reason}", "code": "404", "reason": "Not Found" }, + { "format": "{code} {reason}", "code": "409", "reason": "Conflict" }, + { "format": "{code} {reason}", "code": "422", "reason": "Unprocessable Entity" }, + { "format": "{code} {reason}", "code": "429", "reason": "Too Many Requests" }, + { "format": "{code} {reason}", "code": "500", "reason": "Internal Server Error" }, + { "format": "{code} {reason}", "code": "502", "reason": "Bad Gateway" }, + { "format": "{code} {reason}", "code": "503", "reason": "Service Unavailable" } ] diff --git a/data/misc/language.json b/data/misc/language.json index 566825b..3e5689c 100644 --- a/data/misc/language.json +++ b/data/misc/language.json @@ -1,18 +1,18 @@ [ - { "format": "{name}", "name": ["Arabic"], "code": ["ar"] }, - { "format": "{name}", "name": ["Chinese"], "code": ["zh"] }, - { "format": "{name}", "name": ["Danish"], "code": ["da"] }, - { "format": "{name}", "name": ["Dutch"], "code": ["nl"] }, - { "format": "{name}", "name": ["English"], "code": ["en"] }, - { "format": "{name}", "name": ["Finnish"], "code": ["fi"] }, - { "format": "{name}", "name": ["French"], "code": ["fr"] }, - { "format": "{name}", "name": ["German"], "code": ["de"] }, - { "format": "{name}", "name": ["Italian"], "code": ["it"] }, - { "format": "{name}", "name": ["Japanese"], "code": ["ja"] }, - { "format": "{name}", "name": ["Norwegian"], "code": ["no"] }, - { "format": "{name}", "name": ["Polish"], "code": ["pl"] }, - { "format": "{name}", "name": ["Portuguese"], "code": ["pt"] }, - { "format": "{name}", "name": ["Russian"], "code": ["ru"] }, - { "format": "{name}", "name": ["Spanish"], "code": ["es"] }, - { "format": "{name}", "name": ["Swedish"], "code": ["sv"] } + { "format": "{name}", "name": "Arabic", "code": "ar" }, + { "format": "{name}", "name": "Chinese", "code": "zh" }, + { "format": "{name}", "name": "Danish", "code": "da" }, + { "format": "{name}", "name": "Dutch", "code": "nl" }, + { "format": "{name}", "name": "English", "code": "en" }, + { "format": "{name}", "name": "Finnish", "code": "fi" }, + { "format": "{name}", "name": "French", "code": "fr" }, + { "format": "{name}", "name": "German", "code": "de" }, + { "format": "{name}", "name": "Italian", "code": "it" }, + { "format": "{name}", "name": "Japanese", "code": "ja" }, + { "format": "{name}", "name": "Norwegian", "code": "no" }, + { "format": "{name}", "name": "Polish", "code": "pl" }, + { "format": "{name}", "name": "Portuguese", "code": "pt" }, + { "format": "{name}", "name": "Russian", "code": "ru" }, + { "format": "{name}", "name": "Spanish", "code": "es" }, + { "format": "{name}", "name": "Swedish", "code": "sv" } ] diff --git a/data/misc/mac.json b/data/misc/mac.json index 4b3d5b1..fe5ba69 100644 --- a/data/misc/mac.json +++ b/data/misc/mac.json @@ -1,3 +1 @@ -[ - { "format": "{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" } -] +"{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" diff --git a/data/misc/mimetype.json b/data/misc/mimetype.json index e7ffda2..dcbe676 100644 --- a/data/misc/mimetype.json +++ b/data/misc/mimetype.json @@ -1,17 +1,17 @@ [ - { "format": "{type}", "type": ["application/gzip"], "ext": [".gz"] }, - { "format": "{type}", "type": ["application/json"], "ext": [".json"] }, - { "format": "{type}", "type": ["application/pdf"], "ext": [".pdf"] }, - { "format": "{type}", "type": ["application/xml"], "ext": [".xml"] }, - { "format": "{type}", "type": ["application/zip"], "ext": [".zip"] }, - { "format": "{type}", "type": ["audio/mpeg"], "ext": [".mp3"] }, - { "format": "{type}", "type": ["image/gif"], "ext": [".gif"] }, - { "format": "{type}", "type": ["image/jpeg"], "ext": [".jpg"] }, - { "format": "{type}", "type": ["image/png"], "ext": [".png"] }, - { "format": "{type}", "type": ["image/svg+xml"], "ext": [".svg"] }, - { "format": "{type}", "type": ["image/webp"], "ext": [".webp"] }, - { "format": "{type}", "type": ["text/csv"], "ext": [".csv"] }, - { "format": "{type}", "type": ["text/html"], "ext": [".html"] }, - { "format": "{type}", "type": ["text/plain"], "ext": [".txt"] }, - { "format": "{type}", "type": ["video/mp4"], "ext": [".mp4"] } + { "format": "{type}", "type": "application/gzip", "ext": ".gz" }, + { "format": "{type}", "type": "application/json", "ext": ".json" }, + { "format": "{type}", "type": "application/pdf", "ext": ".pdf" }, + { "format": "{type}", "type": "application/xml", "ext": ".xml" }, + { "format": "{type}", "type": "application/zip", "ext": ".zip" }, + { "format": "{type}", "type": "audio/mpeg", "ext": ".mp3" }, + { "format": "{type}", "type": "image/gif", "ext": ".gif" }, + { "format": "{type}", "type": "image/jpeg", "ext": ".jpg" }, + { "format": "{type}", "type": "image/png", "ext": ".png" }, + { "format": "{type}", "type": "image/svg+xml", "ext": ".svg" }, + { "format": "{type}", "type": "image/webp", "ext": ".webp" }, + { "format": "{type}", "type": "text/csv", "ext": ".csv" }, + { "format": "{type}", "type": "text/html", "ext": ".html" }, + { "format": "{type}", "type": "text/plain", "ext": ".txt" }, + { "format": "{type}", "type": "video/mp4", "ext": ".mp4" } ] diff --git a/data/misc/objectid.json b/data/misc/objectid.json index 9681734..8174210 100644 --- a/data/misc/objectid.json +++ b/data/misc/objectid.json @@ -1,3 +1 @@ -[ - { "format": "{hex(24)}" } -] +"{hex(24)}" diff --git a/data/misc/timezone.json b/data/misc/timezone.json index 62d6cb5..5c14f62 100644 --- a/data/misc/timezone.json +++ b/data/misc/timezone.json @@ -1,29 +1 @@ -[ - "UTC", - "Africa/Cairo", - "Africa/Johannesburg", - "America/Chicago", - "America/Denver", - "America/Los_Angeles", - "America/Mexico_City", - "America/New_York", - "America/Sao_Paulo", - "America/Toronto", - "Asia/Dubai", - "Asia/Hong_Kong", - "Asia/Kolkata", - "Asia/Shanghai", - "Asia/Singapore", - "Asia/Tokyo", - "Australia/Sydney", - "Europe/Amsterdam", - "Europe/Berlin", - "Europe/London", - "Europe/Madrid", - "Europe/Moscow", - "Europe/Oslo", - "Europe/Paris", - "Europe/Stockholm", - "Europe/Warsaw", - "Pacific/Auckland" -] +["UTC", "Africa/Cairo", "Africa/Johannesburg", "America/Chicago", "America/Denver", "America/Los_Angeles", "America/Mexico_City", "America/New_York", "America/Sao_Paulo", "America/Toronto", "Asia/Dubai", "Asia/Hong_Kong", "Asia/Kolkata", "Asia/Shanghai", "Asia/Singapore", "Asia/Tokyo", "Australia/Sydney", "Europe/Amsterdam", "Europe/Berlin", "Europe/London", "Europe/Madrid", "Europe/Moscow", "Europe/Oslo", "Europe/Paris", "Europe/Stockholm", "Europe/Warsaw", "Pacific/Auckland"] diff --git a/data/misc/useragent.json b/data/misc/useragent.json index f42bf9f..e659a1b 100644 --- a/data/misc/useragent.json +++ b/data/misc/useragent.json @@ -1,10 +1 @@ -[ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0" -] +["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"] diff --git a/data/misc/uuid.json b/data/misc/uuid.json index 18cbf98..4fa8075 100644 --- a/data/misc/uuid.json +++ b/data/misc/uuid.json @@ -1,6 +1 @@ -[ - { - "format": "{hex(8)}-{hex(4)}-4{hex(3)}-{variant}{hex(3)}-{hex(12)}", - "variant": ["8", "9", "a", "b"] - } -] +{ "format": "{hex(8)}-{hex(4)}-4{hex(3)}-{variant}{hex(3)}-{hex(12)}", "variant": ["8", "9", "a", "b"] } diff --git a/data/sv_SE/address.json b/data/sv_SE/address.json index 81766cb..25e20ab 100644 --- a/data/sv_SE/address.json +++ b/data/sv_SE/address.json @@ -1,25 +1,21 @@ -[ - { - "format": "{street} {street-number}\n{postal-code} {locality}", - "street": [ - { - "format": "{first}{last}", - "first": ["Ängs", "Bergs", "Björk", "Drottning", "Eke", "Furu", "Hamn", "Köpmans", "Kungs", "Linné", "Norra", "Nybro", "Oden", "Öster", "Park", "Skogs", "Skol", "Slotts", "Söder", "Stations", "Stor", "Strand", "Trädgårds", "Vasa", "Väster"], - "last": ["gatan", "vägen", "stigen", "gränd", "backen", "torget", "allén"] - }, - ["Avenyn", "Birger Jarlsgatan", "Promenaden", "Staby", "Sveavägen", "Vintjärn"] - ], - "street-number": [ - { "format": "{int(1,9)}" }, - { "format": "{int(10,99)}" }, - { "format": "{int(100,999)}", "weight": 0.2 }, - { "format": "{int(1,9)}{upper(1)}", "weight": 0.2 }, - { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 }, - { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 } - ], - "postal-code": [ - { "format": "{int(100,999)} {int(10,99)}" } - ], - "locality": ["Alingsås", "Alvesta", "Ängelholm", "Arboga", "Arvika", "Avesta", "Boden", "Bollnäs", "Borås", "Borlänge", "Enköping", "Eskilstuna", "Eslöv", "Fagersta", "Falkenberg", "Falköping", "Falun", "Finspång", "Gällivare", "Gävle", "Göteborg", "Halmstad", "Haparanda", "Härnösand", "Hässleholm", "Helsingborg", "Huddinge", "Hudiksvall", "Jönköping", "Kalmar", "Karlshamn", "Karlskoga", "Karlskrona", "Karlstad", "Katrineholm", "Kiruna", "Köping", "Kramfors", "Kristianstad", "Kristinehamn", "Landskrona", "Lidingö", "Lidköping", "Lindesberg", "Linköping", "Ljungby", "Ludvika", "Luleå", "Lund", "Lycksele", "Malmö", "Mariestad", "Mjölby", "Mölndal", "Mora", "Motala", "Nacka", "Nässjö", "Norrköping", "Norrtälje", "Nyköping", "Nynäshamn", "Örebro", "Örnsköldsvik", "Oskarshamn", "Östersund", "Piteå", "Rabbalshede", "Ronneby", "Säffle", "Sandviken", "Sävsjö", "Sigtuna", "Skara", "Skellefteå", "Skövde", "Söderhamn", "Södertälje", "Sollentuna", "Solna", "Sölvesborg", "Stockholm", "Strängnäs", "Sundbyberg", "Sundsvall", "Täby", "Tierp", "Tranås", "Trelleborg", "Trollhättan", "Uddevalla", "Ulricehamn", "Umeå", "Upplands Väsby", "Uppsala", "Vänersborg", "Varberg", "Värnamo", "Västerås", "Västervik", "Växjö", "Vetlanda", "Vimmerby", "Visby", "Ystad"] - } -] +{ + "format": "{street} {street-number}\n{postal-code} {locality}", + "street": [ + { + "format": "{first}{last}", + "first": ["Ängs", "Bergs", "Björk", "Drottning", "Eke", "Furu", "Hamn", "Köpmans", "Kungs", "Linné", "Norra", "Nybro", "Oden", "Öster", "Park", "Skogs", "Skol", "Slotts", "Söder", "Stations", "Stor", "Strand", "Trädgårds", "Vasa", "Väster"], + "last": ["gatan", "vägen", "stigen", "gränd", "backen", "torget", "allén"] + }, + ["Avenyn", "Birger Jarlsgatan", "Promenaden", "Staby", "Sveavägen", "Vintjärn"] + ], + "street-number": [ + "{int(1,9)}", + "{int(10,99)}", + { "format": "{int(100,999)}", "weight": 0.2 }, + { "format": "{int(1,9)}{upper(1)}", "weight": 0.2 }, + { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 }, + { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 } + ], + "postal-code": "{int(100,999)} {int(10,99)}", + "locality": ["Alingsås", "Alvesta", "Ängelholm", "Arboga", "Arvika", "Avesta", "Boden", "Bollnäs", "Borås", "Borlänge", "Enköping", "Eskilstuna", "Eslöv", "Fagersta", "Falkenberg", "Falköping", "Falun", "Finspång", "Gällivare", "Gävle", "Göteborg", "Halmstad", "Haparanda", "Härnösand", "Hässleholm", "Helsingborg", "Huddinge", "Hudiksvall", "Jönköping", "Kalmar", "Karlshamn", "Karlskoga", "Karlskrona", "Karlstad", "Katrineholm", "Kiruna", "Köping", "Kramfors", "Kristianstad", "Kristinehamn", "Landskrona", "Lidingö", "Lidköping", "Lindesberg", "Linköping", "Ljungby", "Ludvika", "Luleå", "Lund", "Lycksele", "Malmö", "Mariestad", "Mjölby", "Mölndal", "Mora", "Motala", "Nacka", "Nässjö", "Norrköping", "Norrtälje", "Nyköping", "Nynäshamn", "Örebro", "Örnsköldsvik", "Oskarshamn", "Östersund", "Piteå", "Rabbalshede", "Ronneby", "Säffle", "Sandviken", "Sävsjö", "Sigtuna", "Skara", "Skellefteå", "Skövde", "Söderhamn", "Södertälje", "Sollentuna", "Solna", "Sölvesborg", "Stockholm", "Strängnäs", "Sundbyberg", "Sundsvall", "Täby", "Tierp", "Tranås", "Trelleborg", "Trollhättan", "Uddevalla", "Ulricehamn", "Umeå", "Upplands Väsby", "Uppsala", "Vänersborg", "Varberg", "Värnamo", "Västerås", "Västervik", "Växjö", "Vetlanda", "Vimmerby", "Visby", "Ystad"] +} diff --git a/data/sv_SE/company.json b/data/sv_SE/company.json index a79bb01..b40d001 100644 --- a/data/sv_SE/company.json +++ b/data/sv_SE/company.json @@ -1,14 +1,12 @@ -[ - { - "format": "{base} {suffix}", - "base": [ - { - "format": "{a}{b}", - "a": ["Berg", "Blå", "Eko", "Fjäll", "Grön", "Järn", "Nord", "Ny", "Sjö", "Skog", "Sol", "Sten", "Stor", "Syd", "Söder", "Trä", "Vind", "Väst", "Ås", "Öst"], - "b": ["bygg", "data", "design", "energi", "frakt", "gruppen", "handel", "konsult", "kraft", "logistik", "media", "produkter", "system", "teknik", "verkstad", "vård"] - }, - ["Aktiebolaget Vega", "Bröderna Lund", "Ekonomihuset", "Hantverkargruppen", "Lindberg", "Mälardalens Bygg", "Nordström", "Skandinaviska Handelshuset", "Sundbergs Verkstad", "Svensson & Co", "Vasa Logistik"] - ], - "suffix": ["AB", "AB", "AB", "AB", "HB", "KB"] - } -] +{ + "format": "{base} {suffix}", + "base": [ + { + "format": "{a}{b}", + "a": ["Berg", "Blå", "Eko", "Fjäll", "Grön", "Järn", "Nord", "Ny", "Sjö", "Skog", "Sol", "Sten", "Stor", "Syd", "Söder", "Trä", "Vind", "Väst", "Ås", "Öst"], + "b": ["bygg", "data", "design", "energi", "frakt", "gruppen", "handel", "konsult", "kraft", "logistik", "media", "produkter", "system", "teknik", "verkstad", "vård"] + }, + ["Aktiebolaget Vega", "Bröderna Lund", "Ekonomihuset", "Hantverkargruppen", "Lindberg", "Mälardalens Bygg", "Nordström", "Skandinaviska Handelshuset", "Sundbergs Verkstad", "Svensson & Co", "Vasa Logistik"] + ], + "suffix": [{ "format": "AB", "weight": 4 }, "HB", "KB"] +} diff --git a/data/sv_SE/date.json b/data/sv_SE/date.json index 18de4f5..a591bf7 100644 --- a/data/sv_SE/date.json +++ b/data/sv_SE/date.json @@ -1,15 +1,13 @@ -[ - { - "format": "{year}-{month}-{day}", - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], - "year": [ - { "format": "197{digits(1)}" }, - { "format": "198{digits(1)}" }, - { "format": "199{digits(1)}", "weight": 2 }, - { "format": "200{digits(1)}", "weight": 3 }, - { "format": "201{digits(1)}", "weight": 3 }, - { "format": "202{digits(1)}", "weight": 2 } - ] - } -] +{ + "format": "{year}-{month}-{day}", + "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], + "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], + "year": [ + "197{digits(1)}", + "198{digits(1)}", + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } + ] +} diff --git a/data/sv_SE/email.json b/data/sv_SE/email.json index c1d3fbf..cadf729 100644 --- a/data/sv_SE/email.json +++ b/data/sv_SE/email.json @@ -1,29 +1,26 @@ -[ - { - "format": "{local}@{domain}", - "local": [ - { - "format": "{first}{sep}{last}{n}", - "weight": 3, - "first": ["agnes", "alice", "anders", "anna", "anton", "asa", "axel", "bjorn", "david", "ebba", "elin", "elsa", "emil", "emma", "erik", "ester", "frida", "fredrik", "gustav", "hampus", "hanna", "henrik", "hugo", "ida", "ingrid", "isak", "johan", "jonas", "julia", "karin", "kjell", "klara", "lars", "lena", "linus", "magnus", "maja", "mats", "mattias", "mikael", "moa", "nils", "olle", "oskar", "par", "patrik", "per", "pontus", "rasmus", "samuel", "sara", "sofia", "soren", "stina", "sven", "tobias", "tuva", "viktor", "wilma", "ylva"], - "last": ["ahlberg", "andersson", "berg", "berglund", "bergstrom", "blom", "dahlberg", "eklund", "ekstrom", "engstrom", "eriksson", "falk", "forsberg", "fredriksson", "gustafsson", "hellstrom", "holm", "holmberg", "jakobsson", "johansson", "jonsson", "karlsson", "larsson", "lind", "lindberg", "lindgren", "lindqvist", "lundberg", "lundgren", "lundqvist", "magnusson", "nilsson", "norberg", "nystrom", "olsson", "persson", "petersson", "samuelsson", "sandberg", "sjoberg", "sjogren", "strand", "strom", "sundberg", "svensson", "soderberg", "wallin", "aberg", "oberg"], - "sep": [".", ".", "_", "-", ""], - "n": ["", "", "", "1", "7", "42", "88", "99"] - }, - { - "format": "{adj}{noun}{n}", - "weight": 2, - "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], - "noun": ["abborren", "bamsen", "bjorn", "drake", "ekorre", "falken", "grodan", "hund", "igelkott", "kaninen", "katt", "korpen", "krabban", "musen", "nallen", "ninja", "orn", "panda", "pirat", "raven", "robot", "sillen", "sparven", "uggla", "varg", "vargen"], - "n": ["", "", "7", "13", "14", "42", "77", "99", "123", "420", "2024"] - }, - { - "format": "{w}{n}", - "weight": 1, - "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "doot", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pew", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] - } - ], - "domain": ["example.com", "example.org", "example.net", "example.se", "example.nu", "example.io", "mail.example.se", "webmail.example.se", "inkorg.example.se", "post.example.se", "demo.example.se", "test.example.org", "dev.example.se"] - } -] +{ + "format": "{local}@{domain}", + "local": [ + { + "format": "{first}{sep}{last}{n}", + "weight": 3, + "first": ["agnes", "alice", "anders", "anna", "anton", "asa", "axel", "bjorn", "david", "ebba", "elin", "elsa", "emil", "emma", "erik", "ester", "frida", "fredrik", "gustav", "hampus", "hanna", "henrik", "hugo", "ida", "ingrid", "isak", "johan", "jonas", "julia", "karin", "kjell", "klara", "lars", "lena", "linus", "magnus", "maja", "mats", "mattias", "mikael", "moa", "nils", "olle", "oskar", "par", "patrik", "per", "pontus", "rasmus", "samuel", "sara", "sofia", "soren", "stina", "sven", "tobias", "tuva", "viktor", "wilma", "ylva"], + "last": ["ahlberg", "andersson", "berg", "berglund", "bergstrom", "blom", "dahlberg", "eklund", "ekstrom", "engstrom", "eriksson", "falk", "forsberg", "fredriksson", "gustafsson", "hellstrom", "holm", "holmberg", "jakobsson", "johansson", "jonsson", "karlsson", "larsson", "lind", "lindberg", "lindgren", "lindqvist", "lundberg", "lundgren", "lundqvist", "magnusson", "nilsson", "norberg", "nystrom", "olsson", "persson", "petersson", "samuelsson", "sandberg", "sjoberg", "sjogren", "strand", "strom", "sundberg", "svensson", "soderberg", "wallin", "aberg", "oberg"], + "sep": [{ "format": ".", "weight": 2 }, "_", "-", ""], + "n": [{ "format": "", "weight": 3 }, "1", "7", "42", "88", "99"] + }, + { + "format": "{adj}{noun}{n}", + "weight": 2, + "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], + "noun": ["abborren", "bamsen", "bjorn", "drake", "ekorre", "falken", "grodan", "hund", "igelkott", "kaninen", "katt", "korpen", "krabban", "musen", "nallen", "ninja", "orn", "panda", "pirat", "raven", "robot", "sillen", "sparven", "uggla", "varg", "vargen"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "14", "42", "77", "99", "123", "420", "2024"] + }, + { + "format": "{w}{n}", + "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "doot", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pew", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + } + ], + "domain": ["example.com", "example.org", "example.net", "example.se", "example.nu", "example.io", "mail.example.se", "webmail.example.se", "inkorg.example.se", "post.example.se", "demo.example.se", "test.example.org", "dev.example.se"] +} diff --git a/data/sv_SE/ip.json b/data/sv_SE/ip.json index e16c78a..c9bf575 100644 --- a/data/sv_SE/ip.json +++ b/data/sv_SE/ip.json @@ -2,9 +2,11 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}", "weight": 4 }, { "format": "1{digits(2)}", "weight": 2 }] + "o": [ + { "format": "{digits(1)}", "weight": 3 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "1{digits(2)}", "weight": 2 } + ] }, - { - "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" - } + "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" ] diff --git a/data/sv_SE/person.json b/data/sv_SE/person.json index ff0a9ba..2806e36 100644 --- a/data/sv_SE/person.json +++ b/data/sv_SE/person.json @@ -1,41 +1,32 @@ -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Agnes", "Alice", "Alicia", "Alma", "Amanda", "Anna", "Astrid", "Cornelia", "Ebba", "Elin", "Ella", "Ellen", "Elsa", "Emilia", "Emma", "Ester", "Eva", "Frida", "Hanna", "Hedda", "Ida", "Ingrid", "Isabelle", "Julia", "Karin", "Klara", "Kristina", "Lena", "Linnéa", "Lova", "Maja", "Maria", "Moa", "Molly", "Märta", "Nellie", "Nora", "Olivia", "Saga", "Sara", "Selma", "Signe", "Siri", "Sofia", "Stina", "Tilde", "Tuva", "Wilma", "Ylva", "Åsa"], - "malefirst": ["Adam", "Albin", "Alexander", "Alfred", "Anders", "Anton", "Arvid", "Axel", "Bengt", "Bo", "Carl", "David", "Edvin", "Elias", "Emil", "Erik", "Filip", "Folke", "Fredrik", "Gustav", "Göran", "Hampus", "Hans", "Henrik", "Hugo", "Isak", "Johan", "Jonas", "Karl", "Kjell", "Lars", "Leo", "Linus", "Love", "Magnus", "Mats", "Mattias", "Mikael", "Nils", "Olle", "Oskar", "Otto", "Patrik", "Per", "Pontus", "Rasmus", "Samuel", "Sixten", "Stefan", "Sten", "Sven", "Theodor", "Tobias", "Viktor", "William", "Åke"], - "last": [ - { - "format": "{first}sson", - "weight": 5, - "first": ["Ander", "Arvid", "Bengt", "Daniel", "David", "Erik", "Fredrik", "Gustaf", "Henrik", "Håkan", "Isak", "Jakob", "Johan", "Jon", "Jön", "Karl", "Lar", "Magnu", "Matt", "Mikael", "Mårten", "Nil", "Ol", "Per", "Petter", "Samuel", "Sven"] - }, - { - "format": "{first}{last}", - "weight": 3.6, - "first": ["Ahl", "Berg", "Blom", "Ceder", "Dahl", "Ek", "Eng", "Falk", "Fors", "Gran", "Hag", "Hed", "Holm", "Lind", "Lund", "Mal", "Ny", "Rosen", "Sand", "Sjö", "Skog", "Söder", "Sten", "Strand", "Sund", "Wik", "Öst"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] - }, - { - "format": "{first}{last}", - "weight": 0.267, - "first": ["Hell", "Wall"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] - }, - { - "format": "{first}{last}", - "weight": 0.133, - "first": ["Norr"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "stedt", "sten", "strand", "ström", "vall"] - }, - ["Berg", "Blom", "Eismar", "Falk", "Holm", "Lind", "Norberg", "Strand", "Ström", "von Flemming", "Åberg", "Öberg"] - ], - "prefix": [ - "", - { - "format": "{string} ", - "string": ["dr", "prof"], - "weight": 0.05 - } - ] - } -] +{ + "format": "{prefix}{femalefirst|malefirst} {last}", + "femalefirst": ["Agnes", "Alice", "Alicia", "Alma", "Amanda", "Anna", "Astrid", "Cornelia", "Ebba", "Elin", "Ella", "Ellen", "Elsa", "Emilia", "Emma", "Ester", "Eva", "Frida", "Hanna", "Hedda", "Ida", "Ingrid", "Isabelle", "Julia", "Karin", "Klara", "Kristina", "Lena", "Linnéa", "Lova", "Maja", "Maria", "Moa", "Molly", "Märta", "Nellie", "Nora", "Olivia", "Saga", "Sara", "Selma", "Signe", "Siri", "Sofia", "Stina", "Tilde", "Tuva", "Wilma", "Ylva", "Åsa"], + "malefirst": ["Adam", "Albin", "Alexander", "Alfred", "Anders", "Anton", "Arvid", "Axel", "Bengt", "Bo", "Carl", "David", "Edvin", "Elias", "Emil", "Erik", "Filip", "Folke", "Fredrik", "Gustav", "Göran", "Hampus", "Hans", "Henrik", "Hugo", "Isak", "Johan", "Jonas", "Karl", "Kjell", "Lars", "Leo", "Linus", "Love", "Magnus", "Mats", "Mattias", "Mikael", "Nils", "Olle", "Oskar", "Otto", "Patrik", "Per", "Pontus", "Rasmus", "Samuel", "Sixten", "Stefan", "Sten", "Sven", "Theodor", "Tobias", "Viktor", "William", "Åke"], + "last": [ + { + "format": "{first}sson", + "weight": 5, + "first": ["Ander", "Arvid", "Bengt", "Daniel", "David", "Erik", "Fredrik", "Gustaf", "Henrik", "Håkan", "Isak", "Jakob", "Johan", "Jon", "Jön", "Karl", "Lar", "Magnu", "Matt", "Mikael", "Mårten", "Nil", "Ol", "Per", "Petter", "Samuel", "Sven"] + }, + { + "format": "{first}{last}", + "weight": 3.6, + "first": ["Ahl", "Berg", "Blom", "Ceder", "Dahl", "Ek", "Eng", "Falk", "Fors", "Gran", "Hag", "Hed", "Holm", "Lind", "Lund", "Mal", "Ny", "Rosen", "Sand", "Sjö", "Skog", "Söder", "Sten", "Strand", "Sund", "Wik", "Öst"], + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] + }, + { + "format": "{first}{last}", + "weight": 0.267, + "first": ["Hell", "Wall"], + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] + }, + { + "format": "{first}{last}", + "weight": 0.133, + "first": "Norr", + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "stedt", "sten", "strand", "ström", "vall"] + }, + ["Berg", "Blom", "Eismar", "Falk", "Holm", "Lind", "Norberg", "Strand", "Ström", "von Flemming", "Åberg", "Öberg"] + ], + "prefix": ["", { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 }] +} diff --git a/data/sv_SE/phone.json b/data/sv_SE/phone.json index 0bef075..841d442 100644 --- a/data/sv_SE/phone.json +++ b/data/sv_SE/phone.json @@ -3,15 +3,15 @@ "format": "{prefix}-{a} {b} {c}", "weight": 10, "prefix": ["070", "072", "073", "076", "079"], - "a": [{ "format": "{digits(3)}" }], - "b": [{ "format": "{digits(2)}" }], - "c": [{ "format": "{digits(2)}" }] + "a": "{digits(3)}", + "b": "{digits(2)}", + "c": "{digits(2)}" }, { "format": "{prefix}-{a} {b} {c}", "prefix": ["08", "011", "013", "018", "019", "021", "023", "026", "031", "033", "035", "036", "040", "042", "044", "046", "054", "060", "063", "090"], - "a": [{ "format": "{digits(3)}" }], - "b": [{ "format": "{digits(2)}" }], - "c": [{ "format": "{digits(2)}" }] + "a": "{digits(3)}", + "b": "{digits(2)}", + "c": "{digits(2)}" } ] diff --git a/data/sv_SE/price.json b/data/sv_SE/price.json index 92a33db..12e369b 100644 --- a/data/sv_SE/price.json +++ b/data/sv_SE/price.json @@ -1,13 +1,11 @@ -[ - { - "format": "{amt}{ore} kr", - "amt": [ - { "format": "{int(1,9)}", "weight": 2 }, - { "format": "{int(10,99)}", "weight": 4 }, - { "format": "{int(100,999)}", "weight": 3 }, - { "format": "{int(1,9)} {digits(3)}", "weight": 1 }, - { "format": "{int(10,99)} {digits(3)}", "weight": 0.3 } - ], - "ore": ["", { "format": ",{c}", "c": [{ "format": "{digits(2)}" }], "weight": 0.4 }] - } -] +{ + "format": "{amt}{ore} kr", + "amt": [ + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + "{int(1,9)} {digits(3)}", + { "format": "{int(10,99)} {digits(3)}", "weight": 0.3 } + ], + "ore": ["", { "format": ",{c}", "c": "{digits(2)}", "weight": 0.4 }] +} diff --git a/data/sv_SE/sentence.json b/data/sv_SE/sentence.json index 0cc0414..1b1ede9 100644 --- a/data/sv_SE/sentence.json +++ b/data/sv_SE/sentence.json @@ -7,7 +7,7 @@ "noun": ["arkivet", "berget", "bäcken", "dalen", "eken", "fjället", "floden", "fyren", "fågeln", "gården", "glaciären", "hamnen", "holmen", "klippan", "kusten", "lampan", "lyktan", "motorn", "måsen", "ravinen", "räven", "seglaren", "signalen", "sjön", "skogen", "slätten", "stigen", "stjärnan", "stranden", "tornet", "trädgården", "vandraren", "vinden", "vågen", "ängen"], "verb": ["anar", "bevakar", "bygger", "bär", "döljer", "famnar", "följer", "formar", "gömmer", "hälsar", "korsar", "leder", "lockar", "lyser", "minns", "möter", "når", "skyddar", "speglar", "söker", "tänder", "vaktar", "väcker", "värnar", "återvänder"], "prep": ["bakom", "bland", "bortom", "framför", "genom", "intill", "kring", "längs", "mot", "nära", "ovanför", "runt", "under", "vid", "över"], - "end": [".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 3 }, "!"] }, { "format": "{q} {adj} {noun} {verb} {prep} {noun}?", diff --git a/data/sv_SE/ssn.json b/data/sv_SE/ssn.json index 07ed245..90ab875 100644 --- a/data/sv_SE/ssn.json +++ b/data/sv_SE/ssn.json @@ -1,25 +1,22 @@ -[ - { - "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", - "mmdd": [ - { - "format": "{m}{d}", - "weight": 7, - "m": ["01", "03", "05", "07", "08", "10", "12"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"] - }, - { - "format": "{m}{d}", - "weight": 4, - "m": ["04", "06", "09", "11"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30"] - }, - { - "format": "{m}{d}", - "weight": 1, - "m": ["02"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"] - } - ] - } -] +{ + "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", + "mmdd": [ + { + "format": "{m}{d}", + "weight": 7, + "m": ["01", "03", "05", "07", "08", "10", "12"], + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"] + }, + { + "format": "{m}{d}", + "weight": 4, + "m": ["04", "06", "09", "11"], + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30"] + }, + { + "format": "{m}{d}", + "m": "02", + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"] + } + ] +} diff --git a/data/sv_SE/time.json b/data/sv_SE/time.json index 2084e19..af61b7d 100644 --- a/data/sv_SE/time.json +++ b/data/sv_SE/time.json @@ -1,8 +1,17 @@ -[ - { - "format": "{hour}:{minute}{sec}", - "hour": [{ "format": "0{digits(1)}", "weight": 4 }, { "format": "1{digits(1)}", "weight": 4 }, { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 }], - "minute": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], - "sec": ["", { "format": ":{s}", "s": [{ "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }], "weight": 0.4 }] - } -] +{ + "format": "{hour}:{minute}{sec}", + "hour": [ + { "format": "0{digits(1)}", "weight": 4 }, + { "format": "1{digits(1)}", "weight": 4 }, + { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 } + ], + "minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "sec": [ + "", + { + "format": ":{s}", + "s": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "weight": 0.4 + } + ] +} diff --git a/data/sv_SE/url.json b/data/sv_SE/url.json index 6faca12..131e0b5 100644 --- a/data/sv_SE/url.json +++ b/data/sv_SE/url.json @@ -1,12 +1,17 @@ -[ - { - "format": "https://{host}{path}", - "host": ["example.com", "www.example.com", "example.org", "example.se", "www.example.se", "blogg.example.net", "butik.example.com", "shop.example.se", "api.example.se", "dokument.example.org", "nyheter.example.net", "mail.example.se"], - "path": [ - "", - "", - { "format": "/{seg}", "seg": ["om", "om-oss", "produkter", "blogg", "kontakt", "hjalp", "artiklar", "logga-in", "nyheter", "start", "sok", "kundvagn", "kassa", "villkor", "integritet", "priser", "tjanster", "foretag", "karriar", "support", "konto", "installningar", "registrera"] }, - { "format": "/{seg}/{sub}", "weight": 0.5, "seg": ["blogg", "produkter", "hjalp", "dokument", "kategori"], "sub": ["oversikt", "kom-igang", "guide", "2024", "senaste", "arkiv", "detaljer"] } - ] - } -] +{ + "format": "https://{host}{path}", + "host": ["example.com", "www.example.com", "example.org", "example.se", "www.example.se", "blogg.example.net", "butik.example.com", "shop.example.se", "api.example.se", "dokument.example.org", "nyheter.example.net", "mail.example.se"], + "path": [ + { "format": "", "weight": 2 }, + { + "format": "/{seg}", + "seg": ["om", "om-oss", "produkter", "blogg", "kontakt", "hjalp", "artiklar", "logga-in", "nyheter", "start", "sok", "kundvagn", "kassa", "villkor", "integritet", "priser", "tjanster", "foretag", "karriar", "support", "konto", "installningar", "registrera"] + }, + { + "format": "/{seg}/{sub}", + "weight": 0.5, + "seg": ["blogg", "produkter", "hjalp", "dokument", "kategori"], + "sub": ["oversikt", "kom-igang", "guide", "2024", "senaste", "arkiv", "detaljer"] + } + ] +} diff --git a/data/sv_SE/username.json b/data/sv_SE/username.json index d97df0e..cb8e4ce 100644 --- a/data/sv_SE/username.json +++ b/data/sv_SE/username.json @@ -4,21 +4,20 @@ "weight": 2, "first": ["agnes", "alice", "anders", "anna", "asa", "axel", "bjorn", "ebba", "elin", "emil", "emma", "erik", "frida", "gustav", "hanna", "hugo", "ida", "johan", "jonas", "julia", "karin", "klara", "lars", "lena", "magnus", "maja", "moa", "nils", "oskar", "per", "sara", "sofia", "viktor", "ylva"], "last": ["ahl", "alm", "berg", "bjork", "dahl", "ek", "falk", "gran", "hag", "hed", "holm", "lind", "lund", "mark", "nor", "ohman", "ros", "sand", "sjo", "skog", "sten", "strom", "vik"], - "sep": ["", "", ".", "_"], - "n": ["", "", "", "1", "7", "23", "99", "2024"] + "sep": [{ "format": "", "weight": 2 }, ".", "_"], + "n": [{ "format": "", "weight": 3 }, "1", "7", "23", "99", "2024"] }, { "format": "{adj}{sep}{noun}{n}", "weight": 2, "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], "noun": ["abborre", "bamse", "bjorn", "drake", "ekorre", "falk", "groda", "hund", "igelkott", "kanin", "katt", "korp", "krabba", "mus", "nalle", "ninja", "orn", "panda", "pirat", "rav", "robot", "sill", "sparv", "uggla", "varg"], - "sep": ["", "", ".", "_"], - "n": ["", "", "7", "13", "42", "77", "99", "123", "420", "2024"] + "sep": [{ "format": "", "weight": 2 }, ".", "_"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "42", "77", "99", "123", "420", "2024"] }, { "format": "{w}{n}", - "weight": 1, "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] } ] diff --git a/data/sv_SE/version.json b/data/sv_SE/version.json index f0806d4..43be399 100644 --- a/data/sv_SE/version.json +++ b/data/sv_SE/version.json @@ -1,8 +1,6 @@ -[ - { - "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "{digits(1)}", "weight": 3 }, { "format": "{int(10,99)}" }], - "pre": ["", { "format": "v", "weight": 0.4 }], - "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] - } -] +{ + "format": "{pre}{n}.{n}.{n}{suffix}", + "n": [{ "format": "{digits(1)}", "weight": 3 }, "{int(10,99)}"], + "pre": ["", { "format": "v", "weight": 0.4 }], + "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] +} diff --git a/fejkdata.go b/fejkdata.go index b0fecb8..adff4aa 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -109,10 +109,9 @@ func New(opts ...Option) (*Generator, error) { } // List returns the sorted dotted paths Fake can render: every category, the dotted -// fields within a template, and folder segments — descending transparently through -// single-variant choices the way a reference does. A choice consumes no segment, so -// a path continues through a multi-variant one only where every variant carries it, -// which is the rule Fake applies too: List is the set of paths Fake accepts. +// fields within a template, and folder segments. A choice consumes no segment, so a +// path continues through one only where every variant carries it, which is the +// rule Fake applies too: List is the set of paths Fake accepts. func (f *Generator) List() []string { var out []string for _, name := range sortedNames(f.categories) { @@ -148,9 +147,6 @@ func paths(n node) []string { } return out case *choice: - if len(n.items) == 1 { - return paths(n.items[0]) - } out := []string{""} for p := range n.shared { out = append(out, p) @@ -161,7 +157,7 @@ func paths(n node) []string { } // sharedPaths is the sub-paths every item carries — the only ones a path may step -// through a multi-variant choice to reach. It intersects, bailing as soon as the set +// through a choice to reach. It intersects, bailing as soon as the set // is empty, which is immediate for a choice of plain strings. func sharedPaths(items []node) map[string]bool { shared := subPaths(items[0]) diff --git a/format-migration/reshape.py b/format-migration/reshape.py new file mode 100755 index 0000000..81e5d8d --- /dev/null +++ b/format-migration/reshape.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Rewrite data files to the one spelling each shape has. + + reshape.py data/sv_SE/person.json ... + +A one-item choice becomes its item; an object holding only a format becomes that +string; weight 1 and repeat 1 are dropped; a repeated choice item becomes one +item with a weight. +""" +import json +import sys + +OPTIONS = ("format", "weight", "repeat", "separator") +WIDTH = 110 + + +def reshape(n): + if isinstance(n, list): + items = [reshape(x) for x in n] + if len(items) == 1: + return items[0] + out, at = [], {} + for x in items: + key = json.dumps(x, sort_keys=True) + if key in at: + i = at[key] + if isinstance(out[i], str): + out[i] = {"format": out[i], "weight": 2} + elif isinstance(out[i], dict): + out[i]["weight"] = out[i].get("weight", 1) + 1 + else: + out.append(x) + continue + at[key] = len(out) + out.append(x) + return out + if isinstance(n, dict): + d = {k: (v if k in OPTIONS else reshape(v)) for k, v in n.items()} + if d.get("weight") == 1: + del d["weight"] + if d.get("repeat") == 1: + del d["repeat"] + if list(d) == ["format"]: + return d["format"] + return d + return n + + +def scalar(v): + return json.dumps(v, ensure_ascii=False) + + +def one_line(n): + if isinstance(n, dict): + return "{ " + ", ".join(scalar(k) + ": " + one_line(v) for k, v in n.items()) + " }" + if isinstance(n, list): + return "[" + ", ".join(one_line(x) for x in n) + "]" + return scalar(n) + + +def dump(n, indent=0): + pad = " " * indent + if isinstance(n, dict): + line = one_line(n) + if len(pad) + len(line) <= WIDTH and not any(isinstance(v, dict) for v in n.values()): + return line + body = ",\n".join(pad + " " + scalar(k) + ": " + dump(v, indent + 1) for k, v in n.items()) + return "{\n" + body + "\n" + pad + "}" + if isinstance(n, list): + if all(isinstance(x, str) for x in n): + return one_line(n) + line = one_line(n) + if len(pad) + len(line) <= WIDTH: + return line + body = ",\n".join(pad + " " + dump(x, indent + 1) for x in n) + return "[\n" + body + "\n" + pad + "]" + return scalar(n) + + +if __name__ == "__main__": + for path in sys.argv[1:]: + with open(path) as fh: + before = json.load(fh) + after = dump(reshape(before)) + "\n" + with open(path, "w") as fh: + fh.write(after) + print("reshaped", path) diff --git a/node.go b/node.go index b9cda30..25cabb3 100644 --- a/node.go +++ b/node.go @@ -1,6 +1,7 @@ package fejkdata import ( + "encoding/json" "fmt" "math" "sort" @@ -107,6 +108,12 @@ func compileChoice(items []any) (node, error) { if len(items) == 0 { return nil, fmt.Errorf("empty choice") } + if len(items) == 1 { + return nil, fmt.Errorf("a one-item choice is its item; write the item") + } + if err := checkNoRepeatedItem(items); err != nil { + return nil, err + } c := &choice{items: make([]node, len(items))} cum := make([]float64, len(items)) var total float64 @@ -133,14 +140,33 @@ func compileChoice(items []any) (node, error) { } c.cum = cum } - if len(c.items) > 1 { - // Safe to precompute: a choice's items come from one file, so no group can - // appear inside one, and neither mergeChildren nor linkRefs can reach in. - c.shared = sharedPaths(c.items) - } + // Safe to precompute: a choice's items come from one file, so no group can + // appear inside one, and neither mergeChildren nor linkRefs can reach in. + c.shared = sharedPaths(c.items) return c, nil } +// checkNoRepeatedItem rejects a choice that lists one item twice: a pick is even +// over the items, so a repeat is a second spelling of weight. The error names the +// spelling that does skew a pick. +func checkNoRepeatedItem(items []any) error { + seen := make(map[string]int, len(items)) + for i, raw := range items { + key, err := json.Marshal(raw) + if err != nil { + return err + } + if j, dup := seen[string(key)]; dup { + if s, isString := raw.(string); isString { + return fmt.Errorf("choice item %q is repeated; skew the odds with a weight instead: { \"format\": %q, \"weight\": 2 }", s, s) + } + return fmt.Errorf("choice item %d repeats item %d; skew the odds with a weight on one of them instead", i, j) + } + seen[string(key)] = i + } + return nil +} + func compileTemplate(m map[string]any) (node, error) { format, ok := m["format"].(string) if !ok { @@ -181,6 +207,9 @@ func compileTemplate(m map[string]any) (node, error) { } t.fields[k] = n } + if _, weighted := m["weight"]; len(t.fields) == 0 && repeat == 1 && !weighted { + return nil, fmt.Errorf("an object holding only a format is a string; write %q", format) + } if err := checkTokens(format, t.fields); err != nil { return nil, err } @@ -212,27 +241,24 @@ func checkPath(n node, tail []string, level string) error { } return checkPath(child, tail[1:], level+"."+tail[0]) case *choice: - if len(n.items) > 1 { - if want := strings.Join(tail, "."); !n.shared[want] { - return unreachableInChoice(n, want) - } - // Reachability is settled; each variant still answers for itself, so a - // rule about the level (its repeat) holds behind a choice as in front. - for _, item := range n.items { - if err := checkPath(item, tail, level); err != nil { - return err - } - } - return nil + if want := strings.Join(tail, "."); !n.shared[want] { + return unreachableInChoice(n, want) } - return checkPath(n.items[0], tail, level) + // Reachability is settled; each variant still answers for itself, so a + // rule about the level (its repeat) holds behind a choice as in front. + for _, item := range n.items { + if err := checkPath(item, tail, level); err != nil { + return err + } + } + return nil default: return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) } } // repeatOf reads a template's "repeat" (default 1): how many times its format -// is rendered and concatenated. A present one must be a positive integer. +// is rendered and concatenated. A present one must be an integer above 1. func repeatOf(m map[string]any) (int, error) { rv, ok := m["repeat"] if !ok { @@ -245,6 +271,9 @@ func repeatOf(m map[string]any) (int, error) { if math.IsNaN(r) || math.IsInf(r, 0) || r < 1 || r != math.Trunc(r) { return 0, fmt.Errorf("repeat must be a positive integer, got %v", rv) } + if r == 1 { + return 0, fmt.Errorf("repeat 1 is the default, so it has no effect; drop it") + } if r > maxLen { // cap so a fat-fingered repeat can't build a multi-GB string return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, maxLen) } @@ -269,6 +298,9 @@ func weightOf(raw any) (float64, error) { if w < 0 || math.IsNaN(w) || math.IsInf(w, 0) { return 0, fmt.Errorf("weight must be finite and non-negative, got %v", w) } + if w == 1 { + return 0, fmt.Errorf("weight 1 is the default, so it has no effect; drop it") + } return w, nil } diff --git a/reference.go b/reference.go index 97d108e..c8dd03d 100644 --- a/reference.go +++ b/reference.go @@ -9,8 +9,7 @@ import ( // refPrefix marks a {..path} token: a reference to a node elsewhere in the data // root rather than a sibling field. The path is resolved across every loaded // directory (see linkRefs), and is stricter than the one Fake takes: a reference -// binds one node, so it cannot step through a multi-variant choice even where Fake -// and List can. +// binds one node, so it cannot step through a choice even where Fake and List can. const refPrefix = ".." func isRef(name string) bool { return strings.HasPrefix(name, refPrefix) } @@ -32,6 +31,9 @@ func linkRefs(root map[string]node) error { if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } + if t.fields == nil { + t.fields = map[string]node{} + } t.fields[name] = target } return nil @@ -231,9 +233,8 @@ func sortedNames(m map[string]node) []string { } // lookup finds the single node a reference path names, walking groups and -// template fields by segment and descending a single-variant choice as a -// transparent wrapper. A missing segment, a folder target, or a step through a -// multi-variant choice (which has no one value to bind) is an error. +// template fields by segment. A missing segment, a folder target, or a step +// through a choice (which has no one value to bind) is an error. func lookup(root map[string]node, segments []string) (node, error) { var n node = &group{children: root} for i := 0; i < len(segments); i++ { @@ -251,10 +252,7 @@ func lookup(root map[string]node, segments []string) (node, error) { } n = child case *choice: - if len(c.items) != 1 { - return nil, fmt.Errorf("%q steps through a %d-way choice", segments[i], len(c.items)) - } - n, i = c.items[0], i-1 // a choice consumes no segment; reprocess it unwrapped + return nil, fmt.Errorf("%q steps through a %d-way choice", segments[i], len(c.items)) default: return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[i]) } diff --git a/render.go b/render.go index 4139ab7..82bb26f 100644 --- a/render.go +++ b/render.go @@ -53,10 +53,8 @@ func descend(s *session, n node, segments []string) (node, error) { } return descend(s, child, segments[1:]) case *choice: - if len(n.items) > 1 { - if want := strings.Join(segments, "."); !n.shared[want] { - return nil, unreachableInChoice(n, want) - } + if want := strings.Join(segments, "."); !n.shared[want] { + return nil, unreachableInChoice(n, want) } return descend(s, pick(s, n), segments) default: -- 2.52.0 From f0305f191f02b54b5df96fe364f3e14d72b4b380 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:39:04 +0200 Subject: [PATCH 09/38] Re-pin the seeded outputs now that a one-item choice draws nothing --- template_stability_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template_stability_test.go b/template_stability_test.go index 4e410ed..7204a23 100644 --- a/template_stability_test.go +++ b/template_stability_test.go @@ -19,13 +19,13 @@ func TestSeededOutputIsStable(t *testing.T) { "weights": `[{"format":"big","weight":9},"tiny"]`, }) want := map[string][]string{ - "alt": {"A", "B", "A", "A"}, + "alt": {"A", "A", "B", "A"}, "calc": {"100.00 x 3 = 300.00", "100.00 x 3 = 300.00", "100.00 x 7 = 700.00", "5.00 x 3 = 15.00"}, "classes": {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"}, "escapes": {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"}, "funcs": {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"}, - "nested": {"i-89", "i-67", "i-47", "i-27"}, - "ref": {"see A", "see A", "see B", "see A"}, + "nested": {"i-73", "i-23", "i-67", "i-85"}, + "ref": {"see A", "see A", "see A", "see B"}, "repeat": {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"}, "sums": {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"}, "weights": {"big", "big", "big", "big"}, -- 2.52.0 From 8fa33694b9b38b22b6f7d23235b063e1d901f9f4 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:42:21 +0200 Subject: [PATCH 10/38] Tests for references held like siblings and the lowercase, uppercase and ascii transforms --- calc_test.go | 12 ++++----- reference_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++---- transform_test.go | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 transform_test.go diff --git a/calc_test.go b/calc_test.go index b9d6072..ab38c72 100644 --- a/calc_test.go +++ b/calc_test.go @@ -88,18 +88,18 @@ func TestCalcReproducible(t *testing.T) { } } -// TestCalcTokenOperandsReadsOnlyACalc pins what the helper answers for a body that +// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that // is not a calc, and for one whose expression does not parse: nothing, either way. // checkCalc is what reports a bad expression, so the callers that run after it // never meet one — but they must not have to depend on that order to be safe. -func TestCalcTokenOperandsReadsOnlyACalc(t *testing.T) { +func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) { for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} { - if got := calcTokenOperands(body); got != nil { - t.Errorf("calcTokenOperands(%q) = %v, want none", body, got) + if got := tokenOperands(body); got != nil { + t.Errorf("tokenOperands(%q) = %v, want none", body, got) } } - if got := calcTokenOperands("calc(net * qty)"); len(got) != 2 { - t.Errorf("calcTokenOperands(calc(net * qty)) = %v, want both operands", got) + if got := tokenOperands("calc(net * qty)"); len(got) != 2 { + t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got) } } diff --git a/reference_test.go b/reference_test.go index 7ac7b52..90d35b6 100644 --- a/reference_test.go +++ b/reference_test.go @@ -1,6 +1,9 @@ package fejkdata -import "testing" +import ( + "strings" + "testing" +) // TestRootReferenceAcrossFolders is the headline case: a category in one folder // pulls a value from another via a {..path} reference resolved from the data root. @@ -15,8 +18,7 @@ func TestRootReferenceAcrossFolders(t *testing.T) { } } -// TestReferenceIntoAField reaches a field inside a referenced category, crossing -// a single-variant choice and then a template field (..who.last). +// TestReferenceIntoAField reaches a field inside a referenced category. func TestReferenceIntoAField(t *testing.T) { dir := writeData(t, map[string]string{ "who": `{"format":"{first} {last}","first":"Ada","last":"Byron"}`, @@ -76,8 +78,8 @@ func TestReferenceErrors(t *testing.T) { cases := map[string]map[string]string{ "missing target": {"card": `"{..nope.gone}"`}, "folder target": {"en_US/word": `"w"`, "card": `"{..en_US}"`}, - "multi-variant on the path": { - "who": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, + "a variant on the path lacks the field": { + "who": `[{"format":"{f}","f":"1"},{"format":"{g}","g":"2"}]`, "card": `"{..who.f}"`, }, "empty reference path": {"card": `"{..}"`}, @@ -208,3 +210,58 @@ func TestNewErrorPathIsCanonical(t *testing.T) { } } } + +func TestReferencePathIsHeld(t *testing.T) { + dir := writeData(t, map[string]string{ + "person": `[{"format":"{first} {last}","first":"Anna","last":"Andersson"},{"format":"{first} {last}","first":"Bo","last":"Berg"}]`, + "card": `"{..person.first} {..person.last}"`, + }) + f := newGenerator(t, dir, WithSeed(3)) + seen := map[string]bool{} + for i := 0; i < 50; i++ { + got := fake(t, f, "card") + if got != "Anna Andersson" && got != "Bo Berg" { + t.Fatalf("card = %q, want one person's first and last name", got) + } + seen[got] = true + } + if len(seen) != 2 { + t.Fatalf("card only ever rendered %v", seen) + } +} + +func TestReferenceThroughChoiceNeedsEveryVariant(t *testing.T) { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "who": `[{"format":"{f}{h}","f":"1","h":"x"},{"format":"{g}{h}","g":"2","h":"y"}]`, + "card": `"{..who.f}"`, + }))) + if err == nil || !strings.Contains(err.Error(), "not every variant") { + t.Fatalf("New = %v, want the missing variant named", err) + } +} + +func TestBareReferenceDrawsEachTime(t *testing.T) { + dir := writeData(t, map[string]string{ + "die": `["1","2","3","4","5","6"]`, + "roll": `"{..die} {..die}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "roll"); got[0] != got[2] { + return + } + } + t.Fatal("two bare references always agreed; each should be its own draw") +} + +func TestReferenceOverlapIsRejected(t *testing.T) { + for name, file := range map[string]string{ + "head beside a path": `{"format":"{..cat.p} {..cat.p.first}","p":[{"format":"{first}","first":"A"},{"format":"{first}","first":"B"}]}`, + "sibling path beside a reference path": `{"format":"{p.first} {..cat.p.last}","p":[{"format":"{first}","first":"A","last":"1"},{"format":"{first}","first":"B","last":"2"}]}`, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) + if err == nil || !strings.Contains(err.Error(), "reads a path into") { + t.Errorf("%s: New = %v, want the overlap rejected", name, err) + } + } +} diff --git a/transform_test.go b/transform_test.go new file mode 100644 index 0000000..31d8279 --- /dev/null +++ b/transform_test.go @@ -0,0 +1,53 @@ +package fejkdata + +import "testing" + +func TestTransforms(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `{"format":"{uppercase(x)}|{lowercase(x)}|{ascii(x)}","x":"Åsa Ödegård-Nuñez"}`: "ÅSA ÖDEGÅRD-NUÑEZ|åsa ödegård-nuñez|Asa Odegard-Nunez", + `{"format":"{lowercase(ascii(x))}","x":"Åsa"}`: "asa", + `{"format":"{ascii(x)}","x":"Ærø ß 日本"}`: "AEro ss ", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } +} + +func TestTransformReadsTheHeldDraw(t *testing.T) { + f := engine(2) + src := `{"format":"{p.first}={lowercase(p.first)}","p":[{"format":"{first}","first":"Anna"},{"format":"{first}","first":"Bo"}]}` + for i := 0; i < 50; i++ { + if got := mustRender(t, f, src); got != "Anna=anna" && got != "Bo=bo" { + t.Fatalf("render = %q, want the transform over the same draw", got) + } + } +} + +func TestTransformOverAReference(t *testing.T) { + dir := writeData(t, map[string]string{ + "person": `[{"format":"{first} {last}","first":"Åsa","last":"Öberg"},{"format":"{first} {last}","first":"Bo","last":"Ek"}]`, + "email": `"{..person.first} {..person.last} <{lowercase(ascii(..person.first))}.{lowercase(ascii(..person.last))}@example.com>"`, + }) + f := newGenerator(t, dir, WithSeed(5)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "email"); got != "Åsa Öberg " && got != "Bo Ek " { + t.Fatalf("email = %q, want a record from one draw", got) + } + } +} + +func TestTransformArgs(t *testing.T) { + for _, bad := range []string{ + `{"format":"{lowercase()}","x":"v"}`, + `{"format":"{lowercase(nope)}","x":"v"}`, + `{"format":"{ascii(x,y)}","x":"v","y":"w"}`, + `{"format":"{lowercase(hex(2))}","x":"v"}`, + `{"format":"{lowercase(x)} {x.a}","x":{"format":"{a}","a":"1"}}`, + } { + if _, err := compile(parse(t, bad)); err == nil { + t.Errorf("compile(%s) = nil error, want it rejected", bad) + } + } +} -- 2.52.0 From 31c2f12098e9070bdee5334fbbadb366491119e4 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:52:19 +0200 Subject: [PATCH 11/38] Hold a reference path like a sibling path; add the lowercase, uppercase and ascii transforms --- README.md | 16 ++-- builtins.go | 100 ++++++++++++++++++++- calc.go | 24 +---- node.go | 17 ++-- reference.go | 221 ++++++++++++++++++++++++++++++---------------- render.go | 4 +- template.go | 160 +++++++++++++++++++++------------ transform_test.go | 4 +- 8 files changed, 372 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 5c647bb..b337a5c 100644 --- a/README.md +++ b/README.md @@ -273,10 +273,11 @@ reproducible. A time-based id (UUID v7, ULID) therefore draws its timestamp from the rng, not the clock — the result is a valid, reproducible value, not a real point in time. -There are four kinds. **Derivations** read the digits emitted so far, so put them +There are five kinds. **Derivations** read the digits emitted so far, so put them after their payload; **samples** read only the rng, so they stand alone; one -**session counter** (`seq`) advances state held on the generator; and one -**computation** (`calc`) evaluates arithmetic over sibling fields. Arguments are +**session counter** (`seq`) advances state held on the generator; one +**computation** (`calc`) evaluates arithmetic over sibling fields; and the +**transforms** rewrite one field's value. Arguments are validated at `New` (a bad count, range, country, or expression fails fast); a length, count or decimal place beyond a sane maximum is rejected there too, so a fat-fingered `hex(2000000000)` can't try to allocate gigabytes at render. @@ -299,6 +300,7 @@ fat-fingered `hex(2000000000)` can't try to allocate gigabytes at render. | `{iban(CC)}` | sample | a length- and mod-97-valid IBAN for country `CC` (BE, DE, DK, ES, FI, NO, SE) | | `{seq()}`, `{seq(name)}` | session counter | next integer (from 1) in this generator's sequence; `name` selects an independent counter | | `{calc(expr)}`, `{calc(expr,dp)}` | computation | value of an arithmetic expression over number literals and sibling fields; `dp` rounds | +| `{lowercase(x)}`, `{uppercase(x)}`, `{ascii(x)}` | transform | the field `x` — a name, a path or a `..path` — lower-cased, upper-cased, or folded to ASCII (`Åsa` → `Asa`); they nest: `{lowercase(ascii(x))}` | `{ean()}` is also the ISBN-13 check (an ISBN-13 *is* an EAN-13 — build the 978/979 prefix in data and call `{ean()}`). `{iban()}` is a sample, not a derivation: @@ -339,9 +341,11 @@ data dirs: { "format": "Hej, {..en_US.person}!" } ``` -renders e.g. `Hej, Pat Smith!`. References are bound when you create the generator, so -a path that is unknown, names a folder, or steps through a choice -fails at `New`. A reference that leads back to its own value (directly, mutually, +renders e.g. `Hej, Pat Smith!`. A reference path is held like a sibling path: +`{..person.first} {..person.last}` read one person, and `{lowercase(..person.first)}` +reads that same draw, while a bare `{..die} {..die}` is two draws. References are +bound when you create the generator, so a path that is unknown, names a folder, +or reads a field not every variant of a choice carries fails at `New`. A reference that leads back to its own value (directly, mutually, or through a chain) is a cycle that would never finish rendering, so it too is rejected at `New`. diff --git a/builtins.go b/builtins.go index eca7bba..d264f00 100644 --- a/builtins.go +++ b/builtins.go @@ -6,6 +6,7 @@ import ( "math" "strconv" "strings" + "unicode" ) // maxLen caps sample output lengths (hex, nanoid, base64) and maxDecimals caps @@ -51,9 +52,10 @@ var builtins = map[string]builtin{ cc := a[0] return func(s *session, _ string, _ []string) string { return iban(s, cc) } }}, - // calc is the one builtin that names operands (expand reads them for it); - // every other ignores them. 1 or 2 args: the expression and an optional decimals. - "calc": {arity: -1, check: checkCalc, prep: calcPrep}, + "calc": {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands}, + "lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand}, + "uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand}, + "ascii": {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand}, // seq is the one stateful builtin: a per-session counter from 1, advancing on // each call. An optional name selects an independent counter; no name uses the // default one. Deterministic by construction, so seeded output stays stable. @@ -92,6 +94,98 @@ func chars(alphabet string) func([]string) callFn { const hexDigits = "0123456789abcdef" +// transforms are the builtins that rewrite one operand's value; they nest, so +// {lowercase(ascii(x))} folds then lowers. +var transforms = map[string]func(string) string{ + "ascii": asciiFold, + "lowercase": strings.ToLower, + "uppercase": strings.ToUpper, +} + +// unwrapTransform peels nested transform calls off an operand arg, returning the +// field it finally names and the transforms to apply, innermost last. +func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) { + for { + name, args, isCall := funcCall(arg) + if !isCall { + return arg, chain, nil + } + fn, isTransform := transforms[name] + if !isTransform { + return "", nil, fmt.Errorf("%s(%s) is not a transform, so it cannot be an operand", name, strings.Join(args, ",")) + } + if len(args) != 1 { + return "", nil, fmt.Errorf("%s takes 1 arg, got %d", name, len(args)) + } + chain = append(chain, fn) + arg = args[0] + } +} + +func transformArg(fields map[string]node, a []string) error { + leaf, _, err := unwrapTransform(a[0]) + if err != nil { + return err + } + if isRef(leaf) { + if leaf == refPrefix { + return fmt.Errorf("reference has no path") + } + return nil + } + return checkArm(leaf, fields) +} + +func transformOperand(a []string) []string { + leaf, _, err := unwrapTransform(a[0]) + if err != nil { + return nil + } + return []string{leaf} +} + +func transformPrep(outer func(string) string) func([]string) callFn { + return func(a []string) callFn { + _, chain, err := unwrapTransform(a[0]) + if err != nil { + panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err)) + } + return func(_ *session, _ string, operands []string) string { + v := operands[0] + for i := len(chain) - 1; i >= 0; i-- { + v = chain[i](v) + } + return outer(v) + } + } +} + +// asciiFolds maps the Latin letters with diacritics or ligatures to ASCII. +var asciiFolds = map[rune]string{ + 'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", 'Å': "A", 'Æ': "AE", 'Ç': "C", + 'È': "E", 'É': "E", 'Ê': "E", 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", + 'Ð': "D", 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", 'Ø': "O", + 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", 'Þ': "Th", 'ß': "ss", 'Œ': "OE", + 'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", + 'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", + 'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", + 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'þ': "th", 'ÿ': "y", 'œ': "oe", +} + +// asciiFold rewrites s to ASCII: folded Latin letters stay, any other non-ASCII +// rune is dropped. +func asciiFold(s string) string { + var b strings.Builder + for _, r := range s { + if r < unicode.MaxASCII { + b.WriteRune(r) + } else { + b.WriteString(asciiFolds[r]) + } + } + return b.String() +} + // atoi parses an arg a builtin's check already validated. It panics rather than // returning zero, so a check that stops covering its own args is a stack trace and // not a silently wrong length, range or decimal count. diff --git a/calc.go b/calc.go index f0c5e4b..4d068e4 100644 --- a/calc.go +++ b/calc.go @@ -122,26 +122,10 @@ func indexVars(n calcNode, at map[string]int) calcNode { return n } -// calcOperands lists the sibling-field names every {calc(...)} token in a format -// reads, so cycle detection sees the field edges calc renders through (a function -// token otherwise carries no field edge). -func calcOperands(format string) []string { - var names []string - _ = eachToken(format, func(t ftoken) error { - if t.kind == 'b' { - names = append(names, calcTokenOperands(t.body)...) - } - return nil - }) - return names -} - -// calcTokenOperands lists the sibling-field names one {token} body reads, empty -// for anything that is not a {calc(...)}. checkCalc reports an expression that does -// not parse, so one that does not simply names nothing here. -func calcTokenOperands(body string) []string { - name, args, ok := funcCall(body) - if !ok || name != "calc" || len(args) == 0 { +// calcOperands lists the sibling-field names a calc's args read. checkCalc reports +// an expression that does not parse, so one that does not simply names nothing. +func calcOperands(args []string) []string { + if len(args) == 0 { return nil } expr, err := parseCalc(args[0]) diff --git a/node.go b/node.go index 25cabb3..7859521 100644 --- a/node.go +++ b/node.go @@ -41,10 +41,11 @@ type template struct { fields map[string]node repeat int separator string - ops []op // format compiled once (see compileOps); what expand walks - grow int // minimum output size, to size the render buffer - fixed bool // no op varies, so every render is lit - lit string // the whole output when fixed + ops []op // format compiled once (see compileOps); what expand walks + grow int // minimum output size, to size the render buffer + fixed bool // no op varies, so every render is lit + lit string // the whole output when fixed + refs map[string]string // each {..path} the format reads -> the head it is bound under // bound maps each field the format addresses by dotted path to one path token // reading it, which is the half of an overlap the fences name. nil when the // format takes no path. @@ -92,7 +93,7 @@ func compileString(s string) (node, error) { // compileFormat compiles the format into ops once every field is in place. func (t *template) compileFormat() { - t.ops, t.grow, t.bound, t.held = compileOps(t.format) + t.ops, t.grow, t.bound, t.held = compileOps(t.format, t.refs) t.fixed = true for _, o := range t.ops { if o.kind != 'l' { @@ -214,15 +215,15 @@ func compileTemplate(m map[string]any) (node, error) { return nil, err } t.compileFormat() - if err := checkNoOverlap(format, t.bound); err != nil { + if err := checkNoOverlap(format, t.bound, nil); err != nil { return nil, err } return t, nil } // checkPath reports whether a token's dotted tail can address a node whichever way -// the draw goes, by the reachability rule descend applies — a multi-variant choice -// must carry the whole remaining path in the set every variant shares — plus the +// the draw goes, by the reachability rule descend applies — a choice must carry +// the whole remaining path in the set every variant shares — plus the // rules a held draw adds, which descend has no need of: a level a path reads may // not carry a repeat, and each variant answers for that itself. So a path that // validates here resolves on every render, and a typo is a New-time error. diff --git a/reference.go b/reference.go index c8dd03d..aac8a1d 100644 --- a/reference.go +++ b/reference.go @@ -8,33 +8,47 @@ import ( // refPrefix marks a {..path} token: a reference to a node elsewhere in the data // root rather than a sibling field. The path is resolved across every loaded -// directory (see linkRefs), and is stricter than the one Fake takes: a reference -// binds one node, so it cannot step through a choice even where Fake and List can. +// directory (see linkRefs). const refPrefix = ".." func isRef(name string) bool { return strings.HasPrefix(name, refPrefix) } -// linkRefs resolves every {..path} reference in the assembled tree, binding the -// target node into the referring template's fields under the token's key so the -// ordinary resolver renders it like a sibling. It runs once, after all data is -// merged, so a reference sees the final (override-resolved) tree. A path that is -// unknown, names a folder, or steps through a multi-variant choice fails here, -// keeping a bad reference a New-time error, never a random render-time one. +// linkRefs resolves every {..path} reference in the assembled tree. The head of the +// path — up to the category it names — is bound into the referring template's +// fields, and the rest reads into it the way a sibling path does, so a reference +// is held like a sibling. It runs once, after all data is merged, so a reference +// sees the final (override-resolved) tree. A path that is unknown, names a folder, +// or reads a field not every variant carries fails here, keeping a bad reference a +// New-time error, never a random render-time one. func linkRefs(root map[string]node) error { return walkNodes(root, func(path string, n node) error { t, ok := n.(*template) if !ok { return nil } - for _, name := range refTokens(t.format) { - target, err := lookup(root, strings.Split(name[len(refPrefix):], ".")) + names := refTokens(t.format) + if len(names) == 0 { + return nil + } + if t.fields == nil { + t.fields = map[string]node{} + } + t.refs = make(map[string]string, len(names)) + for _, name := range names { + head, target, tail, err := resolveRef(root, strings.Split(name[len(refPrefix):], ".")) if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } - if t.fields == nil { - t.fields = map[string]node{} + key := refPrefix + strings.Join(head, ".") + if err := checkPath(target, tail, key); err != nil { + return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } - t.fields[name] = target + t.fields[key] = target + t.refs[name] = key + } + t.compileFormat() + if err := checkNoOverlap(t.format, t.bound, t.refs); err != nil { + return fmt.Errorf("%s: %w", path, err) } return nil }) @@ -43,7 +57,7 @@ func linkRefs(root map[string]node) error { // checkBoundLevelsHeld rejects every route to a held name except the ones that read // its draw. An expansion holds one draw of that name; anything else that renders it // draws again, and the two disagree. checkNoOverlap settles the spellings within one -// format (a token, a calc operand); this settles the rest — a reference, whether it +// format (a token, an operand); this settles the rest — a reference, whether it // sits in that format or in anything the format renders, however deep. // // It runs after checkNoCycles, whose guarantee is what lets the walk terminate. @@ -57,33 +71,48 @@ func checkBoundLevelsHeld(root map[string]node) error { for head := range t.held { heads = append(heads, head) } - sort.Strings(heads) // so which overlap is reported does not vary + // Operand heads first, then paths, each in name order: a level read both + // ways is reported by the operand's fence, and which overlap is reported + // does not vary. + sort.Slice(heads, func(i, j int) bool { + _, pi := t.bound[heads[i]] + _, pj := t.bound[heads[j]] + if pi != pj { + return !pi + } + return heads[i] < heads[j] + }) + readers := boundReaders(t.format, t.bound, t.refs) for _, head := range heads { - // What one draw answers for depends on how the draw is read. A path may - // read into anything the level contains; a calc renders its operand, so - // that draw fixes exactly the value the render produces. + // What one draw answers for depends on how the draw is read: a path pins + // the levels it passes through and the leaf it lands on, an operand + // exactly the value its render produces. held := map[node]bool{} reader, isPath := t.bound[head] if isPath { - cover(t.fields[head], held) + for _, r := range readers { + if a := splitArm(r.name, t.refs); a.key == head { + coverPath(t.fields[head], a.tail, held) + } + } } else { operandDraw(t.fields[head], held) } if len(held) == 0 { - continue // an early out: a literal head holds nothing to reach + continue // an early out: a fixed head holds nothing to reach } // One seen set across the edges: a node that cannot reach the level // cannot reach it by another route either, so it is walked once here. seen := map[node]bool{} for _, e := range renderEdges(t) { - if splitArm(e.label).key == head { + if splitArm(e.label, t.refs).key == head { continue // a token or operand reading this draw, the routes allowed } if renders(e.to, held, seen) { if isPath { return fmt.Errorf("%s: %s renders %q, which {%s} reads a path into; name the fields you want instead", path, e.reached(), head, reader) } - return fmt.Errorf("%s: %s renders %q, which a {calc()} also reads; reach it one way so it is drawn once", path, e.reached(), head) + return fmt.Errorf("%s: %s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", path, e.reached(), head, operandReader(t, head)) } } } @@ -91,28 +120,63 @@ func checkBoundLevelsHeld(root map[string]node) error { }) } -// cover collects what one held draw of a level answers for: the level and -// everything contained in it, since a path may read any of it. A fixed string is -// left out — it cannot disagree with itself. -func cover(n node, into map[node]bool) { - if isFixed(n) { +// operandReader names the builtin whose operand holds head. +func operandReader(t *template, head string) string { + fn := "" + _ = eachToken(t.format, func(tok ftoken) error { + if tok.kind != 'b' || fn != "" { + return nil + } + if name, _, isFunc := funcCall(tok.body); isFunc { + for _, operand := range tokenOperands(tok.body) { + if splitArm(operand, t.refs).key == head { + fn = name + } + } + } + return nil + }) + return fn +} + +// coverPath collects what holding one path pins: every choice level the path +// passes through, whole, and the leaf it renders. +func coverPath(n node, tail []string, into map[node]bool) { + if _, isChoice := n.(*choice); isChoice || len(tail) == 0 { + cover(n, into, false) return } - into[n] = true - for _, c := range contained(n) { - cover(c.node, into) + t, ok := n.(*template) + if !ok { + return + } + if child, ok := t.fields[tail[0]]; ok { + coverPath(child, tail[1:], into) } } -// operandDraw collects what one held draw of a {calc()} operand answers for: the -// operand and what rendering it settles inside itself. A calc renders its operand +// cover collects a level and everything contained in it. A fixed string outside a +// choice is left out — it cannot disagree with itself — but inside one each +// variant carries its own, so there it counts. +func cover(n node, into map[node]bool, inChoice bool) { + if isFixed(n) && !inChoice { + return + } + into[n] = true + _, isChoice := n.(*choice) + for _, c := range contained(n) { + cover(c.node, into, inChoice || isChoice) + } +} + +// operandDraw collects what one held draw of an operand answers for: the operand +// and what rendering it settles inside itself. The builtin renders its operand // whole, so that draw fixes every value the render produced, and a second route to // any of them disagrees with it. // // The walk stops at a {..path} edge, which is where the operand's own value ends // and a shared source begins: two names referencing one category are two draws, the -// same rule {word} {word} follows. cover stops there too, by way of named, so both -// halves of the fence end at the same boundary. +// same rule {word} {word} follows. func operandDraw(n node, into map[node]bool) { if isFixed(n) { return @@ -232,43 +296,35 @@ func sortedNames(m map[string]node) []string { return names } -// lookup finds the single node a reference path names, walking groups and -// template fields by segment. A missing segment, a folder target, or a step -// through a choice (which has no one value to bind) is an error. -func lookup(root map[string]node, segments []string) (node, error) { +// resolveRef walks a reference path through the folders to the category it names, +// returning that head, the node, and the tail left to read into it. +func resolveRef(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { var n node = &group{children: root} - for i := 0; i < len(segments); i++ { - switch c := n.(type) { - case *group: - child, ok := c.children[segments[i]] - if !ok { - return nil, fmt.Errorf("no entry %q", segments[i]) - } - n = child - case *template: - child, ok := c.fields[segments[i]] - if !ok { - return nil, fmt.Errorf("no field %q", segments[i]) - } - n = child - case *choice: - return nil, fmt.Errorf("%q steps through a %d-way choice", segments[i], len(c.items)) - default: - return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[i]) + i := 0 + for ; i < len(segments); i++ { + g, ok := n.(*group) + if !ok { + break } + child, ok := g.children[segments[i]] + if !ok { + return nil, nil, nil, fmt.Errorf("no entry %q", segments[i]) + } + n = child } if _, ok := n.(*group); ok { - return nil, fmt.Errorf("names a folder, not a value") + return nil, nil, nil, fmt.Errorf("names a folder, not a value") } - return n, nil + return segments[:i], n, segments[i:], nil } -// refTokens returns just the {..path} reference names among a format's field -// tokens (linkRefs binds each into the template's fields). +// refTokens returns the {..path} names a format reads, as tokens or as operands. func refTokens(format string) []string { var refs []string - for _, name := range fieldTokens(format) { - if isRef(name) { + seen := map[string]bool{} + for _, name := range append(fieldTokens(format), operandTokens(format)...) { + if isRef(name) && !seen[name] { + seen[name] = true refs = append(refs, name) } } @@ -276,27 +332,27 @@ func refTokens(format string) []string { } // renderEdge is a child a node renders into, labelled by what reaches it (a field -// name, reference, or choice index) for a readable cycle report. operand marks a -// label that is a {calc()} operand name rather than a token, so an error can name -// it the way the author wrote it. +// name, reference, or choice index) for a readable cycle report. operand names +// the builtin when the label is its operand rather than a token, so an error can +// name it the way the author wrote it. type renderEdge struct { to node label string - operand bool + operand string } // reached names an edge as the author spelled it, the vocabulary boundReaders uses // for the sibling fence. func (e renderEdge) reached() string { - if e.operand { - return fmt.Sprintf("calc operand %q", e.label) + if e.operand != "" { + return fmt.Sprintf("%s operand %q", e.operand, e.label) } return "{" + e.label + "}" } // renderEdges lists the children rendering n recurses into, mirroring expand: a -// choice's items, and a template's field/reference tokens plus its calc operands. -// A group renders nothing, so it has no edges. +// choice's items, and a template's field/reference tokens plus its operands. A +// group renders nothing, so it has no edges. func renderEdges(n node) []renderEdge { switch n := n.(type) { case *choice: @@ -307,8 +363,8 @@ func renderEdges(n node) []renderEdge { return es case *template: var es []renderEdge - add := func(name string, operand bool) { - a := splitArm(name) + add := func(name, operand string) { + a := splitArm(name, n.refs) c, ok := n.fields[a.key] if !ok { return @@ -317,12 +373,21 @@ func renderEdges(n node) []renderEdge { es = append(es, renderEdge{leaf, name, operand}) } } - for _, name := range fieldTokens(n.format) { - add(name, false) - } - for _, name := range calcOperands(n.format) { - add(name, true) - } + _ = eachToken(n.format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + add(operand, fn) + } + return nil + } + for _, name := range strings.Split(t.body, "|") { + add(name, "") + } + return nil + }) return es default: return nil diff --git a/render.go b/render.go index 82bb26f..df5391a 100644 --- a/render.go +++ b/render.go @@ -143,8 +143,8 @@ func expand(s *session, t *template) string { var operands []string if len(o.operands) > 0 { operands = make([]string, len(o.operands)) - for j, name := range o.operands { - operands[j] = readField(s, t, held, arm{name: name, key: name}) + for j, a := range o.operands { + operands[j] = readField(s, t, held, a) } } b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far diff --git a/template.go b/template.go index 76aed2b..fd6bb61 100644 --- a/template.go +++ b/template.go @@ -69,6 +69,9 @@ type builtin struct { // prep parses validated args once, at compile time, into the closure expand calls. prep func(args []string) callFn check func(fields map[string]node, args []string) error + // operands names the fields the call reads, which expand renders for it; nil + // for a builtin that reads none. + operands func(args []string) []string } // funcCall splits a "{token}" body shaped name(args) into its parts; ok is false @@ -137,23 +140,9 @@ func checkTokens(format string, fields map[string]node) error { } continue // a root reference; its target is checked at New (see linkRefs) } - a := splitArm(name) - if err := checkSegments(a); err != nil { + if err := checkArm(name, fields); err != nil { return fmt.Errorf("token {%s}: %w", t.body, err) } - head, ok := fields[a.key] - if !ok { - if a.key == "" { - return fmt.Errorf("token {%s}: a name is never empty, so this token can name no field", t.body) - } - if isOption(a.key) { - return fmt.Errorf("token {%s}: %q is an option and can never be a field", t.body, a.key) - } - return fmt.Errorf("token {%s}: no field %q", t.body, a.key) - } - if err := checkPath(head, a.tail, a.key); err != nil { - return fmt.Errorf("token {%s}: field %q: %w", t.body, a.key, err) - } } // Last, so an arm broken on its own terms is reported as that: a repeat is // the consequence of such a mistake, not the mistake itself. @@ -161,6 +150,54 @@ func checkTokens(format string, fields map[string]node) error { }) } +// checkArm validates one sibling name or path against a template's fields. +func checkArm(name string, fields map[string]node) error { + a := splitArm(name, nil) + if err := checkSegments(a); err != nil { + return err + } + head, ok := fields[a.key] + if !ok { + if a.key == "" { + return fmt.Errorf("a name is never empty, so this token can name no field") + } + if isOption(a.key) { + return fmt.Errorf("%q is an option and can never be a field", a.key) + } + return fmt.Errorf("no field %q", a.key) + } + if err := checkPath(head, a.tail, a.key); err != nil { + return fmt.Errorf("field %q: %w", a.key, err) + } + return nil +} + +// tokenOperands lists the fields one {token} body reads as operands, empty for a +// field token or a builtin that reads none. +func tokenOperands(body string) []string { + name, args, ok := funcCall(body) + if !ok { + return nil + } + b, known := builtins[name] + if !known || b.operands == nil { + return nil + } + return b.operands(args) +} + +// operandTokens lists every operand the builtins in a format read. +func operandTokens(format string) []string { + var names []string + _ = eachToken(format, func(t ftoken) error { + if t.kind == 'b' { + names = append(names, tokenOperands(t.body)...) + } + return nil + }) + return names +} + // checkNoRepeatedArm rejects {a|a|b}: an alternation picks its arms evenly, so a // repeated one is a second spelling of weight. The error names the spelling that // does skew a pick. @@ -192,10 +229,11 @@ func fieldTokens(format string) []string { return names } -// arm is one alternative of a {a|b} token, split into the key naming the node in -// a template's fields (a sibling field, or the whole "..path" string a reference is -// bound under) and the tail of a dotted path into it. A non-empty tail is what makes -// the arm a bound draw: its head is drawn once per expansion (see compileOps). +// arm is one alternative of a {a|b} token or one operand, split into the key +// naming the node in a template's fields (a sibling field, or the "..path" head a +// reference is bound under) and the tail of a dotted path into it. A non-empty +// tail is what makes the arm a bound draw: its head is drawn once per expansion +// (see compileOps). type arm struct { name string // as written, and the key a bound draw's value is held under key string @@ -203,22 +241,29 @@ type arm struct { steps []string // key per level passed through; the head and leaf hold their own } -// splitArm splits one token alternative into key and tail. A reference keeps its -// dots — linkRefs binds it whole — so only a sibling name reads as a path. -func splitArm(name string) arm { +// splitArm splits one name into key and tail. refs maps a reference to the head +// linkRefs bound it under; before linking, a reference is whole. +func splitArm(name string, refs map[string]string) arm { if isRef(name) { - return arm{name: name, key: name} + key, bound := refs[name] + if !bound || key == name { + return arm{name: name, key: name} + } + return pathArm(name, key, strings.Split(name[len(key)+1:], ".")) } head, tail, dotted := strings.Cut(name, ".") if !dotted { return arm{name: name, key: name} } - segs := strings.Split(tail, ".") + return pathArm(name, head, strings.Split(tail, ".")) +} + +func pathArm(name, key string, segs []string) arm { var steps []string for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own - steps = append(steps, head+"."+strings.Join(segs[:i+1], ".")) + steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) } - return arm{name: name, key: head, tail: segs, steps: steps} + return arm{name: name, key: key, tail: segs, steps: steps} } // checkNoOverlap rejects a format that both renders a level and reads a path into @@ -226,8 +271,8 @@ func splitArm(name string) arm { // level's held draw while rendering the level expands it afresh, so their values // would disagree. Names are compared in sorted order, so which pair is reported // does not depend on where the tokens sit. -func checkNoOverlap(format string, bound map[string]string) error { - names := boundReaders(format, bound) +func checkNoOverlap(format string, bound map[string]string, refs map[string]string) error { + names := boundReaders(format, bound, refs) // Stable over one format-order scan, so two readers of one name (a token and a // calc operand both naming "p") are reported as the format writes them. sort.SliceStable(names, func(i, j int) bool { return names[i].name < names[j].name }) @@ -245,23 +290,24 @@ func checkNoOverlap(format string, bound map[string]string) error { type reader struct{ name, label string } // boundReaders lists every way a format reaches a bound field, in the order the -// format writes them. A calc operand renders its field, so it names a level exactly +// format writes them. An operand renders its field, so it names a level exactly // as a token does; one scan finds both, which is what puts them in one order. -func boundReaders(format string, bound map[string]string) []reader { +func boundReaders(format string, bound map[string]string, refs map[string]string) []reader { var names []reader _ = eachToken(format, func(t ftoken) error { if t.kind != 'b' { return nil } - if _, _, isFunc := funcCall(t.body); isFunc { - for _, operand := range calcTokenOperands(t.body) { - if _, isBound := bound[operand]; isBound { - names = append(names, reader{operand, fmt.Sprintf("calc operand %q", operand)}) + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + a := splitArm(operand, refs) + if _, isBound := bound[a.key]; isBound { + names = append(names, reader{a.name, fmt.Sprintf("%s operand %q", fn, operand)}) } } return nil } - for _, a := range splitArms(t.body) { + for _, a := range splitArms(t.body, refs) { if _, isBound := bound[a.key]; isBound { names = append(names, reader{a.name, "token {" + a.name + "}"}) } @@ -290,11 +336,11 @@ func checkSegments(a arm) error { } // splitArms splits a token body's '|' alternatives. -func splitArms(body string) []arm { +func splitArms(body string, refs map[string]string) []arm { parts := strings.Split(body, "|") arms := make([]arm, len(parts)) for i, p := range parts { - arms[i] = splitArm(p) + arms[i] = splitArm(p, refs) } return arms } @@ -312,30 +358,38 @@ type op struct { lit string // kind 'l' arms []arm // kind 'f': the '|' alternatives, split into key and path once call callFn - // operands names the sibling fields a {calc()} reads, in the order calcVars - // fixed; expand reads them before the call. nil for every other builtin. - operands []string + // operands are the fields the builtin reads, in the order its operands func + // fixed; expand reads them before the call. nil for a builtin that reads none. + operands []arm } // compileOps turns a format string into ops, and returns the size of its literal // text to size the render buffer, plus two sets. bound is the levels the format // addresses by dotted path, each mapped to the first path reading it, which is what // the overlap fences name. held is every name drawn once per expansion — those -// levels, plus the siblings a {calc()} reads, so an operand shown is the operand +// levels, plus the fields an operand reads, so an operand shown is the operand // computed. Both are nil when the format needs neither, so data that uses neither // carries no render-time cost. Call checkTokens first: it is what proves the scan // and every token are valid. -func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { +func compileOps(format string, refs map[string]string) ([]op, int, map[string]string, map[string]bool) { var ops []op var lit strings.Builder var bound map[string]string var held map[string]bool grow := 0 - hold := func(name string) { + hold := func(a arm) { if held == nil { held = map[string]bool{} } - held[name] = true + held[a.key] = true + if len(a.tail) > 0 { + if bound == nil { + bound = map[string]string{} + } + if _, named := bound[a.key]; !named { + bound[a.key] = a.name // the first path reading it, for error messages + } + } } flush := func() { if lit.Len() > 0 { @@ -351,22 +405,18 @@ func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { case 'b': flush() if name, args, ok := funcCall(t.body); ok { - operands := calcTokenOperands(t.body) - for _, operand := range operands { - hold(operand) // a calc renders its operand, so the expansion holds that draw + var operands []arm + for _, operand := range tokenOperands(t.body) { + a := splitArm(operand, refs) + hold(a) // the builtin renders its operand, so the expansion holds that draw + operands = append(operands, a) } ops = append(ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) } else { - arms := splitArms(t.body) + arms := splitArms(t.body, refs) for _, a := range arms { if len(a.tail) > 0 { - if bound == nil { - bound = map[string]string{} - } - if _, named := bound[a.key]; !named { - bound[a.key] = a.name // the first path reading it, for error messages - } - hold(a.key) + hold(a) } } ops = append(ops, op{kind: 'f', arms: arms}) diff --git a/transform_test.go b/transform_test.go index 31d8279..37b8f96 100644 --- a/transform_test.go +++ b/transform_test.go @@ -6,8 +6,8 @@ func TestTransforms(t *testing.T) { f := engine(1) for src, want := range map[string]string{ `{"format":"{uppercase(x)}|{lowercase(x)}|{ascii(x)}","x":"Åsa Ödegård-Nuñez"}`: "ÅSA ÖDEGÅRD-NUÑEZ|åsa ödegård-nuñez|Asa Odegard-Nunez", - `{"format":"{lowercase(ascii(x))}","x":"Åsa"}`: "asa", - `{"format":"{ascii(x)}","x":"Ærø ß 日本"}`: "AEro ss ", + `{"format":"{lowercase(ascii(x))}","x":"Åsa"}`: "asa", + `{"format":"{ascii(x)}","x":"Ærø ß 日本"}`: "AEro ss ", } { if got := mustRender(t, f, src); got != want { t.Errorf("render(%s) = %q, want %q", src, got, want) -- 2.52.0 From 035e51d3e1934b01ab15338747f703e103efc961 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:53:29 +0200 Subject: [PATCH 12/38] Tests that the README examples load, render and print what they claim --- readme_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 readme_test.go diff --git a/readme_test.go b/readme_test.go new file mode 100644 index 0000000..594fabc --- /dev/null +++ b/readme_test.go @@ -0,0 +1,61 @@ +package fejkdata + +import ( + "os" + "regexp" + "strings" + "testing" +) + +var jsonBlock = regexp.MustCompile("(?s)```json\n(.*?)```") + +func readme(t *testing.T) string { + t.Helper() + src, err := os.ReadFile("README.md") + if err != nil { + t.Fatal(err) + } + return string(src) +} + +func TestReadmeExamplesLoadAndRender(t *testing.T) { + n := 0 + for _, m := range jsonBlock.FindAllStringSubmatch(readme(t), -1) { + body := m[1] + if strings.Contains(body, "…") { + continue + } + f, err := New(WithDataPath(writeData(t, map[string]string{"example": body})), WithSeed(1)) + if err != nil { + t.Errorf("README example does not load: %v\n%s", err, body) + continue + } + for i := 0; i < 20; i++ { + if _, err := f.Fake("example"); err != nil { + t.Errorf("README example does not render: %v\n%s", err, body) + break + } + } + n++ + } + if n < 8 { + t.Fatalf("found only %d README examples", n) + } +} + +func TestReadmeSQLExampleOutput(t *testing.T) { + src := readme(t) + src = src[strings.Index(src, "### Your own data"):] + block := jsonBlock.FindStringSubmatch(src) + want := regexp.MustCompile(`(?m)^# (INSERT .*)$`).FindStringSubmatch(src) + if block == nil || want == nil { + t.Fatal("README lost the SQL example or its printed output") + } + f, err := New(WithDataPath(writeData(t, map[string]string{"sql": block[1]})), WithSeed(1)) + if err != nil { + t.Fatal(err) + } + if got := fake(t, f, "sql"); got != want[1] { + t.Errorf("README SQL example with seed 1 = %q, README prints %q", got, want[1]) + } +} -- 2.52.0 From 00a6160aa47332284594e21e15ee58328bed8dcd Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:56:11 +0200 Subject: [PATCH 13/38] Restructure the README around the grammar, record the standing decisions, add the contributor rules --- AGENTS.md | 9 + README.md | 634 +++++++++++++++++++++--------------------------------- 2 files changed, 256 insertions(+), 387 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..87d73c9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,9 @@ +# Rules + +- Go runs only through `docker compose run --rm `; the merge gate is `docker build .`. +- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output is its own commit. +- One-line commit messages: no ticket prefix, no repo name, no authorship trailers. +- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble. +- One spelling per result: reject the other at `New`, and let the error name the spelling to use. +- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. +- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. diff --git a/README.md b/README.md index b337a5c..4804bf9 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,61 @@ # fejkdata +A Go library and CLI for generating locale-aware fake data from JSON templates. Forked from [github.com/Timewave-AB/fakes](https://github.com/Timewave-AB/fakes). -A Go library and CLI for generating fake data, built for -**internationalization**. It exists because the existing Go fake libraries -lacked the locale coverage and format control we needed. +## Goals -- **Standards first** — formats, names and structures follow international and - local standards first and foremost. -- **Locale-aware** — names, addresses, postal codes and phone numbers follow - per-locale data and formats. The shipped data is organised by full locale tag - (`sv_SE`), but the engine treats folders as plain namespaces — name yours - anything. -- **Data lives in JSON** — all source data is recursive JSON on disk, read when - you create a generator and then served from memory. Add or change data without - touching the library. Behavior belongs in data too: the engine grows a - built-in function only for what data can't express (a checksum, a time-based - id), never for what character classes and choices already do. -- **Composable** — templates nest without limit: weighted choices, character - classes and sub-templates combine to model any format. -- **Reproducible** — seed a generator and it emits the same sequence every time, - for a given version of the data: changing how a value is composed shifts the - stream for that value and for everything drawn after it in the same generator. - Every built-in draws only from that seed — no wall-clock, no `crypto/rand` — - so determinism holds end to end. -- **Zero dependencies** — standard library only. +1. **Valid by construction** — every value passes the check its real consumer + applies; facts that belong together come from one draw, within a value and + across categories. +2. **Text means what it says** — a format renders as written; only `{…}` varies, + random characters included (`{digits(3)}`). One spelling per result; the wrong + one is a load error naming the right one. +3. **Every mistake is a load error** — `New` rejects; `Fake` on a loaded generator + fails only for an unknown path. +4. **Zero to a value in one command** — `go install`, then `fejkdata sv_SE.person`: + no checkout, no flag. Flags are GNU-form (`--seed 42`, `-n 3`) in any position; + the first custom template needs no escape and no option. +5. **Data lives in JSON** — a builtin only for what data can't express. +6. **Reproducible** — seed in, same stream out; no builtin reads a clock. +7. **Zero dependencies** — standard library only. +8. **Docs index the grammar** — every syntax feature is a heading; every example + runs under test and shows its output; a rule is stated once. ## CLI -Install the `fejkdata` command and give it a path — it prints one value to stdout. -The shipped data set is built in; each dot segment descends one level: folders, -then the category (a JSON file), then fields inside it. - ```sh go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest fejkdata sv_SE.person # Sara Eriksson -fejkdata sv_SE.person.last # Eriksson (dotted path into a category) -fejkdata --seed 42 sv_SE.address -fejkdata --repeat 3 sv_SE.person # three values, one per line +fejkdata sv_SE.person.last # Eriksson +fejkdata --seed 42 sv_SE.address # the same address every run fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol fejkdata --list # every path the data offers -fejkdata --data-path ./mydata sv_SE.word # layer a dir over the shipped data; last wins a clash +fejkdata --data-path ./mydata sv_SE.word # layer a directory over the shipped data fejkdata --no-shipped-data -d ./mydata --list # only your data ``` -Flags are GNU-style: `--name value` or `--name=value`, short aliases `-d`, `-n`, -`-s`, `-h`, in any position; `--` ends the flags. `--data-path` is repeatable -(last wins a name clash) and `--no-shipped-data` leaves the built-in set out. -`--repeat N` renders the path N times — each an independent draw — joined by -`--separator` (default a newline, so values land one per line). `--list` prints -every path you can ask for; `--version` prints the build version. +A path names a category, or a field inside one: each dot segment descends one +level — folders, then the category (a JSON file), then fields. -Without installing, run it from a checkout with `go run ./cmd/fejkdata …`. Exit -codes: `0` success (including `--list`, `--version`, `--help`), `1` runtime error -(missing dir, unknown path), `2` misuse. +| Flag | | +|------|--| +| `-d`, `--data-path D` | a directory to layer over the shipped data; repeatable, the last wins a name clash | +| `--no-shipped-data` | load only the `--data-path` directories | +| `-s`, `--seed N` | reproducible output | +| `-n`, `--repeat N` | render the path N times, each an independent draw | +| `--separator S` | between repeated values (default a newline) | +| `--list` | print every path, then exit | +| `--version`, `-h`, `--help` | print, then exit | -### Generating a file from a custom template +`--name value` and `--name=value` both work (see [Decisions](#decisions)); flags +go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing +dir, unknown path), `2` misuse. From a checkout: `go run ./cmd/fejkdata …`. -A category is just a JSON file in a data directory, so you can drop in your -own and render it — no code change. Save this as `mydata/sql.json`: +### Your own data + +A category is a JSON file in a directory. Save this as `mydata/sql.json`: ```json { @@ -73,410 +69,283 @@ own and render it — no code change. Save this as `mydata/sql.json`: } ``` -`sql-username` renders `'{username}'` `repeat` times and joins the results with -the `),(` separator; the outer `VALUES(…)` wraps that into one valid row list. - ```sh fejkdata --seed 1 --data-path ./mydata sql # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); -``` - -Raise the template's `repeat` for more rows per statement; use the CLI's -`--repeat` for more statements — together they build a whole seed file: - -```sh fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` ## Library ```sh -go get gitea.larvit.se/larvit/fejkdata # requires Go 1.22+ (for math/rand/v2) +go get gitea.larvit.se/larvit/fejkdata # Go 1.22+ ``` -`New` loads the shipped data set, plus any `WithDataPath` directories layered over -it; generate values by path with `Fake`. Each dot segment descends one level: -folders, then the category (a JSON file), then fields inside it. - ```go -package main - -import ( - "fmt" - "log" - - "gitea.larvit.se/larvit/fejkdata" -) - -func main() { - f, err := fejkdata.New() - if err != nil { - log.Fatal(err) - } - - for _, path := range []string{"sv_SE.person", "sv_SE.address", "sv_SE.phone", "sv_SE.address.locality"} { - v, err := f.Fake(path) - if err != nil { - log.Fatal(err) - } - fmt.Printf("%-24s %s\n", path, v) - } +f, err := fejkdata.New(fejkdata.WithSeed(42)) +if err != nil { + log.Fatal(err) } +v, err := f.Fake("sv_SE.address") // "Kungsvägen 68\n379 17 Stockholm" +paths := f.List() // every path Fake accepts, sorted ``` -``` -sv_SE.person Sara Eriksson -sv_SE.address Kungsvägen 68 - 379 17 Stockholm -sv_SE.phone 072-402 91 67 -sv_SE.address.locality Linköping -``` - -Options: `WithSeed(n)` for a reproducible sequence — same seed + data yields an -identical sequence, handy for stable tests; `WithDataPath(dir)` layers a directory -(repeat it to layer several, last wins a clash); `WithDataFS(fsys)` layers an -`fs.FS`, such as your own `embed.FS`; `WithoutShippedData()` loads only what you -give. - -```go -a, _ := fejkdata.New(fejkdata.WithSeed(42)) -b, _ := fejkdata.New(fejkdata.WithSeed(42)) -av, _ := a.Fake("sv_SE.person") -bv, _ := b.Fake("sv_SE.person") -av == bv // true -``` - -`f.List()` returns the sorted paths the loaded data offers — the categories, their -dotted fields and folder segments (what the CLI's `--list` prints). Every path it -lists renders. +| Option | | +|--------|--| +| `WithSeed(n)` | same seed, same data: identical sequence | +| `WithDataPath(dir)` | layer a directory; repeat to layer several, the last wins a clash | +| `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | +| `WithoutShippedData()` | load only what you give | A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only -when drawn from one goroutine. +when drawn from one goroutine. Changing how a value is composed shifts the seeded +stream for that value and everything drawn after it. ## Data The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) -plus a locale-neutral `misc` folder — is embedded in the library and the CLI, so -both work with no data on disk. Layer your own directories over it; a category or -folder name must not use `.`, `|`, `(`, `{` or `}` (see [Data format](#data-format)), -and dot-prefixed entries are skipped, so a data directory can also be a checkout. +plus a locale-neutral `misc` folder — is embedded, so the CLI and the library +work with no data on disk. A directory is a namespace: each JSON file is a +category named after the file, each subdirectory a dot-path segment, so +`mydata/sv_SE/person.json` is `sv_SE.person` and replaces the shipped one. +Sources merge in order; matching folders combine, any other clash is won by the +last loaded. Names may not use `.`, `|`, `(`, `{` or `}`; dot-prefixed entries +are skipped, so a data directory can also be a checkout. -A directory is just a namespace. Each JSON file is a category named after the -file; each subdirectory is a dot-path segment — folders nest exactly like JSON -objects do. So `mydata/sv_SE/person.json` is `Fake("sv_SE.person")`. - -Sources merge in order: matching folders combine by their children, and any other -clash is won by the last one loaded. That lets you override a shipped category -without copying the rest: - -```go -fejkdata.New(fejkdata.WithDataPath("./mydata")) // mydata/sv_SE/person.json replaces sv_SE.person -``` - -Each shipped locale carries these categories, formatted per locale (e.g. `date` -is `MM/DD/YYYY` in `en_US`, `YYYY-MM-DD` in `sv_SE`; `ssn` is a US SSN vs a -Swedish personnummer): `address`, `color`, `company`, `date`, `email`, `ip`, +Each locale carries `address`, `color`, `company`, `date`, `email`, `ip`, `person`, `phone`, `price`, `sentence`, `ssn`, `time`, `url`, `username`, -`version`, `word`. - -`data/misc` carries locale-neutral categories — universal data that isn't tied to -a language or region: - -- ids & networking: `uuid` (a proper random v4), `mac`, `objectid` (24 hex chars, - MongoDB ObjectID-shaped — the leading bytes are random, not a real timestamp), - `creditcard` (per-network numbers ending in a valid `{luhn()}` digit) -- reference codes: `currency` (ISO 4217), `country` (ISO 3166), `language` (ISO - 639), `timezone` (IANA) -- web & systems: `mimetype`, `httpstatus`, `useragent` -- misc: `coordinate` (a lat/long point), `emoji`, `car` - -Many carry dotted sub-fields — `currency.symbol`, `country.alpha2`, -`mimetype.ext`, `httpstatus.code`, `car.maker`. A time-ordered v7 UUID can't be -expressed as data, so it's the `{uuid()}` builtin instead (see **Functions**). +`version` and `word`, formatted per locale. `misc` carries `car`, `coordinate`, +`country` (ISO 3166), `creditcard` (Luhn-valid), `currency` (ISO 4217), `emoji`, +`httpstatus`, `language` (ISO 639), `mac`, `mimetype`, `objectid`, `timezone` +(IANA), `useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, +`misc.country.alpha2`, `misc.httpstatus.code` — which `--list` shows. ## Data format -Each JSON file in a data directory is a **category** named after the file -(`address.json` → `address`), rendered by `Fake("address")`. Drop in a new file -or folder — no code change, no recompile. +Every value is a **node**, nestable without limit: -Every value is a **node**, one of three shapes, nestable without limit: - -| Node | JSON | Meaning | +| Node | JSON | Renders | |------|------|---------| -| string | `"Malmö"` | a format with no fields: text, or tokens that need none (`"{digits(3)}"`, `"{..path}"`) | -| choice | `["a", "b", …]` | one element, picked at random | -| template | `{"format": "…", …}` | a format string plus the named sub-nodes it references | +| string | `"Malmö"` | its text, with any `{…}` tokens expanded | +| choice | `["a", "b", …]` | one item, picked at random | +| template | `{"format": "…", …}` | its format, with `{name}` tokens rendering the named fields | -**Weight.** A template node may carry a `weight` (default `1`) to skew its odds -within a choice: +### Format string + +Every character is literal except a `{…}` token: + +| Token | Renders | +|-------|---------| +| `{name}` | the sibling field `name` | +| `{name.field}` | `field` of one draw of `name` ([Correlated fields](#correlated-fields)) | +| `{a\|b}` | one of the named fields, even odds | +| `{fn(args)}` | a builtin ([Functions](#functions)) | +| `{..path}` | a node reached from the data root ([References](#references)) | +| `{{`, `}}` | a literal `{` or `}` | + +```json +"100 Main St, Apt {int(1,9)}{upper(1)} — tel {digits(3)}-{digits(4)}" +``` + +Renders e.g. `100 Main St, Apt 4B — tel 555-0199`. Random characters come from +the sample functions, so a phone pattern is `070-{digits(3)} {digits(2)} {digits(2)}`. +A lone `}` is a load error naming `}}`; the arms of `{a|b}` must differ, a +repeated arm being a second spelling of [weight](#weight). + +### Weight + +An item of a choice may carry a `weight` (default `1`) to skew its odds: ```json [ { "format": "070-{digits(3)} {digits(2)} {digits(2)}", "weight": 10 }, - { "format": "01-{digits(3)} {digits(2)} {digits(2)}" }, - { "format": "010-{digits(3)} {digits(2)} {digits(2)}" } + "08-{digits(3)} {digits(2)} {digits(2)}", + { "format": "010-{digits(3)} {digits(2)} {digits(2)}", "weight": 2 } ] ``` -A bare string or nested array in a choice counts as `1`; to weight a string, -write it as `{ "format": "AB", "weight": 3 }`. A repeated item is a load error -naming that spelling, and so is a `weight` of `1`. Weights are checked when you -create the generator: a negative, non-numeric, or all-zero set is rejected at -`New`, so a typo fails fast instead of silently skewing output. +Renders `070-412 38 91` ten times as often as `08-…`. A string item is weighted by +writing it as `{ "format": "AB", "weight": 3 }`. Rejected at load: a weight that +is negative, non-numeric, `1` (the default) or outside a choice, a set summing to +zero, and a repeated item — `["a", "a", "b"]` is `[{ "format": "a", "weight": 2 }, "b"]`, +and the error says so. -**Repeat.** A template node may carry a `repeat` (default `1`) to render its -`format` that many times — each render an independent pick — joined by -`separator` (default `""`): +### Repeat + +A template may carry `repeat` (an integer above `1`) to render its format that +many times — each an independent draw — joined by `separator` (default `""`): ```json { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -This yields e.g. `bar foo baz`. `repeat` must be an integer above `1` and -`separator` a string, both checked at `New`. +Renders e.g. `bar foo baz`. A `separator` without a `repeat` is rejected. -`format`, `weight`, `repeat` and `separator` are the only options; **any other key -is a field**. So write `seperator` and you get a field by that name while the -option stays unset. `New` rejects an option that cannot take effect — a -`separator` without a `repeat`, a `weight` outside a choice, a `weight` or -`repeat` of `1` — an object holding only a `format` (that is a string; write it), -a one-item choice (that is its item; write it), and a name using a character the -grammars reserve: `.` separates the segments of a path, `|` -the arms of a token, `(` opens a function call and `{` `}` delimit the token, so a -name carrying one is a name no format could ever spell. An empty name goes the same -way — it is no path segment at all, so `List` never offers it. That holds for a -category, a folder and a field alike. +### Options and fields -**Functions.** A `{name()}` token calls a built-in function instead of rendering -a field. `{luhn()}` appends a Luhn check digit over the digits emitted **so far** -in the current format (non-digits skipped but kept); unknown functions or wrong -argument counts are rejected at `New`. This is what makes a generated Swedish -personnummer valid — its last digit is a Luhn checksum over the nine before it: +`format`, `weight`, `repeat` and `separator` are the only options; **any other +key is a field** (see [Decisions](#decisions)). An object that does nothing a +string can't — only a `format` — is rejected naming the string, as is a one-item +choice naming its item. -```json -{ "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": [ … ] } -``` +### Functions -renders e.g. `811218-987`, then `{luhn()}` appends `6` → `811218-9876`. Place it -after its payload (it reads what is to its left). The buffer it reads is -per-expansion, so nesting keeps fixed parts out of the sum — e.g. a 12-digit -form prefixes the century outside the checksummed core: +A `{name(args)}` token calls a builtin. Arguments are checked at `New`: a bad +count, range, country or expression fails fast, and a length, count or decimal +place beyond a sane maximum is rejected, so a fat-fingered `hex(2000000000)` +never tries to allocate gigabytes. Every builtin draws only from the seed — a +time-based id takes its timestamp from the rng, not the clock — so seeded output +stays reproducible. + +| Function | Kind | Renders | +|----------|------|---------| +| `{luhn()}` | derivation | Luhn check digit over the digits emitted so far in this expansion | +| `{mod11()}` | derivation | weighted mod-11 check char (weights 2–7 from the right); `X` for 10 | +| `{ean()}` | derivation | EAN-13 / UPC-A / ISBN-13 / GTIN check digit | +| `{digits(n)}` | sample | `n` digits 0–9 | +| `{upper(n)}`, `{lower(n)}` | sample | `n` letters A–Z, a–z | +| `{int(min,max)}` | sample | uniform integer in `[min, max]` | +| `{float(min,max,dp)}` | sample | number in `[min, max]` with `dp` decimals | +| `{hex(n)}` | sample | `n` lowercase hex digits | +| `{base64(n)}` | sample | `n` random bytes, base64 | +| `{uuid()}` | sample | UUID v7 (`misc.uuid` ships v4 as data) | +| `{ulid()}` | sample | ULID, 26 Crockford base32 chars | +| `{nanoid(n)}` | sample | URL-safe Nano ID, `n` chars | +| `{iban(CC)}` | sample | length- and mod-97-valid IBAN for BE, DE, DK, ES, FI, NO or SE | +| `{seq()}`, `{seq(name)}` | counter | next integer from 1 in this generator; `name` selects an independent counter | +| `{calc(expr)}`, `{calc(expr,dp)}` | computation | an arithmetic expression over sibling fields ([Computation](#computation)) | +| `{lowercase(x)}`, `{uppercase(x)}`, `{ascii(x)}` | transform | a field's value rewritten ([Transforms](#transforms)) | + +A derivation reads what is to its left, so place it after its payload; the +buffer is per expansion, so a nested template keeps fixed parts out of the sum. A +Swedish personnummer is a Luhn checksum over the nine digits before it: ```json { "format": "{century}{core}", "century": ["19", "20"], - "core": { "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": [ … ] } } + "core": { "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": ["0115", "0704", "1218"] } } ``` -A function must be deterministic (no wall-clock), so a seeded generator stays -reproducible. A time-based id (UUID v7, ULID) therefore draws its timestamp from -the rng, not the clock — the result is a valid, reproducible value, not a real -point in time. +Renders e.g. `19811218-9876`. `{seq()}` spans `Fake` calls and `repeat`, resets +with a new generator, and is the natural primary key for the SQL example above. -There are five kinds. **Derivations** read the digits emitted so far, so put them -after their payload; **samples** read only the rng, so they stand alone; one -**session counter** (`seq`) advances state held on the generator; one -**computation** (`calc`) evaluates arithmetic over sibling fields; and the -**transforms** rewrite one field's value. Arguments are -validated at `New` (a bad count, range, country, or expression fails fast); a -length, count or decimal place beyond a sane maximum is rejected there too, so a -fat-fingered `hex(2000000000)` can't try to allocate gigabytes at render. +### Computation -| Function | Kind | Emits | -|----------|------|-------| -| `{luhn()}` | derivation | Luhn check digit (mod-10) over preceding digits | -| `{mod11()}` | derivation | weighted mod-11 check char (weights 2–7 from the right); `X` when it would be 10 | -| `{ean()}` | derivation | EAN-13 / UPC-A / ISBN-13 / GTIN check digit | -| `{uuid()}` | sample | UUID v7 (v4 ships as data — see [Data](#data)) | -| `{ulid()}` | sample | ULID, 26-char Crockford base32 | -| `{nanoid(n)}` | sample | URL-safe Nano ID, `n` chars | -| `{hex(n)}` | sample | `n` lowercase hex digits | -| `{digits(n)}` | sample | `n` digits 0–9 | -| `{upper(n)}` | sample | `n` letters A–Z | -| `{lower(n)}` | sample | `n` letters a–z | -| `{base64(n)}` | sample | `n` random bytes, base64 | -| `{int(min,max)}` | sample | uniform integer in `[min, max]` | -| `{float(min,max,dp)}` | sample | number in `[min, max]` with `dp` decimals | -| `{iban(CC)}` | sample | a length- and mod-97-valid IBAN for country `CC` (BE, DE, DK, ES, FI, NO, SE) | -| `{seq()}`, `{seq(name)}` | session counter | next integer (from 1) in this generator's sequence; `name` selects an independent counter | -| `{calc(expr)}`, `{calc(expr,dp)}` | computation | value of an arithmetic expression over number literals and sibling fields; `dp` rounds | -| `{lowercase(x)}`, `{uppercase(x)}`, `{ascii(x)}` | transform | the field `x` — a name, a path or a `..path` — lower-cased, upper-cased, or folded to ASCII (`Åsa` → `Asa`); they nest: `{lowercase(ascii(x))}` | - -`{ean()}` is also the ISBN-13 check (an ISBN-13 *is* an EAN-13 — build the 978/979 -prefix in data and call `{ean()}`). `{iban()}` is a sample, not a derivation: -an IBAN's check digits sit *before* the account number, which a left-to-right -reader can't reach, so it emits the whole value (a generic numeric BBAN — valid -length and checksum, not real bank routing). - -`{seq()}`'s counter lives on the generator, so it spans `Fake` calls (and `repeat`) -and resets when you build a new generator — `seq` is reproducible by being ordered, -not random. It's the natural fit for a primary-key column in the SQL example above. - -**Computation.** `{calc(expr)}` evaluates an arithmetic expression — `+ - * /`, -parentheses and unary minus, the usual precedence — and emits the result. Operands -are number literals and **sibling field names**, each rendered then read as a -number; an optional second arg rounds to that many decimals (`{calc(net * qty, 2)}`), -otherwise the value prints in minimal form. A hyphen is always subtraction, so a -hyphenated field name can't be an operand. The expression is checked at `New` -(parse, and that every name is a real field): +`{calc(expr)}` evaluates `+ - * /`, parentheses and unary minus over number +literals and sibling field names, each rendered then read as a number; a second +argument rounds to that many decimals. A hyphen is always subtraction, so a +hyphenated field can't be an operand. ```json -{ "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99"], "qty": ["3"] } +{ "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } ``` -renders `19.99 x 3 = 59.97`. A field a `calc` reads is drawn **once per -expansion** and held, so the operand shown is the operand computed — give `net` -three prices and the line still multiplies the one it printed. The hold covers -every reading of that name in the format, so `{w} {w} {calc(w)}` is one value -three times; a name no `calc` reads is unaffected, and `{word} {word}` still draws -twice. A field that doesn't render to a number yields `NaN`, and a division by -zero yields `Inf`; both print rather than failing the render. +Renders e.g. `19.99 x 3 = 59.97`: an operand is drawn once per expansion, so the +operand shown is the operand computed. A non-numeric operand yields `NaN` and a +division by zero `Inf`; both print rather than fail. -**References.** A `{..path}` token renders a node from the **data root** instead -of a sibling field — the dot path is the one `Fake` takes, resolved across every -loaded directory. One category can borrow another, even across folders or layered -data dirs: +### Transforms + +`{lowercase(x)}`, `{uppercase(x)}` and `{ascii(x)}` rewrite the value of `x` — a +field, a path or a `..path` — and nest. `ascii` folds Latin letters (`Åsa Öberg` +→ `Asa Oberg`) and drops any other non-ASCII rune. Like a calc operand, `x` is +drawn once per expansion, so an email built from a name matches the name beside it: ```json -{ "format": "Hej, {..en_US.person}!" } +{ "format": "{p.first} {p.last} <{lowercase(ascii(p.first))}.{lowercase(ascii(p.last))}@example.com>", + "p": [ + { "format": "{first} {last}", "first": "Åsa", "last": "Öberg" }, + { "format": "{first} {last}", "first": "Bo", "last": "Ek" } + ] } ``` -renders e.g. `Hej, Pat Smith!`. A reference path is held like a sibling path: -`{..person.first} {..person.last}` read one person, and `{lowercase(..person.first)}` -reads that same draw, while a bare `{..die} {..die}` is two draws. References are -bound when you create the generator, so a path that is unknown, names a folder, -or reads a field not every variant of a choice carries fails at `New`. A reference that leads back to its own value (directly, mutually, -or through a chain) is a cycle that would never finish rendering, so it too is -rejected at `New`. +Renders `Åsa Öberg ` or `Bo Ek `, never +a mix. -**Correlated fields.** A `{name.field}` token addresses a **path** into a sibling, -and a sibling addressed that way is drawn **once per expansion** — so several -tokens read one row. That is how two facts that belong together, such as a -locality and the postal code that really covers it, stay together: +### References + +`{..path}` renders a node from the **data root** — the path `Fake` takes, across +every loaded source — so one category borrows another, even across folders or +layered directories: ```json -{ "format": "{street} {number}\n{place.postal-code} {place.locality}", +"Hej, {..en_US.person}!" +``` + +Renders e.g. `Hej, Pat Smith!`. A reference into a category is held like a +[correlated](#correlated-fields) path — `{..sv_SE.person.first} {..sv_SE.person.last}` +name one person, `{lowercase(..sv_SE.person.first)}` reads that same draw — while +a bare `{..misc.uuid} {..misc.uuid}` is two draws. Rejected at `New`: a path that +is unknown, names a folder, or reads a field not every variant of a choice +carries, and a reference that leads back to its own value, directly, mutually or +through a chain. + +### Correlated fields + +`{name.field}` reads a path into a sibling, and a sibling read that way is drawn +**once per expansion**, so several tokens read one row — a locality and the +postal code that really covers it: + +```json +{ "format": "{street} {int(1,99)}\n{place.postal-code} {place.locality}", + "street": ["Kungsgatan", "Storgatan"], "place": [ { "format": "{locality}", "locality": "Stockholm", "postal-code": "1{digits(2)} {digits(2)}", "weight": 975 }, { "format": "{locality}", "locality": "Tranås", "postal-code": "573 {digits(2)}", "weight": 18 } ] } ``` -renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm postal code -beside Tranås. Each row carries its own `weight`, so how often a place appears is data -too. The rule holds both ways: two tails of one head come from the same row, and -one path read twice reads one value (`{p.first} … {p.first}@…` gives one name). +Renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm code beside +Tranås; each row's `weight` says how often it appears. A path is held at every +level it passes through (`{p.geo.town.name} {p.geo.town.zip}` share the town), +one path read twice reads one value, and a field no path addresses is drawn each +time (`{word} {word}` differs). The hold lasts one expansion: each `repeat` +iteration and each nested template draws again. Every variant of a choice on the +path must carry the rest of it, so a row missing a field is named at load: -**One draw, one spelling.** The hold above is what a dotted token reads, and a -`{calc()}` operand reads its sibling the same way (see **Computation**). A format -may not both *render* a level and *read a path into* it — `{p}` beside -`{p.first}`, or `{place}` beside `{place.locality}`. -Rendering a level expands it afresh while a path reads the level's held draw, so -the two would disagree; naming the fields you want is the one spelling that -always agrees, and the other is a load error. This covers every way a level can -be rendered: a token, a `{calc()}` operand, and a `{..path}` reference — wherever -the reference sits, including in a field the format renders. +```text +token {place.postal-code}: field "place": not every variant of this 2-way choice carries "postal-code"; all carry [locality] +``` -A `{calc()}` operand is held on its own terms too, so the same fence guards it: -`{..cat.net} x 2 = {calc(net * 2, 2)}` names one field two ways and is a load -error, with no path token anywhere. A calc renders its operand whole, so the hold -pins what that render settled, following the operand's plain `{field}` tokens — -`{net.v}` and `{..cat.net.v}` are rejected alike. It stops at a `{..path}`, where -the operand's own value ends and a shared source begins: two names drawing from -one referenced category are two draws, as `{word} {word}` is, so two dice over -one `{..die}` are fine. +The sub-fields stay addressable — `Fake("address.place.locality")` renders, and +`List` advertises it. A path may not read into a level carrying a `repeat`. + +### One draw, one spelling + +A format may not both **render** a level and **read a path into** it — `{p}` +beside `{p.first}`, `{..cat.net}` beside `{calc(net * 2)}`, or `{q}` beside +`{p.first}` where `q` renders `{..cat.p.last}` — because the render draws afresh +while the path reads the held draw, and the two would disagree. Wherever the +second route sits, it is a load error naming the spelling to use: ```text token {p} renders a level that {p.first} reads a path into; name the fields you want instead ``` -For the same reason a path may not read into a level carrying a `repeat`: it -reads one draw, so the repeat could never apply. - -A path is held at **every level it passes through**, not just the first, so the -facts can nest as deeply as they belong: - -```json -{ "format": "{p.geo.town.name} {p.geo.town.zip}, {p.geo.region}", "p": [ … ] } -``` - -renders `Kiruna 98100, Norrbotten` — the town, the zip that covers it and the -region it sits in all come from one draw. Two paths part company exactly where -they diverge: `{p.a.v}` and `{p.b.v}` share the row and nothing below it. - -The binding lasts for one expansion, so each `repeat` iteration draws again and a -nested template keeps its own. A field no dotted token addresses is unaffected: -`{word} {word}` still draws twice. - -`New` checks a path the way `Fake` resolves one: every variant of a -choice must carry the whole path, so a row missing a field is named at load: - -```text -token {place.postal-code}: field "place": not every variant of this 2-way choice -carries "postal-code"; all carry [locality] -``` - -The sub-fields stay addressable on their own — `Fake("address.place.locality")` -renders, and `List` advertises it. - -**Format string.** Every character is literal except a `{…}` token: - -| Token | Expands to | -|-------|-----------| -| `{name}` | render the sibling field `name` | -| `{name.field}` | render `field` of one draw of the sibling `name` (see **Correlated fields**) | -| `{name()}` | call a built-in function (see **Functions**) | -| `{..path}` | render the node at a dot path from the data root (see **References**) | -| `{{`, `}}` | a literal `{` or `}` | - -`{a|b}` renders one of the sibling fields `a` or `b`, chosen at random; an arm -may be a `{..path}` reference too (`{name|..en_US.person}`). The arms are picked -evenly and must differ — `{a|a|b}` would skew the odds, which is what `weight` is -for, so a repeated arm is a load error. - -Text means what it says: `100 Main St` renders `100 Main St`. Random characters -come from the sample functions — `{digits(3)}`, `{upper(1)}`, `{lower(2)}`, -`{int(10,99)}` — so a phone pattern is `070-{digits(3)} {digits(2)} {digits(2)}`. -A lone `}` is a load error naming `}}`. - -**Putting it together** (`person.json`): - -```json -{ - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Anna", "Astrid", "Elin"], - "malefirst": ["Anders", "Erik", "Gustav"], - "last": [ - { "format": "{first}sson", "first": ["Ander", "Erik", "Karl"] }, - ["Berg", "von Flemming"] - ], - "prefix": [ - "", - { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 } - ] -} -``` - -This yields e.g. `Anna Eriksson`, `Erik Berg`, or rarely `dr Astrid von Flemming`. -Any field is reachable by dotted path — `Fake("person.last")` renders just a -surname; choices along the path are resolved at random. A path may continue -*through* a choice only where every variant carries the rest of it — every -`currency` variant carries `symbol`, so `currency.symbol` resolves — which keeps a -path from rendering on one call and failing on the next. - ### Performance -Each file is parsed, validated and weight-indexed once, in `New`. After that a -`Fake` call costs about what its output costs — it scans the chosen format and -renders nested tokens, independent of how large your lists are: +Each file is parsed, validated and weight-indexed once, in `New`. A `Fake` call +then costs about what its output costs: an unweighted pick is O(1) whatever the +list's length, a weighted one O(log n), and long formats, deep nesting and many +tokens add cost in proportion to the output. -- Picking from a list is **O(1)** whatever its length — a 10-name list and a - 100 000-name list cost the same. -- Giving entries a `weight` makes that list's pick **O(log n)** instead (a - search over cumulative weights). Still tiny, but an unweighted list is the - cheapest — only add `weight` where you actually want skew. -- Long `format` strings, deep nesting and many `{tokens}` add cost in - proportion to the output produced. +## Decisions + +- **Options and fields share one namespace.** `format`, `weight`, `repeat` and + `separator` are reserved; every other key is a field. Nesting fields under a + key, or prefixing options, would tax every template to guard against a + misspelt option. +- **`{a|b}` stays beside nested choices.** `[[…], […]]` picks the same way, but + its arms are anonymous; `{femalefirst|malefirst}` keeps `person.femalefirst` + addressable. +- **`--name value` and `--name=value` both work.** GNU getopt_long convention, + which every shell user expects. A single-dash long flag is rejected naming the + double-dash spelling. +- **The shipped data is embedded, not discovered.** A directory a machine happens + to have would make `--seed 42` machine-dependent. Data still lives in `data/` + as JSON; `--data-path` layers over it. +- **Samples say what they emit, transforms what they do.** `{upper(2)}` is two + letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on + whether the argument looks like a number. ## Development @@ -484,7 +353,7 @@ Everything runs in Docker — **no local tooling beyond Docker is needed**. Source is bind-mounted; build caches persist in the `gocache` volume. ```sh -docker compose run --rm test # run tests +docker compose run --rm test # go test -race docker compose run --rm cover # tests with coverage docker compose run --rm bench # benchmarks docker compose run --rm build # compile the library @@ -506,34 +375,25 @@ gate — vet, format check and tests — so run it locally before pushing: ```sh docker build . # latest docker build --build-arg GO_VERSION=1.22.12 . # lowest supported -``` - -Tests run against the latest Go by default. Set `GO_VERSION` to check the lowest -supported version too: - -```sh -GO_VERSION=1.22.12 docker compose run --rm test # lowest supported -docker compose run --rm test # latest +GO_VERSION=1.22.12 docker compose run --rm test # the same tests, without the image build ``` ## Layout ``` -fejkdata.go Generator, New, options, the embedded data set, List -node.go the node model and JSON -> node compilation -render.go Fake and the recursive renderer (choices, format strings, paths, bound draws) -template.go the {token} grammar: scanning, function and path tokens, validation -reference.go {..path} binding across the tree, the render graph, and the walks over it -builtins.go the {name()} function registry and its implementations -calc.go the {calc()} arithmetic evaluator: parser, eval, validation -data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge -cmd/fejkdata/ the `fejkdata` CLI (New + Fake/List over stdout) -data/ shipped data (JSON), embedded at build: locale folders + a misc folder +fejkdata.go Generator, New, options, the embedded data set, List +node.go the node model and JSON -> node compilation +render.go Fake and the recursive renderer (choices, format strings, paths, held draws) +template.go the {token} grammar: scanning, arms, operands, validation +reference.go {..path} binding across the tree, the render graph, and the walks over it +builtins.go the {name()} function registry and its implementations +calc.go the {calc()} arithmetic evaluator: parser, eval, validation +data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge +cmd/fejkdata/ the fejkdata CLI +data/ shipped data (JSON), embedded at build: locale folders + a misc folder +format-migration/ converters from the pre-release grammar; delete before the first tag ``` -To add a category, drop a JSON file into a data directory; to add a locale, add -a subdirectory of JSON files. - ## License MIT — see [LICENSE](LICENSE). -- 2.52.0 From 8cd603a2b1ada7f72cf97e08c3ab87c80cdb664e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 12:59:12 +0200 Subject: [PATCH 14/38] Key the repeated-item check on the string itself; marshal only objects --- node.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/node.go b/node.go index 7859521..793c4f7 100644 --- a/node.go +++ b/node.go @@ -153,11 +153,15 @@ func compileChoice(items []any) (node, error) { func checkNoRepeatedItem(items []any) error { seen := make(map[string]int, len(items)) for i, raw := range items { - key, err := json.Marshal(raw) - if err != nil { - return err + key, isString := raw.(string) + if !isString { + b, err := json.Marshal(raw) + if err != nil { + return err + } + key = "\x00" + string(b) } - if j, dup := seen[string(key)]; dup { + if j, dup := seen[key]; dup { if s, isString := raw.(string); isString { return fmt.Errorf("choice item %q is repeated; skew the odds with a weight instead: { \"format\": %q, \"weight\": 2 }", s, s) } -- 2.52.0 From f6782cf477a762c96af6262309a923d8b4c76420 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 14:08:58 +0200 Subject: [PATCH 15/38] Tests for attached and bundled short flags, and = after a short flag --- cmd/fejkdata/main_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 56cb624..86fd3e3 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -64,6 +64,49 @@ func TestRunSeedSpellings(t *testing.T) { } } +func TestRunShortFlagValues(t *testing.T) { + _, want, _ := runOut("--seed", "42", "--data-path", svSE, "address") + for _, args := range [][]string{ + {"-s42", "-d", svSE, "address"}, + {"-s", "42", "-d" + svSE, "address"}, + {"-d", svSE, "address", "-s42"}, + } { + code, got, errb := runOut(args...) + if code != 0 { + t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb) + } + if got != want { + t.Errorf("run(%v) = %q, want %q", args, got, want) + } + } + _, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word") + if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 { + t.Errorf("-n3 gave %d lines: %q", len(lines), three) + } + for _, c := range []struct { + args []string + want string + }{ + {[]string{"-s=42", "-d", svSE, "address"}, "-s42"}, + {[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"}, + {[]string{"-d=" + svSE, "address"}, "--data-path="}, + {[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs a positive integer, got "d"`}, + {[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`}, + } { + code, out, errb := runOut(c.args...) + if code != 2 || out != "" { + t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } + } + code, out, _ := runOut("-hd", svSE) + if code != 0 || !strings.Contains(out, "Usage") { + t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out) + } +} + func TestRunDoubleDashEndsFlags(t *testing.T) { code, out, errb := runOut("--data-path", svSE, "--", "person") if code != 0 || strings.TrimSpace(out) == "" { -- 2.52.0 From 0ad88028c47c5974d91c89fde8df9baf7b2a7c01 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 14:09:48 +0200 Subject: [PATCH 16/38] One path reads a flag's value: short flags attach and bundle, = after one is rejected naming the spelling --- README.md | 14 ++-- cmd/fejkdata/main.go | 181 +++++++++++++++++++++++++++---------------- 2 files changed, 125 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 4804bf9..a07b7b5 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,9 @@ level — folders, then the category (a JSON file), then fields. | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | -`--name value` and `--name=value` both work (see [Decisions](#decisions)); flags -go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing +`--name value` and `--name=value` both work, a short flag's value attaches or +follows (`-n3`, `-n 3`) and short flags bundle (`-hn 3`) — see +[Decisions](#decisions); flags go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing dir, unknown path), `2` misuse. From a checkout: `go run ./cmd/fejkdata …`. ### Your own data @@ -337,9 +338,12 @@ tokens add cost in proportion to the output. - **`{a|b}` stays beside nested choices.** `[[…], […]]` picks the same way, but its arms are anonymous; `{femalefirst|malefirst}` keeps `person.femalefirst` addressable. -- **`--name value` and `--name=value` both work.** GNU getopt_long convention, - which every shell user expects. A single-dash long flag is rejected naming the - double-dash spelling. +- **Flags follow getopt_long.** `--name value` and `--name=value` both work; a + short flag's value attaches or follows (`-s42`, `-s 42`) and short flags bundle + (`-hn 3`), as every shell user expects. A single-dash long flag is rejected + naming the double-dash spelling, and `-s=42` is rejected naming both short + spellings: `=` belongs to the long form, and reading `=42` as the value would + make `-d=./x` a directory named `=./x`. - **The shipped data is embedded, not discovered.** A directory a machine happens to have would make `--seed 42` machine-dependent. Data still lives in `data/` as JSON; `--data-path` layers over it. diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 9dd65ac..9f35f96 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -21,7 +21,7 @@ import ( const usage = `Usage: fejkdata [flags] - a category, or a dotted path into one (person, person.last) + a category, or a dotted path into one (person, person.last) -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) -h, --help print this help, then exit @@ -32,7 +32,8 @@ const usage = `Usage: fejkdata [flags] --separator S string between repeated values (default newline) --version print the version, then exit -Flags may come before or after ; -- ends the flags. +Flags may come before or after ; -- ends the flags. A short flag's value +attaches or follows (-n3, -n 3); short flags bundle (-hn 3). ` type invocation struct { @@ -98,62 +99,131 @@ func flagByShort(name string) *flagDef { return nil } +// flagArg is one flag as written: its definition and the value attached to it, +// if any. +type flagArg struct { + def *flagDef + value string + inline bool +} + +// splitFlags reads one argv element as flags: --name or --name=value, or -abc +// where each letter is a short flag and the first that takes a value takes the +// rest of the element. +func splitFlags(arg string) ([]flagArg, error) { + if strings.HasPrefix(arg, "--") { + name, value, inline := strings.Cut(arg[2:], "=") + def := flagByLong(name) + if def == nil { + return nil, fmt.Errorf("unknown flag --%s", name) + } + return []flagArg{{def, value, inline}}, nil + } + if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil { + return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name) + } + var flags []flagArg + letters := arg[1:] + for i := 0; i < len(letters); i++ { + letter := letters[i : i+1] + def := flagByShort(letter) + if def == nil { + return nil, fmt.Errorf("unknown flag -%s", letter) + } + if !def.value { + flags = append(flags, flagArg{def: def}) + continue + } + rest := letters[i+1:] + if strings.HasPrefix(rest, "=") { + return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s", + letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:]) + } + return append(flags, flagArg{def, rest, rest != ""}), nil + } + return flags, nil +} + func parseArgs(argv []string) (invocation, error) { in := invocation{repeat: 1, separator: "\n"} for i := 0; i < len(argv); i++ { arg := argv[i] - switch { - case arg == "--": + if arg == "--" { in.paths = append(in.paths, argv[i+1:]...) return in, nil - case strings.HasPrefix(arg, "--"): - name, value, hasValue := strings.Cut(arg[2:], "=") - def := flagByLong(name) - if def == nil { - return in, fmt.Errorf("unknown flag --%s", name) - } - if !def.value { - if hasValue { - return in, fmt.Errorf("--%s takes no value", name) + } + if len(arg) < 2 || arg[0] != '-' { + in.paths = append(in.paths, arg) + continue + } + flags, err := splitFlags(arg) + if err != nil { + return in, err + } + for _, fl := range flags { + if !fl.def.value { + if fl.inline { + return in, fmt.Errorf("--%s takes no value", fl.def.long) } - _ = def.set(&in, "") + _ = fl.def.set(&in, "") continue } - if !hasValue { + value := fl.value + if !fl.inline { if i++; i >= len(argv) { - return in, fmt.Errorf("--%s needs a value", name) + return in, fmt.Errorf("--%s needs a value", fl.def.long) } value = argv[i] } - if err := def.set(&in, value); err != nil { + if err := fl.def.set(&in, value); err != nil { return in, err } - case len(arg) > 1 && arg[0] == '-': - name, _, _ := strings.Cut(arg[1:], "=") - def := flagByShort(name) - if def == nil { - if flagByLong(name) != nil { - return in, fmt.Errorf("unknown flag %s; use --%s", arg, name) - } - return in, fmt.Errorf("unknown flag %s", arg) - } - if !def.value { - _ = def.set(&in, "") - continue - } - if i++; i >= len(argv) { - return in, fmt.Errorf("--%s needs a value", def.long) - } - if err := def.set(&in, argv[i]); err != nil { - return in, err - } - default: - in.paths = append(in.paths, arg) } } return in, nil } +// check rejects a flag combination that cannot run. +func (in invocation) check() error { + if in.noShipped && len(in.dirs) == 0 { + return errors.New("--no-shipped-data needs at least one --data-path") + } + if in.list && len(in.paths) > 0 { + return errors.New("--list takes no path") + } + if !in.list && len(in.paths) != 1 { + return fmt.Errorf("expected one path, got %d", len(in.paths)) + } + return nil +} + +func (in invocation) options() []fejkdata.Option { + var opts []fejkdata.Option + if in.noShipped { + opts = append(opts, fejkdata.WithoutShippedData()) + } + for _, dir := range in.dirs { + opts = append(opts, fejkdata.WithDataPath(dir)) + } + if in.seeded { + opts = append(opts, fejkdata.WithSeed(in.seed)) + } + return opts +} + +// values renders the path repeat times, joined by the separator. +func (in invocation) values(f *fejkdata.Generator) (string, error) { + vals := make([]string, in.repeat) + for i := range vals { + v, err := f.Fake(in.paths[0]) + if err != nil { + return "", err + } + vals[i] = v + } + return strings.Join(vals, in.separator), nil +} + func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } // run returns the exit code: 0 ok, 1 runtime error, 2 misuse. @@ -170,27 +240,10 @@ func run(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stdout, "fejkdata "+buildVersion()) return 0 } - if in.noShipped && len(in.dirs) == 0 { - return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path")) + if err := in.check(); err != nil { + return misuse(stderr, err) } - if in.list && len(in.paths) > 0 { - return misuse(stderr, errors.New("--list takes no path")) - } - if !in.list && len(in.paths) != 1 { - return misuse(stderr, fmt.Errorf("expected one path, got %d", len(in.paths))) - } - - var opts []fejkdata.Option - if in.noShipped { - opts = append(opts, fejkdata.WithoutShippedData()) - } - for _, dir := range in.dirs { - opts = append(opts, fejkdata.WithDataPath(dir)) - } - if in.seeded { - opts = append(opts, fejkdata.WithSeed(in.seed)) - } - f, err := fejkdata.New(opts...) + f, err := fejkdata.New(in.options()...) if err != nil { fmt.Fprintln(stderr, err) return 1 @@ -201,14 +254,12 @@ func run(args []string, stdout, stderr io.Writer) int { } return 0 } - vals := make([]string, in.repeat) - for i := range vals { - if vals[i], err = f.Fake(in.paths[0]); err != nil { - fmt.Fprintln(stderr, err) - return 1 - } + out, err := in.values(f) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 } - fmt.Fprintln(stdout, strings.Join(vals, in.separator)) + fmt.Fprintln(stdout, out) return 0 } -- 2.52.0 From 88e71e666dd1eae0fa03aa2eeae2f5708915ea80 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:18:07 +0200 Subject: [PATCH 17/38] Delete the grammar converters; their output is committed and re-running them corrupts it --- README.md | 1 - format-migration/convert.py | 72 ------------------------------ format-migration/reshape.py | 87 ------------------------------------- 3 files changed, 160 deletions(-) delete mode 100755 format-migration/convert.py delete mode 100755 format-migration/reshape.py diff --git a/README.md b/README.md index a07b7b5..aed8bf4 100644 --- a/README.md +++ b/README.md @@ -395,7 +395,6 @@ calc.go the {calc()} arithmetic evaluator: parser, eval, validation data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge cmd/fejkdata/ the fejkdata CLI data/ shipped data (JSON), embedded at build: locale folders + a misc folder -format-migration/ converters from the pre-release grammar; delete before the first tag ``` ## License diff --git a/format-migration/convert.py b/format-migration/convert.py deleted file mode 100755 index 073d3ef..0000000 --- a/format-migration/convert.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite data files from the class-char format grammar to the literal-text one. - - convert.py data/sv_SE/person.json ... - -Old: 0 1 A a are character classes, # escapes. New: text is literal; a run of -0/A/a becomes {digits(n)}/{upper(n)}/{lower(n)}, 1 followed by n zeros becomes -{int(10^n, 10^(n+1)-1)}, a literal brace becomes {{ or }}. -""" -import re -import sys - -RUN = {"0": "{{digits({})}}", "A": "{{upper({})}}", "a": "{{lower({})}}"} - - -def literal(c): - return {"{": "{{", "}": "}}"}.get(c, c) - - -def convert_format(f): - out, i, n = [], 0, len(f) - while i < n: - c = f[i] - if c == "#": - i += 1 - if i < n: - out.append(literal(f[i])) - i += 1 - else: - out.append("#") - elif c == "{": - j = f.find("}", i) - if j < 0: - out.append(f[i:]) - break - out.append(f[i:j + 1]) - i = j + 1 - elif c in RUN: - k = 0 - while i < n and f[i] == c: - k += 1 - i += 1 - out.append(RUN[c].format(k)) - elif c == "1": - i += 1 - k = 0 - while i < n and f[i] == "0": - k += 1 - i += 1 - out.append("{{int({},{})}}".format(10 ** k, 10 ** (k + 1) - 1)) - else: - out.append(literal(c)) - i += 1 - return "".join(out) - - -FORMAT_VALUE = re.compile(r'("format":\s*")((?:[^"\\]|\\.)*)(")') - - -def convert_text(text): - return FORMAT_VALUE.sub(lambda m: m.group(1) + convert_format(m.group(2)) + m.group(3), text) - - -if __name__ == "__main__": - for path in sys.argv[1:]: - with open(path) as fh: - before = fh.read() - after = convert_text(before) - if after != before: - with open(path, "w") as fh: - fh.write(after) - print("converted", path) diff --git a/format-migration/reshape.py b/format-migration/reshape.py deleted file mode 100755 index 81e5d8d..0000000 --- a/format-migration/reshape.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite data files to the one spelling each shape has. - - reshape.py data/sv_SE/person.json ... - -A one-item choice becomes its item; an object holding only a format becomes that -string; weight 1 and repeat 1 are dropped; a repeated choice item becomes one -item with a weight. -""" -import json -import sys - -OPTIONS = ("format", "weight", "repeat", "separator") -WIDTH = 110 - - -def reshape(n): - if isinstance(n, list): - items = [reshape(x) for x in n] - if len(items) == 1: - return items[0] - out, at = [], {} - for x in items: - key = json.dumps(x, sort_keys=True) - if key in at: - i = at[key] - if isinstance(out[i], str): - out[i] = {"format": out[i], "weight": 2} - elif isinstance(out[i], dict): - out[i]["weight"] = out[i].get("weight", 1) + 1 - else: - out.append(x) - continue - at[key] = len(out) - out.append(x) - return out - if isinstance(n, dict): - d = {k: (v if k in OPTIONS else reshape(v)) for k, v in n.items()} - if d.get("weight") == 1: - del d["weight"] - if d.get("repeat") == 1: - del d["repeat"] - if list(d) == ["format"]: - return d["format"] - return d - return n - - -def scalar(v): - return json.dumps(v, ensure_ascii=False) - - -def one_line(n): - if isinstance(n, dict): - return "{ " + ", ".join(scalar(k) + ": " + one_line(v) for k, v in n.items()) + " }" - if isinstance(n, list): - return "[" + ", ".join(one_line(x) for x in n) + "]" - return scalar(n) - - -def dump(n, indent=0): - pad = " " * indent - if isinstance(n, dict): - line = one_line(n) - if len(pad) + len(line) <= WIDTH and not any(isinstance(v, dict) for v in n.values()): - return line - body = ",\n".join(pad + " " + scalar(k) + ": " + dump(v, indent + 1) for k, v in n.items()) - return "{\n" + body + "\n" + pad + "}" - if isinstance(n, list): - if all(isinstance(x, str) for x in n): - return one_line(n) - line = one_line(n) - if len(pad) + len(line) <= WIDTH: - return line - body = ",\n".join(pad + " " + dump(x, indent + 1) for x in n) - return "[\n" + body + "\n" + pad + "]" - return scalar(n) - - -if __name__ == "__main__": - for path in sys.argv[1:]: - with open(path) as fh: - before = json.load(fh) - after = dump(reshape(before)) + "\n" - with open(path, "w") as fh: - fh.write(after) - print("reshaped", path) -- 2.52.0 From 5514b42b7b74c96f0ada05f29da0e593b762a7a6 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:19:58 +0200 Subject: [PATCH 18/38] Tests for a repeated bare token of a held name --- bound_test.go | 24 ++++++++++++++++++++++++ calc_test.go | 8 ++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/bound_test.go b/bound_test.go index a266a34..89f6d2a 100644 --- a/bound_test.go +++ b/bound_test.go @@ -688,3 +688,27 @@ func TestBoundPathIsReachableByFake(t *testing.T) { } } } + +func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`: "write {w} once", + `{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`: "write {w} once", + `{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`: "write {n} once", + `{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once", + } { + _, err := compile(parse(t, src)) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") { + t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want) + } + } + for _, ok := range []string{ + `{"format":"{w} {w}","w":["a","b"]}`, + `{"format":"{w} {uppercase(w)}","w":["a","b"]}`, + `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`, + `{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`, + } { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v, want it accepted", ok, err) + } + } +} diff --git a/calc_test.go b/calc_test.go index ab38c72..f37f783 100644 --- a/calc_test.go +++ b/calc_test.go @@ -140,16 +140,16 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) { } // TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the -// expansion, not to the calc, so a bare token rendering the same name reads it too. +// expansion, not to the calc, so the bare token rendering the same name reads it too. func TestCalcOperandSharesOneDraw(t *testing.T) { dir := writeData(t, map[string]string{ - "same": `{"format":"{w} {w} {calc(w)}","w":["1","2","3","4","5"]}`, + "same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`, }) f := newGenerator(t, dir, WithSeed(5)) for i := 0; i < 200; i++ { got := fake(t, f, "same") - if p := strings.Fields(got); len(p) != 3 || p[0] != p[1] || p[0] != p[2] { - t.Fatalf("same = %q, want one value three times", got) + if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] { + t.Fatalf("same = %q, want one value twice", got) } } } -- 2.52.0 From c5c06375d87cbedef5492f4c4c9df7564e24202f Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:21:03 +0200 Subject: [PATCH 19/38] Reject a repeated bare token of a held name, naming the single-token spelling --- README.md | 40 +++++++++------- node.go | 19 +++++--- reference.go | 3 +- template.go | 129 ++++++++++++++++++++++++++++++++++----------------- 4 files changed, 123 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index aed8bf4..ecb68b2 100644 --- a/README.md +++ b/README.md @@ -240,16 +240,16 @@ hyphenated field can't be an operand. { "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } ``` -Renders e.g. `19.99 x 3 = 59.97`: an operand is drawn once per expansion, so the -operand shown is the operand computed. A non-numeric operand yields `NaN` and a +Renders e.g. `19.99 x 3 = 59.97`. A non-numeric operand yields `NaN` and a division by zero `Inf`; both print rather than fail. ### Transforms `{lowercase(x)}`, `{uppercase(x)}` and `{ascii(x)}` rewrite the value of `x` — a field, a path or a `..path` — and nest. `ascii` folds Latin letters (`Åsa Öberg` -→ `Asa Oberg`) and drops any other non-ASCII rune. Like a calc operand, `x` is -drawn once per expansion, so an email built from a name matches the name beside it: +→ `Asa Oberg`) and drops any other non-ASCII rune. `x` is held +([One draw, one spelling](#one-draw-one-spelling)), so an email built from a name +matches the name beside it: ```json { "format": "{p.first} {p.last} <{lowercase(ascii(p.first))}.{lowercase(ascii(p.last))}@example.com>", @@ -282,9 +282,9 @@ through a chain. ### Correlated fields -`{name.field}` reads a path into a sibling, and a sibling read that way is drawn -**once per expansion**, so several tokens read one row — a locality and the -postal code that really covers it: +`{name.field}` reads a path into a sibling, which holds the sibling to one draw +([One draw, one spelling](#one-draw-one-spelling)), so several tokens read one +row — a locality and the postal code that really covers it: ```json { "format": "{street} {int(1,99)}\n{place.postal-code} {place.locality}", @@ -297,11 +297,9 @@ postal code that really covers it: Renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm code beside Tranås; each row's `weight` says how often it appears. A path is held at every -level it passes through (`{p.geo.town.name} {p.geo.town.zip}` share the town), -one path read twice reads one value, and a field no path addresses is drawn each -time (`{word} {word}` differs). The hold lasts one expansion: each `repeat` -iteration and each nested template draws again. Every variant of a choice on the -path must carry the rest of it, so a row missing a field is named at load: +level it passes through: `{p.geo.town.name} {p.geo.town.zip}` share the town. +Every variant of a choice on the path must carry the rest of it, so a row missing +a field is named at load: ```text token {place.postal-code}: field "place": not every variant of this 2-way choice carries "postal-code"; all carry [locality] @@ -312,14 +310,17 @@ The sub-fields stay addressable — `Fake("address.place.locality")` renders, an ### One draw, one spelling -A format may not both **render** a level and **read a path into** it — `{p}` -beside `{p.first}`, `{..cat.net}` beside `{calc(net * 2)}`, or `{q}` beside -`{p.first}` where `q` renders `{..cat.p.last}` — because the render draws afresh -while the path reads the held draw, and the two would disagree. Wherever the -second route sits, it is a load error naming the spelling to use: +A name any token reads as a path (`{p.first}`) or as an operand (`{calc(net * 2)}`, +`{uppercase(w)}`) is drawn **once per expansion**, and every other route to it — +a bare `{p}`, a second bare `{w}`, `{..cat.net}`, a nested template rendering +`{..cat.p.last}`, at any depth — is a load error naming the spelling to use. A +name nothing reads that way is drawn each time: `{word} {word}` differs. An +expansion is one render of one format, so each `repeat` iteration and each nested +template draws again. ```text token {p} renders a level that {p.first} reads a path into; name the fields you want instead +token {w} is repeated, and uppercase operand "w" holds "w" to one draw per expansion; write {w} once ``` ### Performance @@ -347,6 +348,11 @@ tokens add cost in proportion to the output. - **The shipped data is embedded, not discovered.** A directory a machine happens to have would make `--seed 42` machine-dependent. Data still lives in `data/` as JSON; `--data-path` layers over it. +- **A bare reference draws each time; a reference path is held.** `{..p} {..p}` + is two draws, as `{word} {word}` is, while `{..p.first}` beside a nested template + rendering `{..p.first}` is a load error: a bare token is by contract an + independent draw, a path pins its level, and any route into a pinned level from + another expansion could show another row. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. diff --git a/node.go b/node.go index 793c4f7..61566c1 100644 --- a/node.go +++ b/node.go @@ -87,13 +87,17 @@ func compileString(s string) (node, error) { return nil, err } t := &template{format: s, repeat: 1} - t.compileFormat() + if err := t.compileFormat(); err != nil { + return nil, err + } return t, nil } -// compileFormat compiles the format into ops once every field is in place. -func (t *template) compileFormat() { - t.ops, t.grow, t.bound, t.held = compileOps(t.format, t.refs) +// compileFormat compiles the format into ops once every field is in place, and +// applies the fences that need the compiled reads. +func (t *template) compileFormat() error { + c := compileOps(t.format, t.refs) + t.ops, t.grow, t.bound, t.held = c.ops, c.grow, c.bound, c.held t.fixed = true for _, o := range t.ops { if o.kind != 'l' { @@ -103,6 +107,10 @@ func (t *template) compileFormat() { if t.fixed && len(t.ops) == 1 { t.lit = t.ops[0].lit } + if err := checkNoOverlap(t.format, t.bound, t.refs); err != nil { + return err + } + return checkNoRepeatedRead(t.format, c, t.refs) } func compileChoice(items []any) (node, error) { @@ -218,8 +226,7 @@ func compileTemplate(m map[string]any) (node, error) { if err := checkTokens(format, t.fields); err != nil { return nil, err } - t.compileFormat() - if err := checkNoOverlap(format, t.bound, nil); err != nil { + if err := t.compileFormat(); err != nil { return nil, err } return t, nil diff --git a/reference.go b/reference.go index aac8a1d..0089f23 100644 --- a/reference.go +++ b/reference.go @@ -46,8 +46,7 @@ func linkRefs(root map[string]node) error { t.fields[key] = target t.refs[name] = key } - t.compileFormat() - if err := checkNoOverlap(t.format, t.bound, t.refs); err != nil { + if err := t.compileFormat(); err != nil { return fmt.Errorf("%s: %w", path, err) } return nil diff --git a/template.go b/template.go index fd6bb61..9c2e3a9 100644 --- a/template.go +++ b/template.go @@ -363,38 +363,69 @@ type op struct { operands []arm } -// compileOps turns a format string into ops, and returns the size of its literal -// text to size the render buffer, plus two sets. bound is the levels the format -// addresses by dotted path, each mapped to the first path reading it, which is what -// the overlap fences name. held is every name drawn once per expansion — those -// levels, plus the fields an operand reads, so an operand shown is the operand -// computed. Both are nil when the format needs neither, so data that uses neither -// carries no render-time cost. Call checkTokens first: it is what proves the scan -// and every token are valid. -func compileOps(format string, refs map[string]string) ([]op, int, map[string]string, map[string]bool) { - var ops []op - var lit strings.Builder - var bound map[string]string - var held map[string]bool - grow := 0 - hold := func(a arm) { - if held == nil { - held = map[string]bool{} +// formatOps is a compiled format: its ops, the size of its literal text (to size +// the render buffer), and the names drawn once per expansion. bound maps each level +// a path reads into to the first such path; held is every such level plus the +// fields an operand reads; holder maps each held name to the first reader holding +// it, for error messages. The maps are nil when the format holds nothing, so data +// that holds nothing carries no render-time cost. +type formatOps struct { + ops []op + grow int + bound map[string]string + held map[string]bool + holder map[string]string +} + +func (c *formatOps) hold(a arm, label string) { + if c.held == nil { + c.held = map[string]bool{} + c.holder = map[string]string{} + } + c.held[a.key] = true + if _, named := c.holder[a.key]; !named { + c.holder[a.key] = label + } + if len(a.tail) > 0 { + if c.bound == nil { + c.bound = map[string]string{} } - held[a.key] = true - if len(a.tail) > 0 { - if bound == nil { - bound = map[string]string{} - } - if _, named := bound[a.key]; !named { - bound[a.key] = a.name // the first path reading it, for error messages - } + if _, named := c.bound[a.key]; !named { + c.bound[a.key] = a.name } } +} + +func (c *formatOps) function(body string, refs map[string]string) { + name, args, _ := funcCall(body) + var operands []arm + for _, operand := range tokenOperands(body) { + a := splitArm(operand, refs) + c.hold(a, fmt.Sprintf("%s operand %q", name, operand)) + operands = append(operands, a) + } + c.ops = append(c.ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) +} + +func (c *formatOps) field(body string, refs map[string]string) { + arms := splitArms(body, refs) + for _, a := range arms { + if len(a.tail) > 0 { + c.hold(a, "token {"+a.name+"}") + } + } + c.ops = append(c.ops, op{kind: 'f', arms: arms}) +} + +// compileOps compiles a format string. Call checkTokens first: it is what proves +// the scan and every token are valid. +func compileOps(format string, refs map[string]string) formatOps { + var c formatOps + var lit strings.Builder flush := func() { if lit.Len() > 0 { - grow += lit.Len() - ops = append(ops, op{kind: 'l', lit: lit.String()}) + c.grow += lit.Len() + c.ops = append(c.ops, op{kind: 'l', lit: lit.String()}) lit.Reset() } } @@ -404,26 +435,38 @@ func compileOps(format string, refs map[string]string) ([]op, int, map[string]st lit.WriteRune(t.r) case 'b': flush() - if name, args, ok := funcCall(t.body); ok { - var operands []arm - for _, operand := range tokenOperands(t.body) { - a := splitArm(operand, refs) - hold(a) // the builtin renders its operand, so the expansion holds that draw - operands = append(operands, a) - } - ops = append(ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) + if _, _, isFunc := funcCall(t.body); isFunc { + c.function(t.body, refs) } else { - arms := splitArms(t.body, refs) - for _, a := range arms { - if len(a.tail) > 0 { - hold(a) - } - } - ops = append(ops, op{kind: 'f', arms: arms}) + c.field(t.body, refs) } } return nil }) flush() - return ops, grow, bound, held + return c +} + +// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside +// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The +// error names the single-token spelling. +func checkNoRepeatedRead(format string, c formatOps, refs map[string]string) error { + count := map[string]int{} + return eachToken(format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if _, _, isFunc := funcCall(t.body); isFunc { + return nil + } + for _, a := range splitArms(t.body, refs) { + if len(a.tail) > 0 || !c.held[a.key] { + continue + } + if count[a.key]++; count[a.key] > 1 { + return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name) + } + } + return nil + }) } -- 2.52.0 From 34c933b2b3c7e364a12337f4ebd77f755c134d12 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:22:23 +0200 Subject: [PATCH 20/38] Tests for weight 0, a never-numeric calc operand, the repeat product along a path, and ErrNoData --- calc_test.go | 35 +++++++++++++++++++++++++++++++---- cmd/fejkdata/main_test.go | 4 ++++ loading_test.go | 20 ++++++++++++++++++++ shipped_data_test.go | 5 +++-- template_test.go | 10 ++++------ 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/calc_test.go b/calc_test.go index f37f783..b24696c 100644 --- a/calc_test.go +++ b/calc_test.go @@ -72,12 +72,16 @@ func TestCalcFields(t *testing.T) { } } -// TestCalcNonNumericIsNaN pins the never-fail rule: a field that doesn't render -// to a number becomes NaN, which propagates and prints visibly. +// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does +// not render to a number becomes NaN then, which propagates and prints visibly. func TestCalcNonNumericIsNaN(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{calc(x * 2)}","x":"abc"}`); got != "NaN" { - t.Fatalf("calc over non-numeric field = %q, want NaN", got) + f := engine(1) + for i := 0; i < 50; i++ { + if got := mustRender(t, f, `{"format":"{calc(x * 2)}","x":["abc","1"]}`); got == "NaN" { + return + } } + t.Fatal("calc over a sometimes non-numeric field never printed NaN in 50 draws") } // TestCalcReproducible pins that a calc over a random operand stays seed-stable. @@ -218,3 +222,26 @@ func TestCalcHoldIsPerTemplate(t *testing.T) { t.Fatal("the nested template never differed from its parent, want its own draw") } } + +func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"{calc(x * 2)}","x":"abc"}`: `"abc"`, + `{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`, + `{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`, + } { + _, err := compile(parse(t, src)) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") { + t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want) + } + } + for _, ok := range []string{ + `{"format":"{calc(x * 2)}","x":"3"}`, + `{"format":"{calc(x * 2)}","x":" 2.5 "}`, + `{"format":"{calc(x * 2)}","x":["1","abc"]}`, + `{"format":"{calc(x * 2)}","x":"{digits(2)}"}`, + } { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v, want it accepted", ok, err) + } + } +} diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 86fd3e3..d8bc864 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -311,4 +311,8 @@ func TestRunNoShippedData(t *testing.T) { if code != 0 || strings.TrimSpace(out) == "" { t.Errorf("run = %d, out=%q", code, out) } + code, _, errb = runOut("--no-shipped-data", "sv_SE.person") + if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") { + t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb) + } } diff --git a/loading_test.go b/loading_test.go index f7aadfc..9c711f7 100644 --- a/loading_test.go +++ b/loading_test.go @@ -181,3 +181,23 @@ func TestHiddenEntriesAreSkipped(t *testing.T) { } } } + +func TestRepeatProductAlongAPathIsCapped(t *testing.T) { + for name, files := range map[string]map[string]string{ + "nested": {"cat": `{"format":"{a}","repeat":2048,"a":{"format":"{b}","repeat":2048,"b":"x"}}`}, + "through a reference": { + "a": `{"format":"{..b}","repeat":2048}`, + "b": `{"format":"x","repeat":2048}`, + }, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))) + if err == nil || !strings.Contains(err.Error(), "repeat") || !strings.Contains(err.Error(), "1048576") { + t.Errorf("%s: New = %v, want the repeat product rejected naming the maximum", name, err) + } + } + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{a}{c}","repeat":1024,"a":{"format":"{b}","repeat":1024,"b":"x"},"c":{"format":"y","repeat":1024}}`, + }))); err != nil { + t.Errorf("New = %v, want 1024 x 1024 along one path accepted", err) + } +} diff --git a/shipped_data_test.go b/shipped_data_test.go index c1d97e3..24aad04 100644 --- a/shipped_data_test.go +++ b/shipped_data_test.go @@ -1,6 +1,7 @@ package fejkdata import ( + "errors" "reflect" "slices" "strings" @@ -55,8 +56,8 @@ func TestUserDataMayReferenceShipped(t *testing.T) { func TestWithoutShippedDataNeedsASource(t *testing.T) { _, err := New(WithoutShippedData()) - if err == nil || !strings.Contains(err.Error(), "WithDataPath") { - t.Fatalf("New(WithoutShippedData()) = %v, want an error naming WithDataPath", err) + if err == nil || !strings.Contains(err.Error(), "WithDataPath") || !errors.Is(err, ErrNoData) { + t.Fatalf("New(WithoutShippedData()) = %v, want ErrNoData naming WithDataPath", err) } } diff --git a/template_test.go b/template_test.go index 1f8a59a..1f7f8f4 100644 --- a/template_test.go +++ b/template_test.go @@ -121,12 +121,10 @@ func TestAlternationPicksOneField(t *testing.T) { } } -func TestWeightZeroNeverChosen(t *testing.T) { - f := engine(5) - for i := 0; i < 200; i++ { - if got := mustRender(t, f, `[{"format":"X","weight":0},"Y"]`); got != "Y" { - t.Fatalf("weight-0 variant chosen: %q", got) - } +func TestWeightZeroIsRejected(t *testing.T) { + _, err := compile(parse(t, `[{"format":"X","weight":0},"Y"]`)) + if err == nil || !strings.Contains(err.Error(), "weight 0") || !strings.Contains(err.Error(), "remove") { + t.Fatalf("compile(weight 0) = %v, want it rejected naming the fix", err) } } -- 2.52.0 From 1b1b93b35f0d5047649fde0b84b8290b34cc468e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:23:03 +0200 Subject: [PATCH 21/38] Reject weight 0, a never-numeric calc operand and a repeat product past the cap; the CLI classifies ErrNoData --- README.md | 13 ++++++++----- builtins.go | 7 ++++--- calc.go | 30 +++++++++++++++++++++++++++++- cmd/fejkdata/main.go | 6 +++--- data.go | 3 +++ fejkdata.go | 6 +++++- node.go | 11 +++++++---- reference.go | 30 ++++++++++++++++++++++++++++++ 8 files changed, 89 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ecb68b2..85857ff 100644 --- a/README.md +++ b/README.md @@ -167,8 +167,8 @@ An item of a choice may carry a `weight` (default `1`) to skew its odds: Renders `070-412 38 91` ten times as often as `08-…`. A string item is weighted by writing it as `{ "format": "AB", "weight": 3 }`. Rejected at load: a weight that -is negative, non-numeric, `1` (the default) or outside a choice, a set summing to -zero, and a repeated item — `["a", "a", "b"]` is `[{ "format": "a", "weight": 2 }, "b"]`, +is negative, non-numeric, `0` (never drawn — remove the item), `1` (the default) +or outside a choice, and a repeated item — `["a", "a", "b"]` is `[{ "format": "a", "weight": 2 }, "b"]`, and the error says so. ### Repeat @@ -180,7 +180,9 @@ many times — each an independent draw — joined by `separator` (default `""`) { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -Renders e.g. `bar foo baz`. A `separator` without a `repeat` is rejected. +Renders e.g. `bar foo baz`. A `separator` without a `repeat` is rejected, and so +is a `repeat` that multiplies to more than 1 048 576 renders along any path of +nested repeats. ### Options and fields @@ -240,8 +242,9 @@ hyphenated field can't be an operand. { "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } ``` -Renders e.g. `19.99 x 3 = 59.97`. A non-numeric operand yields `NaN` and a -division by zero `Inf`; both print rather than fail. +Renders e.g. `19.99 x 3 = 59.97`. An operand that can never be a number (`"abc"`, +or a choice of such) is rejected at load; one that sometimes is not yields `NaN`, +and a division by zero `Inf` — both print rather than fail. ### Transforms diff --git a/builtins.go b/builtins.go index d264f00..ae025ac 100644 --- a/builtins.go +++ b/builtins.go @@ -9,9 +9,10 @@ import ( "unicode" ) -// maxLen caps sample output lengths (hex, nanoid, base64) and maxDecimals caps -// float/calc decimal places, so a fat-fingered or overflowing argument fails at -// New instead of trying to allocate gigabytes — or panicking — at render. +// maxLen caps sample output lengths (hex, nanoid, base64) and the renders a repeat +// multiplies to along any path; maxDecimals caps float/calc decimal places. So a +// fat-fingered or overflowing argument fails at New instead of trying to allocate +// gigabytes — or panicking — at render. const ( maxLen = 1 << 20 // 1,048,576 chars/bytes maxDecimals = 1024 diff --git a/calc.go b/calc.go index 4d068e4..a9a8653 100644 --- a/calc.go +++ b/calc.go @@ -70,9 +70,13 @@ func checkCalc(fields map[string]node, args []string) error { return fmt.Errorf("calc(%q): %w", args[0], err) } for _, name := range calcVars(expr) { - if _, ok := fields[name]; !ok { + operand, ok := fields[name] + if !ok { return fmt.Errorf("calc(%q): no field %q", args[0], name) } + if text, never := neverNumeric(operand); never { + return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text) + } } if len(args) == 2 { if dp, err := strconv.Atoi(args[1]); err != nil || dp < 0 || dp > maxDecimals { @@ -82,6 +86,30 @@ func checkCalc(fields map[string]node, args []string) error { return nil } +// neverNumeric reports a node no render of which is a number: fixed text that does +// not parse, or a choice of only such items. text is one such render. +func neverNumeric(n node) (text string, never bool) { + switch n := n.(type) { + case *template: + if !n.fixed || n.repeat > 1 { + return "", false + } + if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil { + return n.lit, true + } + case *choice: + for _, it := range n.items { + t, itemNever := neverNumeric(it) + if !itemNever { + return "", false + } + text = t + } + return text, true + } + return "", false +} + // calcPrep parses the expression and decimals once, at compile time, and places each // operand name at the position expand will read it into. checkCalc proved both args // valid, so no step here can fail; dp -1 prints the minimal form. diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 9f35f96..0b335a1 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -185,9 +185,6 @@ func parseArgs(argv []string) (invocation, error) { // check rejects a flag combination that cannot run. func (in invocation) check() error { - if in.noShipped && len(in.dirs) == 0 { - return errors.New("--no-shipped-data needs at least one --data-path") - } if in.list && len(in.paths) > 0 { return errors.New("--list takes no path") } @@ -244,6 +241,9 @@ func run(args []string, stdout, stderr io.Writer) int { return misuse(stderr, err) } f, err := fejkdata.New(in.options()...) + if errors.Is(err, fejkdata.ErrNoData) { + return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path")) + } if err != nil { fmt.Fprintln(stderr, err) return 1 diff --git a/data.go b/data.go index 0be4a02..bfc4a40 100644 --- a/data.go +++ b/data.go @@ -64,6 +64,9 @@ func loadData(sources []dataSource) (map[string]node, error) { if err := checkNoCycles(root); err != nil { return nil, err } + if err := checkRepeatReach(root); err != nil { + return nil, err + } if err := checkBoundLevelsHeld(root); err != nil { return nil, err } diff --git a/fejkdata.go b/fejkdata.go index adff4aa..ff7c052 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -16,6 +16,7 @@ import ( crand "crypto/rand" "embed" "encoding/binary" + "errors" "fmt" "io/fs" "math/rand/v2" @@ -27,6 +28,9 @@ import ( //go:embed data var shippedFS embed.FS +// ErrNoData is returned by New when no source is loaded at all. +var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") + // Generator generates fake data from a loaded namespace tree. Create one with [New]. // It is safe for concurrent use; a seeded sequence is reproducible only when drawn // from one goroutine. @@ -99,7 +103,7 @@ func New(opts ...Option) (*Generator, error) { } sources = append(sources, c.sources...) if len(sources) == 0 { - return nil, fmt.Errorf("fejkdata: no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") + return nil, fmt.Errorf("fejkdata: %w", ErrNoData) } cats, err := loadData(sources) if err != nil { diff --git a/node.go b/node.go index 61566c1..d6fa84f 100644 --- a/node.go +++ b/node.go @@ -144,8 +144,8 @@ func compileChoice(items []any) (node, error) { c.items[i] = n } if weighted { // uniform choices skip the weight table and pick in O(1) - if total <= 0 || math.IsInf(total, 1) { - return nil, fmt.Errorf("choice weights must sum to a finite positive number, got %v", total) + if math.IsInf(total, 1) { + return nil, fmt.Errorf("choice weights must sum to a finite number, got %v", total) } c.cum = cum } @@ -293,7 +293,7 @@ func repeatOf(m map[string]any) (int, error) { } // weightOf reads a node's "weight" (default 1) from its raw JSON form. Only -// template objects carry weight; a present one must be finite and non-negative. +// template objects carry weight; a present one must be finite and positive. func weightOf(raw any) (float64, error) { m, ok := raw.(map[string]any) if !ok { @@ -308,7 +308,10 @@ func weightOf(raw any) (float64, error) { return 0, fmt.Errorf("weight must be a number, got %T", wv) } if w < 0 || math.IsNaN(w) || math.IsInf(w, 0) { - return 0, fmt.Errorf("weight must be finite and non-negative, got %v", w) + return 0, fmt.Errorf("weight must be finite and positive, got %v", w) + } + if w == 0 { + return 0, fmt.Errorf("weight 0 means the item is never drawn; remove the item instead") } if w == 1 { return 0, fmt.Errorf("weight 1 is the default, so it has no effect; drop it") diff --git a/reference.go b/reference.go index 0089f23..b240665 100644 --- a/reference.go +++ b/reference.go @@ -412,6 +412,36 @@ func pathLeaves(n node, tail []string) []node { return pathLeaves(child(n, tail[0]), tail[1:]) } +// checkRepeatReach bounds the renders a repeat multiplies to along any root-to-leaf +// path, so nested repeats cannot build what one repeat may not. It runs after +// checkNoCycles, whose guarantee is what lets the walk terminate. +func checkRepeatReach(root map[string]node) error { + reach := map[node]int{} + var of func(n node) int + of = func(n node) int { + if r, done := reach[n]; done { + return r + } + r := 1 + for _, e := range renderEdges(n) { + if c := of(e.to); c > r { + r = c + } + } + if t, ok := n.(*template); ok { + r *= t.repeat + } + reach[n] = r + return r + } + return walkNodes(root, func(path string, n node) error { + if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > maxLen { + return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), maxLen) + } + return nil + }) +} + // checkNoCycles rejects a reference cycle: a node whose rendering can reach itself // — directly, mutually, or through a chain — never terminates, so it must fail at // New rather than stack-overflow at render. It is a depth-first walk of the render -- 2.52.0 From 5d1e567b8ddbbb0cb3ea2d3fc98479bd4cf1c259 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:24:58 +0200 Subject: [PATCH 22/38] Tests for reference sigils: / the root, . this folder, .. the folder above --- bound_test.go | 67 ++++++++++---------- edge_test.go | 8 +-- loading_test.go | 6 +- reference_test.go | 125 +++++++++++++++++++++++++------------ shipped_data_test.go | 2 +- template_stability_test.go | 2 +- transform_test.go | 2 +- 7 files changed, 129 insertions(+), 83 deletions(-) diff --git a/bound_test.go b/bound_test.go index 89f6d2a..76f2053 100644 --- a/bound_test.go +++ b/bound_test.go @@ -92,13 +92,13 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { // afresh beside the path that reads its held draw — the same overlap by // another spelling. rejected := map[string]string{ - "reference names the head": `{"format":"{p.first}|{..cat.p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, - "reference names the leaf": `{"format":"{p.addr}|{..cat.p.addr}","p":{"format":"x","addr":["A","B","C","D"]}}`, + "reference names the head": `{"format":"{p.first}|{/cat.p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + "reference names the leaf": `{"format":"{p.addr}|{/cat.p.addr}","p":{"format":"x","addr":["A","B","C","D"]}}`, // The reference need not sit in the format that binds: any field it renders // reaches the level just the same, however deep. "reference from a sibling field": `{"format":"{p.first}|{inner}","p":[` + `{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` + - `"inner":"{..cat.p}"}`, + `"inner":"{/cat.p}"}`, } for name, file := range rejected { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) @@ -108,7 +108,7 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { } // A reference to anything this format does not bind is untouched. if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{p.first} {..surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + "cat": `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`, "surname": `["Eriksson","Lindqvist"]`, }))); err != nil { t.Errorf("New = %v, want a reference outside the bound level accepted", err) @@ -120,7 +120,7 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { // cycle walk has to follow it there. A head whose own format names nothing // would otherwise hide the cycle until render, where it is fatal. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"static","x":"{..a}"}}`, + "a": `{"format":"{p.x}","p":{"format":"static","x":"{/a}"}}`, }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) @@ -131,7 +131,7 @@ func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) { // {p.a} renders q, so it is a route to the level {q.x} holds — even though p's // own format names nothing. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{..thing.q}"},` + + "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/thing.q}"},` + `"q":{"format":"{x}","x":["1","2"]}}`, }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { @@ -151,14 +151,14 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { }{ "a reference beside the operand": { map[string]string{ - "cat": `{"format":"{..cat.net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`, + "cat": `{"format":"{/cat.net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`, }, - `{..cat.net} renders "net"`, + `{/cat.net} renders "net"`, }, "a reference one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` + - `"q":"{..cat.net}"}`, + `"q":"{/cat.net}"}`, }, `{q} renders "net"`, }, @@ -167,7 +167,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference to an operand wrapped in a choice": { map[string]string{ "cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` + - `"q":"{..cat.n}"}`, + `"q":"{/cat.n}"}`, }, `{q} renders "n"`, }, @@ -176,7 +176,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "an operand reaching another operand": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` + - `"b":"{..cat.a}"}`, + `"b":"{/cat.a}"}`, }, `calc operand "b" renders "a"`, }, @@ -185,15 +185,15 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { // rejected, so the reference spelling has to be. "a reference into the operand": { map[string]string{ - "cat": `{"format":"{calc(net * 2, 2)}|{..cat.net.v}",` + + "cat": `{"format":"{calc(net * 2, 2)}|{/cat.net.v}",` + `"net":{"format":"{v}","v":["10.00","20.00"]}}`, }, - `{..cat.net.v} renders "net"`, + `{/cat.net.v} renders "net"`, }, "a reference into the operand one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` + - `"q":"{..cat.net.v}"}`, + `"q":"{/cat.net.v}"}`, }, `{q} renders "net"`, }, @@ -203,7 +203,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a violation on the second of two held heads": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` + - `"b":{"format":"{y}","y":["3","4"]},"w":"{..cat.b}"}`, + `"b":{"format":"{y}","y":["3","4"]},"w":"{/cat.b}"}`, }, `{w} renders "b"`, }, @@ -230,24 +230,24 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference to a sibling the operand never renders": { "cat": `{"format":"{calc(net * 2, 2)} {unit}",` + `"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` + - `"unit":"{..cat.net.spare}"}`, + `"unit":"{/cat.net.spare}"}`, }, // Two operands drawing from one source are two names, so two draws: each is // held under its own name and shown once, and neither can disagree. "two operands sharing one source": { "die": `["1","2","3","4","5","6"]`, - "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{..die}",` + - `"d2":"{..die}"}`, + "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` + + `"d2":"{/die}"}`, }, - // Likewise a reference drawing from what the operand draws from: {..common} + // Likewise a reference drawing from what the operand draws from: {/common} // and net are two names, not two spellings of one field. "a reference to what an operand renders through": { "common": `["1","2"]`, - "cat": `{"format":"{calc(net * 2, 2)} {..common}","net":"{..common}"}`, + "cat": `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`, }, // A fixed string cannot disagree with itself, so it needs no fence. "a literal operand named twice": { - "cat": `{"format":"{calc(n * 2, 0)} {..cat.n}","n":"5"}`, + "cat": `{"format":"{calc(n * 2, 0)} {/cat.n}","n":"5"}`, }, // One node reached twice while walking the operand: the walk must not // revisit it, and the repeat is not a second route to anything. @@ -269,11 +269,11 @@ func TestAPathReachesEveryVariantItMightDraw(t *testing.T) { // to a held level disagrees silently. rejected := map[string]struct{ file, want string }{ "a cycle in a later variant": { - `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{..cat}"}]}`, + `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/cat}"}]}`, "reference cycle", }, "a second route in a later variant": { - `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{..cat.q}"}],` + + `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/cat.q}"}],` + `"q":{"format":"{y}","y":["1","2"]}}`, "reads a path into", }, @@ -292,9 +292,9 @@ func TestALevelAPathNeverRendersIsAccepted(t *testing.T) { // path at all. Both orders must load. accepted := map[string]string{ "reference in the head's own format": `{"format":"{p.first} {q.a}",` + - `"p":{"format":"{first} {..thing.q}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}`, + `"p":{"format":"{first} {/thing.q}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}`, "the mirror shape": `{"format":"{p.first} {q.a}",` + - `"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {..thing.p}","a":["1","2"]}}`, + `"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/thing.p}","a":["1","2"]}}`, } for name, file := range accepted { if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"thing": file}))); err != nil { @@ -309,9 +309,9 @@ func TestADeepDiamondChainLoads(t *testing.T) { files := map[string]string{"l0": `"x"`} for i := 1; i <= 30; i++ { files[fmt.Sprintf("l%d", i)] = fmt.Sprintf( - `{"format":"{a}{b}","a":"{..l%d}","b":"{..l%d}"}`, i-1, i-1) + `{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1) } - files["thing"] = `{"format":"{p.first} {..l30}","p":{"format":"x","first":["A","B"]}}` + files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}` if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { t.Fatalf("New = %v, want a deep diamond chain to load", err) } @@ -323,7 +323,7 @@ func TestASharedNodeIsWalkedOnce(t *testing.T) { // the shared node once per route. f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` + - `"q":{"format":"{a}{b}","a":"{..shared}","b":"{..shared}"}}`, + `"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`, "shared": `"x"`, })), WithSeed(1)) if err != nil { @@ -351,7 +351,7 @@ func TestReferenceToAMatchingStringIsAccepted(t *testing.T) { // held one. Two unrelated literals that merely spell the same text must not // read as the same node — the trap when comparing a value type. if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "cat": `{"format":"{p.city} {..other.tag}","p":{"format":"{city}","city":"Stockholm"}}`, + "cat": `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`, "other": `{"format":"x","tag":"Stockholm"}`, }))); err != nil { t.Fatalf("New = %v, want a reference to a matching string accepted", err) @@ -642,12 +642,11 @@ func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) { } func TestEmptyPathSegmentIsRejected(t *testing.T) { - // "{a.}", "{.b}" and "{a..b}" are unfinished paths, and the segment naming - // nothing is reported as that rather than as a missing field. An empty name is - // rejected where it is authored, so no data can make these resolve. + // "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is + // reported as that rather than as a missing field. An empty name is rejected + // where it is authored, so no data can make these resolve. rejected := map[string]string{ "trailing dot": `{"format":"[{a.}]","a":"x"}`, - "leading dot": `{"format":"[{.b}]","a":{"format":"{b}","b":"V"}}`, "double dot": `{"format":"[{a..b}]","a":"x"}`, } for name, file := range rejected { @@ -667,7 +666,7 @@ func TestCycleThroughAPathTokenIsRejected(t *testing.T) { // must be caught at New. Reaching render would be fatal: the recursion never // terminates, and a stack overflow cannot be recovered. _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"{x}","x":"{..a}"}}`, + "a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/a}"}}`, }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) diff --git a/edge_test.go b/edge_test.go index eee05c8..6dd2c1c 100644 --- a/edge_test.go +++ b/edge_test.go @@ -120,7 +120,7 @@ func TestNewErrors(t *testing.T) { `category "a|b" contains "|"`, }, // An empty name is not a path segment, so List never offered it — while a - // bare {}, a trailing dot in Fake("a.") and a {..a.} reference all reached + // bare {}, a trailing dot in Fake("a.") and a {/a.} reference all reached // it. The engine accepted spellings it would never advertise. "empty field name": { map[string]string{"a": `{"format":"[{}]","":"VALUE"}`}, @@ -146,13 +146,13 @@ func TestNewErrors(t *testing.T) { // A reference arm is the only kind that reaches the repeat check by passing // the per-arm checks rather than falling through them. "repeated reference arm": { - map[string]string{"a": `"x"`, "b": `"{..a|..a}"`}, - `arm "..a" is repeated`, + map[string]string{"a": `"x"`, "b": `"{/a|/a}"`}, + `arm "/a" is repeated`, }, // An arm that is broken on its own terms is reported as that, not as a // repeat: the repeat is a consequence of the real mistake. "repeated arm with no path": { - map[string]string{"a": `"{..|..}"`}, + map[string]string{"a": `"{/|/}"`}, "reference has no path", }, // No field can be named "", so the token is told that rather than sent to diff --git a/loading_test.go b/loading_test.go index 9c711f7..b590b6c 100644 --- a/loading_test.go +++ b/loading_test.go @@ -16,8 +16,8 @@ func TestList(t *testing.T) { "person": `{"format":"{first} {last}","first":"A","last":"B"}`, "word": `["x", "y"]`, "geo/city": `"Z"`, - // A bound {..path} reference is a render edge, not an addressable field. - "greeting": `{"format":"hej {..person.first} and {own}","own":"x"}`, + // A bound {/path} reference is a render edge, not an addressable field. + "greeting": `{"format":"hej {/person.first} and {own}","own":"x"}`, // Only the fields every variant carries are addressable, so "extra" is not. "coin": `[{"format":"{code}","code":"A","name":"Aa"},{"format":"{code}","code":"B","name":"Bb","extra":"x"}]`, }) @@ -186,7 +186,7 @@ func TestRepeatProductAlongAPathIsCapped(t *testing.T) { for name, files := range map[string]map[string]string{ "nested": {"cat": `{"format":"{a}","repeat":2048,"a":{"format":"{b}","repeat":2048,"b":"x"}}`}, "through a reference": { - "a": `{"format":"{..b}","repeat":2048}`, + "a": `{"format":"{/b}","repeat":2048}`, "b": `{"format":"x","repeat":2048}`, }, } { diff --git a/reference_test.go b/reference_test.go index 90d35b6..a87e68a 100644 --- a/reference_test.go +++ b/reference_test.go @@ -6,11 +6,11 @@ import ( ) // TestRootReferenceAcrossFolders is the headline case: a category in one folder -// pulls a value from another via a {..path} reference resolved from the data root. +// pulls a value from another via a {/path} reference resolved from the data root. func TestRootReferenceAcrossFolders(t *testing.T) { dir := writeData(t, map[string]string{ "en_US/person": `"Pat Smith"`, - "sv_SE/greeting": `"Hej, {..en_US.person}!"`, + "sv_SE/greeting": `"Hej, {/en_US.person}!"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "sv_SE.greeting"); got != "Hej, Pat Smith!" { @@ -22,7 +22,7 @@ func TestRootReferenceAcrossFolders(t *testing.T) { func TestReferenceIntoAField(t *testing.T) { dir := writeData(t, map[string]string{ "who": `{"format":"{first} {last}","first":"Ada","last":"Byron"}`, - "card": `"signed {..who.last}"`, + "card": `"signed {/who.last}"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "card"); got != "signed Byron" { @@ -30,12 +30,12 @@ func TestReferenceIntoAField(t *testing.T) { } } -// TestReferenceInAlternation lets a reference stand as one arm of a {a|..b} +// TestReferenceInAlternation lets a reference stand as one arm of a {a|/b} // alternation, so a field and a cross-file value share one slot. func TestReferenceInAlternation(t *testing.T) { dir := writeData(t, map[string]string{ "far": `"X"`, - "near": `{"format":"{here|..far}","here":"H"}`, + "near": `{"format":"{here|/far}","here":"H"}`, }) f := newGenerator(t, dir, WithSeed(2)) seen := map[string]bool{} @@ -51,7 +51,7 @@ func TestReferenceInAlternation(t *testing.T) { // model: data layered from two dirs can point at each other through the root. func TestReferenceCombinesLoadedPaths(t *testing.T) { a := writeData(t, map[string]string{"en_US/word": `"river"`}) - b := writeData(t, map[string]string{"mine/slug": `"the-{..en_US.word}"`}) + b := writeData(t, map[string]string{"mine/slug": `"the-{/en_US.word}"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "mine.slug"); got != "the-river" { t.Fatalf("slug = %q, want the-river", got) @@ -62,8 +62,8 @@ func TestReferenceCombinesLoadedPaths(t *testing.T) { // linking order cannot matter. func TestReferenceChain(t *testing.T) { dir := writeData(t, map[string]string{ - "a": `"{..b}"`, - "b": `"{..c}"`, + "a": `"{/b}"`, + "b": `"{/c}"`, "c": `"deep"`, }) f := newGenerator(t, dir, WithSeed(1)) @@ -76,37 +76,37 @@ func TestReferenceChain(t *testing.T) { // fails at load, never at a random render. func TestReferenceErrors(t *testing.T) { cases := map[string]map[string]string{ - "missing target": {"card": `"{..nope.gone}"`}, - "folder target": {"en_US/word": `"w"`, "card": `"{..en_US}"`}, + "missing target": {"card": `"{/nope.gone}"`}, + "folder target": {"en_US/word": `"w"`, "card": `"{/en_US}"`}, "a variant on the path lacks the field": { "who": `[{"format":"{f}","f":"1"},{"format":"{g}","g":"2"}]`, - "card": `"{..who.f}"`, + "card": `"{/who.f}"`, }, - "empty reference path": {"card": `"{..}"`}, + "empty reference path": {"card": `"{/}"`}, // A reference that leads back to its own value never terminates at render, // so New must reject the cycle up front (direct, mutual, or chained). - "direct cycle": {"a": `"x{..a}"`}, - "mutual cycle": {"a": `"{..b}"`, "b": `"{..a}"`}, - "chain cycle": {"a": `"{..b}"`, "b": `"{..c}"`, "c": `"{..a}"`}, + "direct cycle": {"a": `"x{/a}"`}, + "mutual cycle": {"a": `"{/b}"`, "b": `"{/a}"`}, + "chain cycle": {"a": `"{/b}"`, "b": `"{/c}"`, "c": `"{/a}"`}, // calc renders its operands, so a cycle through one must be caught too. - "calc operand cycle": {"x": `{"format":"{calc(y)}","y":"{..x}"}`}, + "calc operand cycle": {"x": `{"format":"{calc(y)}","y":"{/x}"}`}, // A field its parent's format never renders is still reachable by dot path, // so a cycle hiding in one must fail at New rather than at render. - "cycle in an unrendered field": {"cat": `{"format":"hi","x":"{..cat.x}"}`}, + "cycle in an unrendered field": {"cat": `{"format":"hi","x":"{/cat.x}"}`}, "mutual cycle between unrendered fields": { - "cat": `{"format":"hi","x":"{..cat.y}","y":"{..cat.x}"}`, + "cat": `{"format":"hi","x":"{/cat.y}","y":"{/cat.x}"}`, }, "cycle in an unrendered field of a choice arm": { - "cat": `{"format":"hi","x":"{..cat.x}"}`, + "cat": `{"format":"hi","x":"{/cat.x}"}`, }, // The shipped layout puts categories in folders, so a cycle one level down // is the common case, not an edge case. - "cycle in a subfolder": {"sv_SE/a": `"x{..sv_SE.a}"`}, - "mutual cycle within a subfolder": {"sv_SE/a": `"{..sv_SE.b}"`, "sv_SE/b": `"{..sv_SE.a}"`}, - "mutual cycle across two folders": {"en_US/a": `"{..sv_SE.b}"`, "sv_SE/b": `"{..en_US.a}"`}, + "cycle in a subfolder": {"sv_SE/a": `"x{/sv_SE.a}"`}, + "mutual cycle within a subfolder": {"sv_SE/a": `"{/sv_SE.b}"`, "sv_SE/b": `"{/sv_SE.a}"`}, + "mutual cycle across two folders": {"en_US/a": `"{/sv_SE.b}"`, "sv_SE/b": `"{/en_US.a}"`}, // ".." is reserved for bound references, so an authored key using it would // name a node nothing can reach and nothing would validate. - "field key using the reference prefix": {"cat": `{"format":"hi","..x":"{..nope}"}`}, + "field key using the reference prefix": {"cat": `{"format":"hi","..x":"{/nope}"}`}, } for name, files := range cases { if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { @@ -122,8 +122,8 @@ func TestReferenceErrors(t *testing.T) { func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { f := newGenerator(t, writeData(t, map[string]string{ "sv_SE/ok": `"fine"`, - "sv_SE/..bad": `"{..nope}"`, - "sv_SE/..y/ct": `"{..nope}"`, + "sv_SE/..bad": `"{/nope}"`, + "sv_SE/..y/ct": `"{/nope}"`, ".git/config": `"not data"`, }), WithSeed(1)) if got := f.List(); len(got) != 1 || got[0] != "sv_SE.ok" { @@ -135,7 +135,7 @@ func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { // over-rejecting: a field the format never renders may point back at its own // category, which terminates, and stays renderable by path. func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { - dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":"see {..cat}"}`}) + dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":"see {/cat}"}`}) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "cat"); got != "hi" { t.Fatalf("cat = %q, want hi", got) @@ -151,9 +151,9 @@ func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { func TestNewErrorIsDeterministic(t *testing.T) { cases := map[string]map[string]string{ "three bad references": { - "a": `"{..nope.one}"`, - "b": `"{..nope.two}"`, - "c": `"{..nope.three}"`, + "a": `"{/nope.one}"`, + "b": `"{/nope.two}"`, + "c": `"{/nope.three}"`, }, "two bad fields in one template": { "cat": `{"format":"hi","aaa":{"no":1},"zzz":{"no":2}}`, @@ -180,7 +180,7 @@ func TestNewErrorIsDeterministic(t *testing.T) { } // TestNewErrorPathIsCanonical pins the node path a load error names: a choice arm -// adds no segment, and a bound {..path} reference is not a containment segment at +// adds no segment, and a bound {/path} reference is not a containment segment at // all, so a bad reference is reported against the node that holds it. func TestNewErrorPathIsCanonical(t *testing.T) { cases := []struct { @@ -190,13 +190,13 @@ func TestNewErrorPathIsCanonical(t *testing.T) { }{ { "cycle inside a choice arm", - map[string]string{"cat": `{"format":"hi","x":"{..cat.x}"}`}, - "fejkdata: reference cycle: cat.x -> ..cat.x", + map[string]string{"cat": `{"format":"hi","x":"{/cat.x}"}`}, + "fejkdata: reference cycle: cat.x -> /cat.x", }, { "bad reference reached through another reference", - map[string]string{"a": `"{..b}"`, "b": `"{..nope}"`}, - `fejkdata: b: reference {..nope}: no entry "nope"`, + map[string]string{"a": `"{/b}"`, "b": `"{/nope}"`}, + `fejkdata: b: reference {/nope}: no entry "nope"`, }, } for _, c := range cases { @@ -214,7 +214,7 @@ func TestNewErrorPathIsCanonical(t *testing.T) { func TestReferencePathIsHeld(t *testing.T) { dir := writeData(t, map[string]string{ "person": `[{"format":"{first} {last}","first":"Anna","last":"Andersson"},{"format":"{first} {last}","first":"Bo","last":"Berg"}]`, - "card": `"{..person.first} {..person.last}"`, + "card": `"{/person.first} {/person.last}"`, }) f := newGenerator(t, dir, WithSeed(3)) seen := map[string]bool{} @@ -233,7 +233,7 @@ func TestReferencePathIsHeld(t *testing.T) { func TestReferenceThroughChoiceNeedsEveryVariant(t *testing.T) { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "who": `[{"format":"{f}{h}","f":"1","h":"x"},{"format":"{g}{h}","g":"2","h":"y"}]`, - "card": `"{..who.f}"`, + "card": `"{/who.f}"`, }))) if err == nil || !strings.Contains(err.Error(), "not every variant") { t.Fatalf("New = %v, want the missing variant named", err) @@ -243,7 +243,7 @@ func TestReferenceThroughChoiceNeedsEveryVariant(t *testing.T) { func TestBareReferenceDrawsEachTime(t *testing.T) { dir := writeData(t, map[string]string{ "die": `["1","2","3","4","5","6"]`, - "roll": `"{..die} {..die}"`, + "roll": `"{/die} {/die}"`, }) f := newGenerator(t, dir, WithSeed(1)) for i := 0; i < 50; i++ { @@ -256,8 +256,8 @@ func TestBareReferenceDrawsEachTime(t *testing.T) { func TestReferenceOverlapIsRejected(t *testing.T) { for name, file := range map[string]string{ - "head beside a path": `{"format":"{..cat.p} {..cat.p.first}","p":[{"format":"{first}","first":"A"},{"format":"{first}","first":"B"}]}`, - "sibling path beside a reference path": `{"format":"{p.first} {..cat.p.last}","p":[{"format":"{first}","first":"A","last":"1"},{"format":"{first}","first":"B","last":"2"}]}`, + "head beside a path": `{"format":"{/cat.p} {/cat.p.first}","p":[{"format":"{first}","first":"A"},{"format":"{first}","first":"B"}]}`, + "sibling path beside a reference path": `{"format":"{p.first} {/cat.p.last}","p":[{"format":"{first}","first":"A","last":"1"},{"format":"{first}","first":"B","last":"2"}]}`, } { _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { @@ -265,3 +265,50 @@ func TestReferenceOverlapIsRejected(t *testing.T) { } } } + +func TestRelativeReferences(t *testing.T) { + dir := writeData(t, map[string]string{ + "sv_SE/username": `"bob"`, + "sv_SE/email": `"{.username}@example.com"`, + "sv_SE/deep/card": `"{..username} via {/sv_SE.username}"`, + "top": `"{.sv_SE.username}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for path, want := range map[string]string{ + "sv_SE.email": "bob@example.com", + "sv_SE.deep.card": "bob via bob", + "top": "bob", + } { + if got := fake(t, f, path); got != want { + t.Errorf("%s = %q, want %q", path, got, want) + } + } +} + +func TestRelativeAndRootSpellingsBindOneDraw(t *testing.T) { + dir := writeData(t, map[string]string{ + "sv_SE/person": `[{"format":"{first} {last}","first":"Anna","last":"Andersson"},{"format":"{first} {last}","first":"Bo","last":"Berg"}]`, + "sv_SE/card": `"{.person.first} {/sv_SE.person.last}"`, + }) + f := newGenerator(t, dir, WithSeed(3)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "sv_SE.card"); got != "Anna Andersson" && got != "Bo Berg" { + t.Fatalf("card = %q, want both spellings to read one person", got) + } + } +} + +func TestReferenceSigilErrors(t *testing.T) { + for name, files := range map[string]map[string]string{ + "no folder above the root": {"a": `"{..b}"`, "b": `"x"`}, + "three dots": {"a": `"{...b}"`, "b": `"x"`}, + "root sigil alone": {"a": `"{/}"`}, + "dot alone": {"a": `"{.}"`}, + "missing sibling": {"sv_SE/a": `"{.nope}"`}, + "slash in a field name": {"a": `{"format":"{x}","x":"1","a/b":"2"}`}, + } { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { + t.Errorf("%s: New = nil error, want a reference error", name) + } + } +} diff --git a/shipped_data_test.go b/shipped_data_test.go index 24aad04..be776c1 100644 --- a/shipped_data_test.go +++ b/shipped_data_test.go @@ -44,7 +44,7 @@ func TestWithDataPathLayersOverShipped(t *testing.T) { } func TestUserDataMayReferenceShipped(t *testing.T) { - dir := writeData(t, map[string]string{"greeting": `"Hej {..sv_SE.person}!"`}) + dir := writeData(t, map[string]string{"greeting": `"Hej {/sv_SE.person}!"`}) f, err := New(WithDataPath(dir), WithSeed(1)) if err != nil { t.Fatalf("New = %v", err) diff --git a/template_stability_test.go b/template_stability_test.go index 7204a23..e0e584f 100644 --- a/template_stability_test.go +++ b/template_stability_test.go @@ -13,7 +13,7 @@ func TestSeededOutputIsStable(t *testing.T) { "escapes": `{"format":"01Aa#{x}","x":"!"}`, "funcs": `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`, "nested": `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`, - "ref": `"see {..alt}"`, + "ref": `"see {/alt}"`, "repeat": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`, "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`, "weights": `[{"format":"big","weight":9},"tiny"]`, diff --git a/transform_test.go b/transform_test.go index 37b8f96..d201427 100644 --- a/transform_test.go +++ b/transform_test.go @@ -28,7 +28,7 @@ func TestTransformReadsTheHeldDraw(t *testing.T) { func TestTransformOverAReference(t *testing.T) { dir := writeData(t, map[string]string{ "person": `[{"format":"{first} {last}","first":"Åsa","last":"Öberg"},{"format":"{first} {last}","first":"Bo","last":"Ek"}]`, - "email": `"{..person.first} {..person.last} <{lowercase(ascii(..person.first))}.{lowercase(ascii(..person.last))}@example.com>"`, + "email": `"{/person.first} {/person.last} <{lowercase(ascii(/person.first))}.{lowercase(ascii(/person.last))}@example.com>"`, }) f := newGenerator(t, dir, WithSeed(5)) for i := 0; i < 50; i++ { -- 2.52.0 From 1bf5470c6ff61c3ecbd45274ba2a664794d00bd8 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:26:02 +0200 Subject: [PATCH 23/38] Reference sigils follow the filesystem: / the root, . this folder, .. the folder above --- README.md | 44 ++++++++++------- builtins.go | 6 +-- data.go | 2 +- node.go | 21 ++++---- reference.go | 132 +++++++++++++++++++++++++++++++++++++++++---------- render.go | 2 +- template.go | 40 +++++++++------- 7 files changed, 169 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 85857ff..2a939bf 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ work with no data on disk. A directory is a namespace: each JSON file is a category named after the file, each subdirectory a dot-path segment, so `mydata/sv_SE/person.json` is `sv_SE.person` and replaces the shipped one. Sources merge in order; matching folders combine, any other clash is won by the -last loaded. Names may not use `.`, `|`, `(`, `{` or `}`; dot-prefixed entries +last loaded. Names may not use `.`, `|`, `(`, `{`, `}` or `/`; dot-prefixed entries are skipped, so a data directory can also be a checkout. Each locale carries `address`, `color`, `company`, `date`, `email`, `ip`, @@ -141,7 +141,7 @@ Every character is literal except a `{…}` token: | `{name.field}` | `field` of one draw of `name` ([Correlated fields](#correlated-fields)) | | `{a\|b}` | one of the named fields, even odds | | `{fn(args)}` | a builtin ([Functions](#functions)) | -| `{..path}` | a node reached from the data root ([References](#references)) | +| `{/path}`, `{.path}`, `{..path}` | a node reached from the data root, this file's folder, or the folder above ([References](#references)) | | `{{`, `}}` | a literal `{` or `}` | ```json @@ -267,21 +267,25 @@ a mix. ### References -`{..path}` renders a node from the **data root** — the path `Fake` takes, across -every loaded source — so one category borrows another, even across folders or -layered directories: +A reference renders a node from elsewhere in the data — the path `Fake` takes, +across every loaded source — so one category borrows another. The sigil says +where the path starts, as in a filesystem: `{/en_US.person}` from the data root, +`{.username}` from the folder this file sits in, `{..username}` from the folder +above. `data/sv_SE/email.json` can therefore read its own locale's `username` +without naming `sv_SE`: ```json -"Hej, {..en_US.person}!" +"Hej, {/en_US.person}!" ``` Renders e.g. `Hej, Pat Smith!`. A reference into a category is held like a -[correlated](#correlated-fields) path — `{..sv_SE.person.first} {..sv_SE.person.last}` -name one person, `{lowercase(..sv_SE.person.first)}` reads that same draw — while -a bare `{..misc.uuid} {..misc.uuid}` is two draws. Rejected at `New`: a path that -is unknown, names a folder, or reads a field not every variant of a choice -carries, and a reference that leads back to its own value, directly, mutually or -through a chain. +[correlated](#correlated-fields) path — `{.person.first} {.person.last}` name one +person, `{lowercase(.person.first)}` reads that same draw, and `{.person.first}` +beside `{/sv_SE.person.last}` in `sv_SE` is one person too — while a bare +`{/misc.uuid} {/misc.uuid}` is two draws. Rejected at `New`: a path that is +unknown, names a folder, has no folder above, or reads a field not every variant +of a choice carries, and a reference that leads back to its own value, directly, +mutually or through a chain. ### Correlated fields @@ -315,8 +319,8 @@ The sub-fields stay addressable — `Fake("address.place.locality")` renders, an A name any token reads as a path (`{p.first}`) or as an operand (`{calc(net * 2)}`, `{uppercase(w)}`) is drawn **once per expansion**, and every other route to it — -a bare `{p}`, a second bare `{w}`, `{..cat.net}`, a nested template rendering -`{..cat.p.last}`, at any depth — is a load error naming the spelling to use. A +a bare `{p}`, a second bare `{w}`, `{/cat.net}`, a nested template rendering +`{/cat.p.last}`, at any depth — is a load error naming the spelling to use. A name nothing reads that way is drawn each time: `{word} {word}` differs. An expansion is one render of one format, so each `repeat` iteration and each nested template draws again. @@ -351,11 +355,15 @@ tokens add cost in proportion to the output. - **The shipped data is embedded, not discovered.** A directory a machine happens to have would make `--seed 42` machine-dependent. Data still lives in `data/` as JSON; `--data-path` layers over it. -- **A bare reference draws each time; a reference path is held.** `{..p} {..p}` - is two draws, as `{word} {word}` is, while `{..p.first}` beside a nested template - rendering `{..p.first}` is a load error: a bare token is by contract an +- **A bare reference draws each time; a reference path is held.** `{/p} {/p}` + is two draws, as `{word} {word}` is, while `{/p.first}` beside a nested template + rendering `{/p.first}` is a load error: a bare token is by contract an independent draw, a path pins its level, and any route into a pinned level from another expansion could show another row. +- **Reference sigils follow the filesystem.** `/` is the root, `.` this file's + folder, `..` the folder above — what those spellings already mean to anyone who + has typed a path. A locale's files reach each other without naming the locale, + so a folder renames and copies without editing its references. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. @@ -398,7 +406,7 @@ fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation render.go Fake and the recursive renderer (choices, format strings, paths, held draws) template.go the {token} grammar: scanning, arms, operands, validation -reference.go {..path} binding across the tree, the render graph, and the walks over it +reference.go {/path} binding across the tree, the render graph, and the walks over it builtins.go the {name()} function registry and its implementations calc.go the {calc()} arithmetic evaluator: parser, eval, validation data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge diff --git a/builtins.go b/builtins.go index ae025ac..dfb6ce6 100644 --- a/builtins.go +++ b/builtins.go @@ -129,10 +129,8 @@ func transformArg(fields map[string]node, a []string) error { return err } if isRef(leaf) { - if leaf == refPrefix { - return fmt.Errorf("reference has no path") - } - return nil + _, _, err := refShape(leaf) + return err } return checkArm(leaf, fields) } diff --git a/data.go b/data.go index bfc4a40..fe78367 100644 --- a/data.go +++ b/data.go @@ -31,7 +31,7 @@ func (s dataSource) name(p string) string { // keyed by its base name (address.json -> "address"); each subdirectory becomes a // nested group, so folders turn into dot-path segments. Sources merge left to right: // matching groups merge by their children, and any other clash is won by the last -// source loaded. Once merged, linkRefs binds every {..path} reference against the +// source loaded. Once merged, linkRefs binds every reference against the // final tree. func loadData(sources []dataSource) (map[string]node, error) { root := map[string]node{} diff --git a/node.go b/node.go index d6fa84f..84da64c 100644 --- a/node.go +++ b/node.go @@ -41,11 +41,11 @@ type template struct { fields map[string]node repeat int separator string - ops []op // format compiled once (see compileOps); what expand walks - grow int // minimum output size, to size the render buffer - fixed bool // no op varies, so every render is lit - lit string // the whole output when fixed - refs map[string]string // each {..path} the format reads -> the head it is bound under + ops []op // format compiled once (see compileOps); what expand walks + grow int // minimum output size, to size the render buffer + fixed bool // no op varies, so every render is lit + lit string // the whole output when fixed + refs map[string]refBinding // each reference the format reads -> what it is bound to // bound maps each field the format addresses by dotted path to one path token // reading it, which is the half of an overlap the fences name. nil when the // format takes no path. @@ -208,9 +208,6 @@ func compileTemplate(m map[string]any) (node, error) { if isOption(k) { continue } - if isRef(k) { - return nil, fmt.Errorf("field %q starts with %q, which is reserved for {..path} bindings", k, refPrefix) - } if err := checkName(k); err != nil { return nil, fmt.Errorf("field %w", err) } @@ -321,10 +318,10 @@ func weightOf(raw any) (float64, error) { // reservedInName is what a category, folder or field name may not contain: a dot // separates the segments of a path, '|' the arms of a token, '(' opens a function -// call and braces delimit the token. A name carrying one is reachable by no format, -// so it is rejected where it is authored rather than at the token that cannot -// reach it. -const reservedInName = ".|({}" +// call, braces delimit the token and '/' starts a reference. A name carrying one is +// reachable by no format, so it is rejected where it is authored rather than at +// the token that cannot reach it. +const reservedInName = ".|({}/" // reservedList spells reservedInName for an error message, so the two cannot drift. var reservedList = strings.Join(strings.Split(reservedInName, ""), " ") diff --git a/reference.go b/reference.go index b240665..0d59f23 100644 --- a/reference.go +++ b/reference.go @@ -6,26 +6,70 @@ import ( "strings" ) -// refPrefix marks a {..path} token: a reference to a node elsewhere in the data -// root rather than a sibling field. The path is resolved across every loaded -// directory (see linkRefs). -const refPrefix = ".." +// A reference names a node by path rather than as a sibling field: {/a.b} from the +// data root, {.a} from the folder this file sits in, {..a} from the folder above. +func isRef(name string) bool { return strings.HasPrefix(name, ".") || strings.HasPrefix(name, "/") } -func isRef(name string) bool { return strings.HasPrefix(name, refPrefix) } +// refBinding is what a reference was bound to: the head key its category is held +// under, and the tail read into it. +type refBinding struct { + key string + tail []string +} -// linkRefs resolves every {..path} reference in the assembled tree. The head of the -// path — up to the category it names — is bound into the referring template's -// fields, and the rest reads into it the way a sibling path does, so a reference -// is held like a sibling. It runs once, after all data is merged, so a reference -// sees the final (override-resolved) tree. A path that is unknown, names a folder, -// or reads a field not every variant carries fails here, keeping a bad reference a -// New-time error, never a random render-time one. -func linkRefs(root map[string]node) error { - return walkNodes(root, func(path string, n node) error { - t, ok := n.(*template) - if !ok { - return nil +// refShape splits a reference into its sigil and the dotted path after it. +func refShape(name string) (sigil, rest string, err error) { + switch { + case strings.HasPrefix(name, "/"): + sigil, rest = "/", name[1:] + case strings.HasPrefix(name, ".."): + sigil, rest = "..", name[2:] + default: + sigil, rest = ".", name[1:] + } + if rest == "" { + return "", "", fmt.Errorf("reference has no path") + } + if strings.HasPrefix(rest, ".") { + return "", "", fmt.Errorf("a reference starts with / (the root), . (this folder) or .. (the folder above)") + } + for _, seg := range strings.Split(rest, ".") { + if seg == "" { + return "", "", fmt.Errorf("path has an empty segment") } + } + return sigil, rest, nil +} + +// refSegments resolves a reference written in folder to a path from the root. +func refSegments(name string, folder []string) ([]string, error) { + sigil, rest, err := refShape(name) + if err != nil { + return nil, err + } + var base []string + switch sigil { + case ".": + base = folder + case "..": + if len(folder) == 0 { + return nil, fmt.Errorf("no folder above the root") + } + base = folder[:len(folder)-1] + } + return append(append([]string{}, base...), strings.Split(rest, ".")...), nil +} + +// linkRefs resolves every reference in the assembled tree. The head of the path — +// up to the category it names — is bound into the referring template's fields +// under its root path, and the rest reads into it the way a sibling path does, so +// a reference is held like a sibling and two spellings of one target are one +// draw. It runs once, after all data is merged, so a reference sees the final +// (override-resolved) tree. A path that is unknown, names a folder, or reads a +// field not every variant carries fails here, keeping a bad reference a New-time +// error, never a random render-time one. +func linkRefs(root map[string]node) error { + return eachTemplate(root, func(folder []string, path string, t *template) error { names := refTokens(t.format) if len(names) == 0 { return nil @@ -33,18 +77,22 @@ func linkRefs(root map[string]node) error { if t.fields == nil { t.fields = map[string]node{} } - t.refs = make(map[string]string, len(names)) + t.refs = make(map[string]refBinding, len(names)) for _, name := range names { - head, target, tail, err := resolveRef(root, strings.Split(name[len(refPrefix):], ".")) + segments, err := refSegments(name, folder) if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } - key := refPrefix + strings.Join(head, ".") + head, target, tail, err := resolveRef(root, segments) + if err != nil { + return fmt.Errorf("%s: reference {%s}: %w", path, name, err) + } + key := "/" + strings.Join(head, ".") if err := checkPath(target, tail, key); err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } t.fields[key] = target - t.refs[name] = key + t.refs[name] = refBinding{key, tail} } if err := t.compileFormat(); err != nil { return fmt.Errorf("%s: %w", path, err) @@ -53,6 +101,42 @@ func linkRefs(root map[string]node) error { }) } +// eachTemplate calls fn once per template, with the folder its category sits in +// and the dot path reaching it, folders and names in sorted order. +func eachTemplate(root map[string]node, fn func(folder []string, path string, t *template) error) error { + var inCategory func(folder []string, path string, n node) error + inCategory = func(folder []string, path string, n node) error { + if t, ok := n.(*template); ok { + if err := fn(folder, path, t); err != nil { + return err + } + } + for _, c := range contained(n) { + if err := inCategory(folder, join(path, c.name), c.node); err != nil { + return err + } + } + return nil + } + var inFolder func(folder []string, children map[string]node) error + inFolder = func(folder []string, children map[string]node) error { + for _, name := range sortedNames(children) { + path := join(strings.Join(folder, "."), name) + if g, ok := children[name].(*group); ok { + if err := inFolder(append(folder[:len(folder):len(folder)], name), g.children); err != nil { + return err + } + continue + } + if err := inCategory(folder, path, children[name]); err != nil { + return err + } + } + return nil + } + return inFolder(nil, root) +} + // checkBoundLevelsHeld rejects every route to a held name except the ones that read // its draw. An expansion holds one draw of that name; anything else that renders it // draws again, and the two disagree. checkNoOverlap settles the spellings within one @@ -173,7 +257,7 @@ func cover(n node, into map[node]bool, inChoice bool) { // whole, so that draw fixes every value the render produced, and a second route to // any of them disagrees with it. // -// The walk stops at a {..path} edge, which is where the operand's own value ends +// The walk stops at a reference edge, which is where the operand's own value ends // and a shared source begins: two names referencing one category are two draws, the // same rule {word} {word} follows. func operandDraw(n node, into map[node]bool) { @@ -271,7 +355,7 @@ func contained(n node) []namedNode { } } -// named skips a bound {..path} key: it is a render edge, not containment, so using +// named skips a bound {/path} key: it is a render edge, not containment, so using // it as a path segment would report a node under a path that does not reach it. Only // a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a // group's children never carry the prefix — so this one skip serves both. @@ -317,7 +401,7 @@ func resolveRef(root map[string]node, segments []string) (head []string, target return segments[:i], n, segments[i:], nil } -// refTokens returns the {..path} names a format reads, as tokens or as operands. +// refTokens returns the reference names a format reads, as tokens or as operands. func refTokens(format string) []string { var refs []string seen := map[string]bool{} diff --git a/render.go b/render.go index df5391a..937874e 100644 --- a/render.go +++ b/render.go @@ -162,7 +162,7 @@ type draws struct { } // readField renders one arm of a token. An arm's key is a sibling field or a -// {..path} reference, which linkRefs bound into fields too. A name the expansion +// {/path} reference, which linkRefs bound into fields too. A name the expansion // holds — a level some token addresses by dotted path, or a sibling a {calc()} // reads — is drawn once and kept, so {place.postal-code} and {place.locality} read // one row, either read twice gives one value, and a shown operand is the operand diff --git a/template.go b/template.go index 9c2e3a9..bb337cd 100644 --- a/template.go +++ b/template.go @@ -135,10 +135,10 @@ func checkTokens(format string, fields map[string]node) error { names := strings.Split(t.body, "|") for _, name := range names { if isRef(name) { - if name == refPrefix { - return fmt.Errorf("token {%s}: reference has no path", t.body) + if _, _, err := refShape(name); err != nil { + return fmt.Errorf("token {%s}: %w", t.body, err) } - continue // a root reference; its target is checked at New (see linkRefs) + continue // its target is checked at New (see linkRefs) } if err := checkArm(name, fields); err != nil { return fmt.Errorf("token {%s}: %w", t.body, err) @@ -230,7 +230,7 @@ func fieldTokens(format string) []string { } // arm is one alternative of a {a|b} token or one operand, split into the key -// naming the node in a template's fields (a sibling field, or the "..path" head a +// naming the node in a template's fields (a sibling field, or the head a // reference is bound under) and the tail of a dotted path into it. A non-empty // tail is what makes the arm a bound draw: its head is drawn once per expansion // (see compileOps). @@ -241,15 +241,19 @@ type arm struct { steps []string // key per level passed through; the head and leaf hold their own } -// splitArm splits one name into key and tail. refs maps a reference to the head -// linkRefs bound it under; before linking, a reference is whole. -func splitArm(name string, refs map[string]string) arm { +// splitArm splits one name into key and tail. refs maps a reference to what +// linkRefs bound it to; before linking, a reference is whole. +func splitArm(name string, refs map[string]refBinding) arm { if isRef(name) { - key, bound := refs[name] - if !bound || key == name { - return arm{name: name, key: name} + b, bound := refs[name] + if !bound || len(b.tail) == 0 { + key := name + if bound { + key = b.key + } + return arm{name: name, key: key} } - return pathArm(name, key, strings.Split(name[len(key)+1:], ".")) + return pathArm(name, b.key, b.tail) } head, tail, dotted := strings.Cut(name, ".") if !dotted { @@ -271,7 +275,7 @@ func pathArm(name, key string, segs []string) arm { // level's held draw while rendering the level expands it afresh, so their values // would disagree. Names are compared in sorted order, so which pair is reported // does not depend on where the tokens sit. -func checkNoOverlap(format string, bound map[string]string, refs map[string]string) error { +func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error { names := boundReaders(format, bound, refs) // Stable over one format-order scan, so two readers of one name (a token and a // calc operand both naming "p") are reported as the format writes them. @@ -292,7 +296,7 @@ type reader struct{ name, label string } // boundReaders lists every way a format reaches a bound field, in the order the // format writes them. An operand renders its field, so it names a level exactly // as a token does; one scan finds both, which is what puts them in one order. -func boundReaders(format string, bound map[string]string, refs map[string]string) []reader { +func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader { var names []reader _ = eachToken(format, func(t ftoken) error { if t.kind != 'b' { @@ -336,7 +340,7 @@ func checkSegments(a arm) error { } // splitArms splits a token body's '|' alternatives. -func splitArms(body string, refs map[string]string) []arm { +func splitArms(body string, refs map[string]refBinding) []arm { parts := strings.Split(body, "|") arms := make([]arm, len(parts)) for i, p := range parts { @@ -396,7 +400,7 @@ func (c *formatOps) hold(a arm, label string) { } } -func (c *formatOps) function(body string, refs map[string]string) { +func (c *formatOps) function(body string, refs map[string]refBinding) { name, args, _ := funcCall(body) var operands []arm for _, operand := range tokenOperands(body) { @@ -407,7 +411,7 @@ func (c *formatOps) function(body string, refs map[string]string) { c.ops = append(c.ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) } -func (c *formatOps) field(body string, refs map[string]string) { +func (c *formatOps) field(body string, refs map[string]refBinding) { arms := splitArms(body, refs) for _, a := range arms { if len(a.tail) > 0 { @@ -419,7 +423,7 @@ func (c *formatOps) field(body string, refs map[string]string) { // compileOps compiles a format string. Call checkTokens first: it is what proves // the scan and every token are valid. -func compileOps(format string, refs map[string]string) formatOps { +func compileOps(format string, refs map[string]refBinding) formatOps { var c formatOps var lit strings.Builder flush := func() { @@ -450,7 +454,7 @@ func compileOps(format string, refs map[string]string) formatOps { // checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside // {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The // error names the single-token spelling. -func checkNoRepeatedRead(format string, c formatOps, refs map[string]string) error { +func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error { count := map[string]int{} return eachToken(format, func(t ftoken) error { if t.kind != 'b' { -- 2.52.0 From 7581587a198e96a93eb7f4b2287b18cce4f2e403 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:29:36 +0200 Subject: [PATCH 24/38] Tests for the path walk; test files named for the modules they exercise --- data_test.go | 37 ++++ edge_test.go | 385 ---------------------------------- bound_test.go => hold_test.go | 0 loading_test.go | 202 ++++++++++++++++++ path_test.go | 138 ++++++++++++ template_test.go | 36 ++++ 6 files changed, 413 insertions(+), 385 deletions(-) delete mode 100644 edge_test.go rename bound_test.go => hold_test.go (100%) create mode 100644 path_test.go diff --git a/data_test.go b/data_test.go index 7de80ec..7e0b6b6 100644 --- a/data_test.go +++ b/data_test.go @@ -264,3 +264,40 @@ func luhnValid(s string) bool { } return sum%10 == 0 } + +// swedishName matches one or more letter-words, optionally space/hyphen joined +// ("Storgatan", "Norra Promenaden", "von Flemming"). Used by the composition +// tests so shipped name lists can grow without re-enumerating them here. +var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`) + +func TestShippedStreetComposition(t *testing.T) { + // street is a choice of composed {first}{last} templates and literal names. + f := newGenerator(t, "data/sv_SE", WithSeed(5)) + for i := 0; i < 300; i++ { + if s := fake(t, f, "address.street"); !swedishName.MatchString(s) { + t.Fatalf("street %q is not a Swedish street name", s) + } + } +} + +func TestShippedLastNameComposition(t *testing.T) { + // last is a choice of patronymic {first}sson templates, compound + // {first}{last} templates and literal surnames. + f := newGenerator(t, "data/sv_SE", WithSeed(6)) + for i := 0; i < 300; i++ { + if s := fake(t, f, "person.last"); !swedishName.MatchString(s) { + t.Fatalf("last name %q is not a Swedish surname", s) + } + } +} + +func TestShippedStreetNumberFormats(t *testing.T) { + // Reachable via a hyphenated path; covers all five weighted number variants. + f := newGenerator(t, "data/sv_SE", WithSeed(8)) + re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`) + for i := 0; i < 300; i++ { + if n := fake(t, f, "address.street-number"); !re.MatchString(n) { + t.Fatalf("street-number %q does not match %s", n, re) + } + } +} diff --git a/edge_test.go b/edge_test.go deleted file mode 100644 index 6dd2c1c..0000000 --- a/edge_test.go +++ /dev/null @@ -1,385 +0,0 @@ -package fejkdata - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - "testing" -) - -// --- format string edge cases --- - -func TestHashIsLiteral(t *testing.T) { - f := engine(1) - cases := map[string]string{ - `{"format":"","x":"v"}`: "", - `{"format":"#","x":"v"}`: "#", - `{"format":"##","x":"v"}`: "##", - `{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a", - `{"format":"#{x}","x":"v"}`: "#v", - } - for tmpl, want := range cases { - if got := mustRender(t, f, tmpl); got != want { - t.Errorf("render(%s) = %q, want %q", tmpl, got, want) - } - } -} - -func TestMultibyteFormat(t *testing.T) { - // Scanning is rune-aware: multibyte literals coexist with class chars and - // tokens without corrupting indices. - got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`) - if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { - t.Fatalf("multibyte format = %q", got) - } -} - -func TestAlternationThreeWay(t *testing.T) { - f := engine(4) - seen := map[string]bool{} - for i := 0; i < 200; i++ { - seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true - } - if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 { - t.Fatalf("3-way alternation produced %v, want A, B and C", seen) - } -} - -func TestNewErrors(t *testing.T) { - // Pointing New at a file (not a directory) fails. - file := filepath.Join(t.TempDir(), "xx_XX") - if err := os.WriteFile(file, []byte("{}"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := New(WithoutShippedData(), WithDataPath(file)); err == nil { - t.Error("New(file) = nil error, want not-a-directory error") - } - // Invalid JSON in a category file fails. - if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"broken": `{ not json`}))); err == nil { - t.Error("New(invalid JSON) = nil error") - } - // An option that cannot take effect, and a category or folder no dot path can - // reach, are mistakes New must name rather than accept and ignore. - rejected := map[string]struct { - files map[string]string - want string - }{ - "separator without repeat": { - map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, - "has no effect without a repeat above 1", - }, - "separator with an explicit repeat of 1": { - map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, - "has no effect without a repeat above 1", - }, - "weight outside a choice": { - map[string]string{"a": `{"format":"x","weight":5}`}, - "weight only skews a choice's items", - }, - "non-numeric weight outside a choice": { - map[string]string{"a": `{"format":"x","weight":"bad"}`}, - "weight only skews a choice's items", - }, - "weight used as a field": { - map[string]string{"a": `{"format":"{name} {weight}kg","name":"Anvil","weight":["7"]}`}, - "can never be a field", - }, - "an option name used as a token": { - map[string]string{"a": `"{weight}"`}, - `"weight" is an option and can never be a field`, - }, - "category name with a dot": { - map[string]string{"a.b": `"1"`}, - `category "a.b" contains "."`, - }, - "folder name with a dot": { - map[string]string{"a.b/cat": `"1"`}, - `/a.b: folder "a.b" contains "."`, - }, - // The token grammar reserves three more characters. A name carrying one - // still resolves by dot path, but no format can name it, so it is rejected - // where it is authored rather than at the token that cannot reach it. - "field name with a pipe": { - map[string]string{"a": `{"format":"{x}","x":"1","b|c":"2"}`}, - `field "b|c" contains "|"`, - }, - "field name with a paren": { - map[string]string{"a": `{"format":"{x}","x":"1","b(c":"2"}`}, - `field "b(c" contains "("`, - }, - "field name with a closing brace": { - map[string]string{"a": `{"format":"{x}","x":"1","b}c":"2"}`}, - `field "b}c" contains "}"`, - }, - "category name with a pipe": { - map[string]string{"a|b": `"1"`}, - `category "a|b" contains "|"`, - }, - // An empty name is not a path segment, so List never offered it — while a - // bare {}, a trailing dot in Fake("a.") and a {/a.} reference all reached - // it. The engine accepted spellings it would never advertise. - "empty field name": { - map[string]string{"a": `{"format":"[{}]","":"VALUE"}`}, - `field "" is empty`, - }, - "folder name with a paren": { - map[string]string{"a(b/cat": `"1"`}, - `folder "a(b" contains "("`, - }, - // A repeated arm skews an alternation, which weight is the spelling for. - "repeated alternation arm": { - map[string]string{"a": `{"format":"{x|x}","x":"1"}`}, - `arm "x" is repeated`, - }, - "repeated arm among others": { - map[string]string{"a": `{"format":"{x|y|x}","x":"1","y":"2"}`}, - `arm "x" is repeated`, - }, - "repeated path arm": { - map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":"1"}}`}, - `arm "p.v" is repeated`, - }, - // A reference arm is the only kind that reaches the repeat check by passing - // the per-arm checks rather than falling through them. - "repeated reference arm": { - map[string]string{"a": `"x"`, "b": `"{/a|/a}"`}, - `arm "/a" is repeated`, - }, - // An arm that is broken on its own terms is reported as that, not as a - // repeat: the repeat is a consequence of the real mistake. - "repeated arm with no path": { - map[string]string{"a": `"{/|/}"`}, - "reference has no path", - }, - // No field can be named "", so the token is told that rather than sent to - // name one — the fix "no field" points at is itself a load error. - "repeated empty arm": { - map[string]string{"a": `"{|}"`}, - "a name is never empty", - }, - "bare empty token": { - map[string]string{"a": `{"format":"[{}]","x":"1"}`}, - "a name is never empty", - }, - } - for name, c := range rejected { - _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) - if err == nil { - t.Errorf("%s: New = nil error, want it rejected at load", name) - continue - } - if !strings.Contains(err.Error(), c.want) { - t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) - } - } - // Data that works today must keep working, and stay reachable. - accepted := map[string]struct { - files map[string]string - path string - want string - }{ - "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":"Anvil","Weight":"7"}`}, "a", "Anvil 7kg"}, - "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":"PDF"}`}, "a", "PDF"}, - "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":"1"}`}, "a.x-y", "1"}, - "category named Format": {map[string]string{"Format": `"1"`}, "Format", "1"}, - "folder named Repeat": {map[string]string{"Repeat/cat": `"1"`}, "Repeat.cat", "1"}, - "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":"2"}`}, "a", "2"}, - "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":"1"}`}, "a", "111"}, - // One name in two separate tokens is two independent draws, not a repeated - // arm; only a repeat within one alternation is rejected. - "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":"1"}`}, "a", "11"}, - } - for name, c := range accepted { - f, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) - if err != nil { - t.Errorf("%s: New = %v, want it accepted", name, err) - continue - } - if got, err := f.Fake(c.path); err != nil || got != c.want { - t.Errorf("%s: Fake(%q) = %q, %v, want %q", name, c.path, got, err, c.want) - } - } -} - -// --- deep path navigation --- - -func TestDeepDottedPath(t *testing.T) { - // A 5-segment path descends through alternating object/array nodes; choices - // on the path are single-variant, so it resolves deterministically. - f := engine(1) - f.categories = map[string]node{ - "deep": compiled(t, `{"format":"{a}","a":{"format":"{b}","b":{"format":"{c}","c":{"format":"{d}","d":"leaf"}}}}`), - } - if got, err := f.Fake("deep.a.b.c.d"); err != nil || got != "leaf" { - t.Fatalf("Fake(deep.a.b.c.d) = %q, %v, want leaf", got, err) - } - // Rendering the whole tree resolves the same chain. - if got, err := f.Fake("deep"); err != nil || got != "leaf" { - t.Fatalf("Fake(deep) = %q, %v, want leaf", got, err) - } -} - -func TestDescendIntoStringErrors(t *testing.T) { - f := engine(1) - f.categories = map[string]node{"greeting": compiled(t, `"hej"`)} - if _, err := f.Fake("greeting.extra"); err == nil || !strings.Contains(err.Error(), `no field "extra"`) { - t.Fatalf("Fake(greeting.extra) = %v, want a no-field error", err) - } -} - -// TestPathThroughChoice pins the rule that keeps a dotted path from rendering on -// one call and failing on the next: every variant must carry the rest of the path. -func TestPathThroughChoice(t *testing.T) { - dir := writeData(t, map[string]string{ - "every": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, - "notall": `[{"format":"{f}","f":"1"},"plain"]`, - "some": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2","extra":"x"}]`, - }) - f := newGenerator(t, dir, WithSeed(1)) - for i := 0; i < 200; i++ { - if got := fake(t, f, "every.f"); got != "1" && got != "2" { - t.Fatalf("every.f = %q, want 1 or 2", got) - } - } - // A path only some variants carry is reported against what all of them carry. - if _, err := f.Fake("some.extra"); err == nil || !strings.Contains(err.Error(), "all carry [f]") { - t.Errorf("Fake(some.extra) = %v, want it to name what every variant carries", err) - } - var first string - for i := 0; i < 200; i++ { - _, err := f.Fake("notall.f") - if err == nil { - t.Fatal("notall.f = nil error, want the same failure every call") - } - if i == 0 { - first = err.Error() - } else if err.Error() != first { - t.Fatalf("notall.f error varies between calls:\n %s\n %s", first, err.Error()) - } - } -} - -// TestPathKeyIsUnambiguous pins that the one shape which could collide cannot be -// written: a field literally named "a.b" and a field "a" holding "b" would both -// spell "a.b", so a dotted field name is rejected at New and a dot means a path -// wherever it appears. -func TestPathKeyIsUnambiguous(t *testing.T) { - dir := writeData(t, map[string]string{ - "cat": `[{"format":"{a.b}","a.b":"1"},{"format":"{a}","a":{"format":"{b}","b":"2"}}]`, - }) - _, err := New(WithoutShippedData(), WithDataPath(dir)) - if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { - t.Fatalf("New = %v, want the dotted field name rejected", err) - } - // The same data without the dotted key is fine, and the path resolves. - f := newGenerator(t, writeData(t, map[string]string{ - "cat": `{"format":"{a.b}","a":{"format":"{b}","b":"2"}}`, - }), WithSeed(1)) - if !slices.Contains(f.List(), "cat.a.b") { - t.Error("List() omits cat.a.b, which the data carries") - } - if got := fake(t, f, "cat"); got != "2" { - t.Fatalf("cat = %q, want 2", got) - } -} - -// TestMissingFieldNamesItself keeps the precise diagnosis for the ordinary typo: a -// single-variant choice always picks the same item, so it needs no every-variant -// guard and the error can name the field that is missing. -func TestMissingFieldNamesItself(t *testing.T) { - f := newGenerator(t, "data/sv_SE", WithSeed(1)) - _, err := f.Fake("person.typo") - if err == nil || !strings.Contains(err.Error(), `no field "typo"`) { - t.Errorf("Fake(person.typo) = %v, want it to name the missing field", err) - } -} - -// --- category root shapes --- - -func TestCategoryRootShapes(t *testing.T) { - dir := writeData(t, map[string]string{ - "obj": `"{digits(2)}"`, // object root - "lit": `"hello"`, // bare-string root - }) - f := newGenerator(t, dir, WithSeed(1)) - if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { - t.Errorf("object-root category = %q, want two digits", got) - } - if got := fake(t, f, "lit"); got != "hello" { - t.Errorf("string-root category = %q, want hello", got) - } -} - -// --- very long lists --- - -func TestLongStringList(t *testing.T) { - const n = 2000 - names := make([]string, n) - for i := range names { - names[i] = fmt.Sprintf("name-%04d", i) - } - list, err := json.Marshal(names) - if err != nil { - t.Fatal(err) - } - f := newGenerator(t, writeData(t, map[string]string{"name": string(list)}), WithSeed(1)) - - valid := map[string]bool{} - for _, v := range names { - valid[v] = true - } - seen := map[string]bool{} - for i := 0; i < 20000; i++ { - v := fake(t, f, "name") - if !valid[v] { - t.Fatalf("got %q, not in the list", v) - } - seen[v] = true - } - if len(seen) < n*8/10 { - t.Fatalf("only %d/%d distinct values seen; selection looks skewed", len(seen), n) - } -} - -// --- composition against the shipped sv_SE data --- - -// swedishName matches one or more letter-words, optionally space/hyphen joined -// ("Storgatan", "Norra Promenaden", "von Flemming"). Used by the composition -// tests so shipped name lists can grow without re-enumerating them here. -var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`) - -func TestShippedStreetComposition(t *testing.T) { - // street is a choice of composed {first}{last} templates and literal names. - f := newGenerator(t, "data/sv_SE", WithSeed(5)) - for i := 0; i < 300; i++ { - if s := fake(t, f, "address.street"); !swedishName.MatchString(s) { - t.Fatalf("street %q is not a Swedish street name", s) - } - } -} - -func TestShippedLastNameComposition(t *testing.T) { - // last is a choice of patronymic {first}sson templates, compound - // {first}{last} templates and literal surnames. - f := newGenerator(t, "data/sv_SE", WithSeed(6)) - for i := 0; i < 300; i++ { - if s := fake(t, f, "person.last"); !swedishName.MatchString(s) { - t.Fatalf("last name %q is not a Swedish surname", s) - } - } -} - -func TestShippedStreetNumberFormats(t *testing.T) { - // Reachable via a hyphenated path; covers all five weighted number variants. - f := newGenerator(t, "data/sv_SE", WithSeed(8)) - re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`) - for i := 0; i < 300; i++ { - if n := fake(t, f, "address.street-number"); !re.MatchString(n) { - t.Fatalf("street-number %q does not match %s", n, re) - } - } -} diff --git a/bound_test.go b/hold_test.go similarity index 100% rename from bound_test.go rename to hold_test.go diff --git a/loading_test.go b/loading_test.go index b590b6c..be407ed 100644 --- a/loading_test.go +++ b/loading_test.go @@ -1,9 +1,12 @@ package fejkdata import ( + "encoding/json" + "fmt" "os" "path/filepath" "reflect" + "regexp" "slices" "strings" "testing" @@ -201,3 +204,202 @@ func TestRepeatProductAlongAPathIsCapped(t *testing.T) { t.Errorf("New = %v, want 1024 x 1024 along one path accepted", err) } } + +func TestNewErrors(t *testing.T) { + // Pointing New at a file (not a directory) fails. + file := filepath.Join(t.TempDir(), "xx_XX") + if err := os.WriteFile(file, []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := New(WithoutShippedData(), WithDataPath(file)); err == nil { + t.Error("New(file) = nil error, want not-a-directory error") + } + // Invalid JSON in a category file fails. + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"broken": `{ not json`}))); err == nil { + t.Error("New(invalid JSON) = nil error") + } + // An option that cannot take effect, and a category or folder no dot path can + // reach, are mistakes New must name rather than accept and ignore. + rejected := map[string]struct { + files map[string]string + want string + }{ + "separator without repeat": { + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, + "has no effect without a repeat above 1", + }, + "separator with an explicit repeat of 1": { + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, + "has no effect without a repeat above 1", + }, + "weight outside a choice": { + map[string]string{"a": `{"format":"x","weight":5}`}, + "weight only skews a choice's items", + }, + "non-numeric weight outside a choice": { + map[string]string{"a": `{"format":"x","weight":"bad"}`}, + "weight only skews a choice's items", + }, + "weight used as a field": { + map[string]string{"a": `{"format":"{name} {weight}kg","name":"Anvil","weight":["7"]}`}, + "can never be a field", + }, + "an option name used as a token": { + map[string]string{"a": `"{weight}"`}, + `"weight" is an option and can never be a field`, + }, + "category name with a dot": { + map[string]string{"a.b": `"1"`}, + `category "a.b" contains "."`, + }, + "folder name with a dot": { + map[string]string{"a.b/cat": `"1"`}, + `/a.b: folder "a.b" contains "."`, + }, + // The token grammar reserves three more characters. A name carrying one + // still resolves by dot path, but no format can name it, so it is rejected + // where it is authored rather than at the token that cannot reach it. + "field name with a pipe": { + map[string]string{"a": `{"format":"{x}","x":"1","b|c":"2"}`}, + `field "b|c" contains "|"`, + }, + "field name with a paren": { + map[string]string{"a": `{"format":"{x}","x":"1","b(c":"2"}`}, + `field "b(c" contains "("`, + }, + "field name with a closing brace": { + map[string]string{"a": `{"format":"{x}","x":"1","b}c":"2"}`}, + `field "b}c" contains "}"`, + }, + "category name with a pipe": { + map[string]string{"a|b": `"1"`}, + `category "a|b" contains "|"`, + }, + // An empty name is not a path segment, so List never offered it — while a + // bare {}, a trailing dot in Fake("a.") and a {/a.} reference all reached + // it. The engine accepted spellings it would never advertise. + "empty field name": { + map[string]string{"a": `{"format":"[{}]","":"VALUE"}`}, + `field "" is empty`, + }, + "folder name with a paren": { + map[string]string{"a(b/cat": `"1"`}, + `folder "a(b" contains "("`, + }, + // A repeated arm skews an alternation, which weight is the spelling for. + "repeated alternation arm": { + map[string]string{"a": `{"format":"{x|x}","x":"1"}`}, + `arm "x" is repeated`, + }, + "repeated arm among others": { + map[string]string{"a": `{"format":"{x|y|x}","x":"1","y":"2"}`}, + `arm "x" is repeated`, + }, + "repeated path arm": { + map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":"1"}}`}, + `arm "p.v" is repeated`, + }, + // A reference arm is the only kind that reaches the repeat check by passing + // the per-arm checks rather than falling through them. + "repeated reference arm": { + map[string]string{"a": `"x"`, "b": `"{/a|/a}"`}, + `arm "/a" is repeated`, + }, + // An arm that is broken on its own terms is reported as that, not as a + // repeat: the repeat is a consequence of the real mistake. + "repeated arm with no path": { + map[string]string{"a": `"{/|/}"`}, + "reference has no path", + }, + // No field can be named "", so the token is told that rather than sent to + // name one — the fix "no field" points at is itself a load error. + "repeated empty arm": { + map[string]string{"a": `"{|}"`}, + "a name is never empty", + }, + "bare empty token": { + map[string]string{"a": `{"format":"[{}]","x":"1"}`}, + "a name is never empty", + }, + } + for name, c := range rejected { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) + if err == nil { + t.Errorf("%s: New = nil error, want it rejected at load", name) + continue + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) + } + } + // Data that works today must keep working, and stay reachable. + accepted := map[string]struct { + files map[string]string + path string + want string + }{ + "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":"Anvil","Weight":"7"}`}, "a", "Anvil 7kg"}, + "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":"PDF"}`}, "a", "PDF"}, + "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":"1"}`}, "a.x-y", "1"}, + "category named Format": {map[string]string{"Format": `"1"`}, "Format", "1"}, + "folder named Repeat": {map[string]string{"Repeat/cat": `"1"`}, "Repeat.cat", "1"}, + "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":"2"}`}, "a", "2"}, + "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":"1"}`}, "a", "111"}, + // One name in two separate tokens is two independent draws, not a repeated + // arm; only a repeat within one alternation is rejected. + "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":"1"}`}, "a", "11"}, + } + for name, c := range accepted { + f, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) + if err != nil { + t.Errorf("%s: New = %v, want it accepted", name, err) + continue + } + if got, err := f.Fake(c.path); err != nil || got != c.want { + t.Errorf("%s: Fake(%q) = %q, %v, want %q", name, c.path, got, err, c.want) + } + } +} + +func TestCategoryRootShapes(t *testing.T) { + dir := writeData(t, map[string]string{ + "obj": `"{digits(2)}"`, // object root + "lit": `"hello"`, // bare-string root + }) + f := newGenerator(t, dir, WithSeed(1)) + if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { + t.Errorf("object-root category = %q, want two digits", got) + } + if got := fake(t, f, "lit"); got != "hello" { + t.Errorf("string-root category = %q, want hello", got) + } +} + +func TestLongStringList(t *testing.T) { + const n = 2000 + names := make([]string, n) + for i := range names { + names[i] = fmt.Sprintf("name-%04d", i) + } + list, err := json.Marshal(names) + if err != nil { + t.Fatal(err) + } + f := newGenerator(t, writeData(t, map[string]string{"name": string(list)}), WithSeed(1)) + + valid := map[string]bool{} + for _, v := range names { + valid[v] = true + } + seen := map[string]bool{} + for i := 0; i < 20000; i++ { + v := fake(t, f, "name") + if !valid[v] { + t.Fatalf("got %q, not in the list", v) + } + seen[v] = true + } + if len(seen) < n*8/10 { + t.Fatalf("only %d/%d distinct values seen; selection looks skewed", len(seen), n) + } +} diff --git a/path_test.go b/path_test.go new file mode 100644 index 0000000..57a385f --- /dev/null +++ b/path_test.go @@ -0,0 +1,138 @@ +package fejkdata + +import ( + "slices" + "strings" + "testing" +) + +func TestWalkPathStopsAtAMissingSegment(t *testing.T) { + n := compiled(t, `{"format":"{a}","a":{"format":"{b}","b":"leaf"}}`) + var seen []string + walk := pathWalk{ + level: func(tm *template, rest []string) error { seen = append(seen, "level:"+rest[0]); return nil }, + leaf: func(n node) error { seen = append(seen, "leaf"); return nil }, + } + if err := walkPath(n, []string{"a", "b"}, walk); err != nil { + t.Fatalf("walkPath(a.b) = %v", err) + } + if want := []string{"level:a", "level:b", "leaf"}; !slices.Equal(seen, want) { + t.Errorf("walk visited %v, want %v", seen, want) + } + seen = nil + err := walkPath(n, []string{"a", "nope", "deeper"}, walk) + if err == nil || !strings.Contains(err.Error(), `no field "nope"`) { + t.Errorf("walkPath(a.nope.deeper) = %v, want the missing segment named", err) + } + if slices.Contains(seen, "leaf") { + t.Errorf("walk reached a leaf past a missing segment: %v", seen) + } +} + +func TestWalkPathChoiceConsumesNoSegment(t *testing.T) { + n := compiled(t, `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`) + var leaves []node + err := walkPath(n, []string{"f"}, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if len(rest) != 1 || rest[0] != "f" { + t.Errorf("choice saw rest %v, want [f]", rest) + } + return c.items, nil + }, + leaf: func(n node) error { leaves = append(leaves, n); return nil }, + }) + if err != nil || len(leaves) != 2 { + t.Fatalf("walkPath through a choice = %v, %d leaves, want both variants' f", err, len(leaves)) + } +} + +func TestDeepDottedPath(t *testing.T) { + // A 5-segment path descends through alternating object/array nodes; choices + // on the path are single-variant, so it resolves deterministically. + f := engine(1) + f.categories = map[string]node{ + "deep": compiled(t, `{"format":"{a}","a":{"format":"{b}","b":{"format":"{c}","c":{"format":"{d}","d":"leaf"}}}}`), + } + if got, err := f.Fake("deep.a.b.c.d"); err != nil || got != "leaf" { + t.Fatalf("Fake(deep.a.b.c.d) = %q, %v, want leaf", got, err) + } + // Rendering the whole tree resolves the same chain. + if got, err := f.Fake("deep"); err != nil || got != "leaf" { + t.Fatalf("Fake(deep) = %q, %v, want leaf", got, err) + } +} + +func TestDescendIntoStringErrors(t *testing.T) { + f := engine(1) + f.categories = map[string]node{"greeting": compiled(t, `"hej"`)} + if _, err := f.Fake("greeting.extra"); err == nil || !strings.Contains(err.Error(), `no field "extra"`) { + t.Fatalf("Fake(greeting.extra) = %v, want a no-field error", err) + } +} + +// TestPathThroughChoice pins the rule that keeps a dotted path from rendering on +// one call and failing on the next: every variant must carry the rest of the path. +func TestPathThroughChoice(t *testing.T) { + dir := writeData(t, map[string]string{ + "every": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, + "notall": `[{"format":"{f}","f":"1"},"plain"]`, + "some": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2","extra":"x"}]`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 200; i++ { + if got := fake(t, f, "every.f"); got != "1" && got != "2" { + t.Fatalf("every.f = %q, want 1 or 2", got) + } + } + // A path only some variants carry is reported against what all of them carry. + if _, err := f.Fake("some.extra"); err == nil || !strings.Contains(err.Error(), "all carry [f]") { + t.Errorf("Fake(some.extra) = %v, want it to name what every variant carries", err) + } + var first string + for i := 0; i < 200; i++ { + _, err := f.Fake("notall.f") + if err == nil { + t.Fatal("notall.f = nil error, want the same failure every call") + } + if i == 0 { + first = err.Error() + } else if err.Error() != first { + t.Fatalf("notall.f error varies between calls:\n %s\n %s", first, err.Error()) + } + } +} + +// TestPathKeyIsUnambiguous pins that the one shape which could collide cannot be +// written: a field literally named "a.b" and a field "a" holding "b" would both +// spell "a.b", so a dotted field name is rejected at New and a dot means a path +// wherever it appears. +func TestPathKeyIsUnambiguous(t *testing.T) { + dir := writeData(t, map[string]string{ + "cat": `[{"format":"{a.b}","a.b":"1"},{"format":"{a}","a":{"format":"{b}","b":"2"}}]`, + }) + _, err := New(WithoutShippedData(), WithDataPath(dir)) + if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { + t.Fatalf("New = %v, want the dotted field name rejected", err) + } + // The same data without the dotted key is fine, and the path resolves. + f := newGenerator(t, writeData(t, map[string]string{ + "cat": `{"format":"{a.b}","a":{"format":"{b}","b":"2"}}`, + }), WithSeed(1)) + if !slices.Contains(f.List(), "cat.a.b") { + t.Error("List() omits cat.a.b, which the data carries") + } + if got := fake(t, f, "cat"); got != "2" { + t.Fatalf("cat = %q, want 2", got) + } +} + +// TestMissingFieldNamesItself keeps the precise diagnosis for the ordinary typo: a +// single-variant choice always picks the same item, so it needs no every-variant +// guard and the error can name the field that is missing. +func TestMissingFieldNamesItself(t *testing.T) { + f := newGenerator(t, "data/sv_SE", WithSeed(1)) + _, err := f.Fake("person.typo") + if err == nil || !strings.Contains(err.Error(), `no field "typo"`) { + t.Errorf("Fake(person.typo) = %v, want it to name the missing field", err) + } +} diff --git a/template_test.go b/template_test.go index 1f7f8f4..f1ddcb8 100644 --- a/template_test.go +++ b/template_test.go @@ -311,3 +311,39 @@ func quote(s string) string { } return string(b) } + +func TestHashIsLiteral(t *testing.T) { + f := engine(1) + cases := map[string]string{ + `{"format":"","x":"v"}`: "", + `{"format":"#","x":"v"}`: "#", + `{"format":"##","x":"v"}`: "##", + `{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a", + `{"format":"#{x}","x":"v"}`: "#v", + } + for tmpl, want := range cases { + if got := mustRender(t, f, tmpl); got != want { + t.Errorf("render(%s) = %q, want %q", tmpl, got, want) + } + } +} + +func TestMultibyteFormat(t *testing.T) { + // Scanning is rune-aware: multibyte literals coexist with class chars and + // tokens without corrupting indices. + got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`) + if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { + t.Fatalf("multibyte format = %q", got) + } +} + +func TestAlternationThreeWay(t *testing.T) { + f := engine(4) + seen := map[string]bool{} + for i := 0; i < 200; i++ { + seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true + } + if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 { + t.Fatalf("3-way alternation produced %v, want A, B and C", seen) + } +} -- 2.52.0 From bcbf044290a31fab6093649fc6f4d5522271a769 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:29:45 +0200 Subject: [PATCH 25/38] One path walk under checkPath, descend, readField, pathLeaves and coverPath; split the hold and the render graph into their own files --- README.md | 23 +-- graph.go | 222 +++++++++++++++++++++++++++++ hold.go | 345 +++++++++++++++++++++++++++++++++++++++++++++ node.go | 41 +----- path.go | 109 +++++++++++++++ reference.go | 388 --------------------------------------------------- render.go | 129 ++--------------- template.go | 127 ----------------- 8 files changed, 705 insertions(+), 679 deletions(-) create mode 100644 graph.go create mode 100644 hold.go create mode 100644 path.go diff --git a/README.md b/README.md index 2a939bf..773d875 100644 --- a/README.md +++ b/README.md @@ -402,16 +402,19 @@ GO_VERSION=1.22.12 docker compose run --rm test # the same tests, without the im ## Layout ``` -fejkdata.go Generator, New, options, the embedded data set, List -node.go the node model and JSON -> node compilation -render.go Fake and the recursive renderer (choices, format strings, paths, held draws) -template.go the {token} grammar: scanning, arms, operands, validation -reference.go {/path} binding across the tree, the render graph, and the walks over it -builtins.go the {name()} function registry and its implementations -calc.go the {calc()} arithmetic evaluator: parser, eval, validation -data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge -cmd/fejkdata/ the fejkdata CLI -data/ shipped data (JSON), embedded at build: locale folders + a misc folder +fejkdata.go Generator, New, options, the embedded data set, List +node.go the node model and JSON -> node compilation +path.go the dotted-path walk, and proving a path resolves +render.go Fake and the recursive renderer (choices, format strings, expansions) +template.go the {token} grammar: scanning, tokens, operands, validation, compiling a format +hold.go the hold: one draw per expansion for paths and operands, and its fences +reference.go reference sigils, and binding references across the tree +graph.go the render graph: edges, cycles, the repeat bound, tree walks +builtins.go the {name()} function registry and its implementations +calc.go the {calc()} arithmetic evaluator: parser, eval, validation +data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge +cmd/fejkdata/ the fejkdata CLI +data/ shipped data (JSON), embedded at build: locale folders + a misc folder ``` ## License diff --git a/graph.go b/graph.go new file mode 100644 index 0000000..4e2d09a --- /dev/null +++ b/graph.go @@ -0,0 +1,222 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// walkNodes calls fn once per contained node, passing the dot path that reaches it, +// visiting keys in sorted order so which of several broken nodes gets reported does +// not depend on map iteration. +func walkNodes(root map[string]node, fn func(path string, n node) error) error { + seen := map[node]bool{} + var visit func(string, node) error + visit = func(path string, n node) error { + if n == nil || seen[n] { + return nil + } + seen[n] = true + if err := fn(path, n); err != nil { + return err + } + for _, c := range contained(n) { + if err := visit(join(path, c.name), c.node); err != nil { + return err + } + } + return nil + } + for _, name := range sortedNames(root) { + if err := visit(name, root[name]); err != nil { + return err + } + } + return nil +} + +// namedNode is a contained child and the segment reaching it; a choice's items carry +// no segment, matching how a dot path steps over a choice. +type namedNode struct { + name string + node node +} + +func contained(n node) []namedNode { + switch n := n.(type) { + case *group: + return named(n.children) + case *choice: + out := make([]namedNode, len(n.items)) + for i, it := range n.items { + out[i] = namedNode{node: it} + } + return out + case *template: + return named(n.fields) + default: + return nil + } +} + +// named skips a bound {/path} key: it is a render edge, not containment, so using +// it as a path segment would report a node under a path that does not reach it. Only +// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a +// group's children never carry the prefix — so this one skip serves both. +func named(m map[string]node) []namedNode { + out := make([]namedNode, 0, len(m)) + for _, name := range sortedNames(m) { + if isRef(name) { + continue + } + out = append(out, namedNode{name: name, node: m[name]}) + } + return out +} + +func sortedNames(m map[string]node) []string { + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// renderEdge is a child a node renders into, labelled by what reaches it (a field +// name, reference, or choice index) for a readable cycle report. operand names +// the builtin when the label is its operand rather than a token, so an error can +// name it the way the author wrote it. +type renderEdge struct { + to node + label string + operand string +} + +// reached names an edge as the author spelled it, the vocabulary boundReaders uses +// for the sibling fence. +func (e renderEdge) reached() string { + if e.operand != "" { + return fmt.Sprintf("%s operand %q", e.operand, e.label) + } + return "{" + e.label + "}" +} + +// renderEdges lists the children rendering n recurses into, mirroring expand: a +// choice's items, and a template's field/reference tokens plus its operands. A +// group renders nothing, so it has no edges. +func renderEdges(n node) []renderEdge { + switch n := n.(type) { + case *choice: + es := make([]renderEdge, len(n.items)) + for i, it := range n.items { + es[i] = renderEdge{to: it, label: fmt.Sprintf("[%d]", i)} + } + return es + case *template: + var es []renderEdge + add := func(name, operand string) { + a := splitArm(name, n.refs) + c, ok := n.fields[a.key] + if !ok { + return + } + for _, leaf := range pathLeaves(c, a.tail) { + es = append(es, renderEdge{leaf, name, operand}) + } + } + _ = eachToken(n.format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + add(operand, fn) + } + return nil + } + for _, name := range strings.Split(t.body, "|") { + add(name, "") + } + return nil + }) + return es + default: + return nil + } +} + +// pathLeaves lists what a token's dotted tail renders: a choice on the way +// contributes every variant, since any of them may be the one drawn. checkPath has +// already proved the tail resolves in every variant. +func pathLeaves(n node, tail []string) []node { + var out []node + _ = walkPath(n, tail, pathWalk{ + choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil }, + leaf: func(n node) error { out = append(out, n); return nil }, + }) + return out +} + +// checkRepeatReach bounds the renders a repeat multiplies to along any root-to-leaf +// path, so nested repeats cannot build what one repeat may not. It runs after +// checkNoCycles, whose guarantee is what lets the walk terminate. +func checkRepeatReach(root map[string]node) error { + reach := map[node]int{} + var of func(n node) int + of = func(n node) int { + if r, done := reach[n]; done { + return r + } + r := 1 + for _, e := range renderEdges(n) { + if c := of(e.to); c > r { + r = c + } + } + if t, ok := n.(*template); ok { + r *= t.repeat + } + reach[n] = r + return r + } + return walkNodes(root, func(path string, n node) error { + if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > maxLen { + return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), maxLen) + } + return nil + }) +} + +// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself +// — directly, mutually, or through a chain — never terminates, so it must fail at +// New rather than stack-overflow at render. It is a depth-first walk of the render +// graph (renderEdges); grey marks nodes on the current path so a back-edge to one +// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped. +// Every node is a root: a field its parent's format never renders is still reachable +// by dot path, so a cycle in one would otherwise reach render and be fatal there. +func checkNoCycles(root map[string]node) error { + const ( + grey = 1 + black = 2 + ) + color := map[node]int{} + var visit func(n node, path string) error + visit = func(n node, path string) error { + switch color[n] { + case grey: + return fmt.Errorf("reference cycle: %s", path) + case black: + return nil + } + color[n] = grey + for _, e := range renderEdges(n) { + if err := visit(e.to, path+" -> "+e.label); err != nil { + return err + } + } + color[n] = black + return nil + } + return walkNodes(root, func(path string, n node) error { return visit(n, path) }) +} diff --git a/hold.go b/hold.go new file mode 100644 index 0000000..965dce2 --- /dev/null +++ b/hold.go @@ -0,0 +1,345 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// checkBoundLevelsHeld rejects every route to a held name except the ones that read +// its draw. An expansion holds one draw of that name; anything else that renders it +// draws again, and the two disagree. checkNoOverlap settles the spellings within one +// format (a token, an operand); this settles the rest — a reference, whether it +// sits in that format or in anything the format renders, however deep. +// +// It runs after checkNoCycles, whose guarantee is what lets the walk terminate. +func checkBoundLevelsHeld(root map[string]node) error { + return walkNodes(root, func(path string, n node) error { + t, ok := n.(*template) + if !ok || len(t.held) == 0 { + return nil + } + heads := make([]string, 0, len(t.held)) + for head := range t.held { + heads = append(heads, head) + } + // Operand heads first, then paths, each in name order: a level read both + // ways is reported by the operand's fence, and which overlap is reported + // does not vary. + sort.Slice(heads, func(i, j int) bool { + _, pi := t.bound[heads[i]] + _, pj := t.bound[heads[j]] + if pi != pj { + return !pi + } + return heads[i] < heads[j] + }) + readers := boundReaders(t.format, t.bound, t.refs) + for _, head := range heads { + // What one draw answers for depends on how the draw is read: a path pins + // the levels it passes through and the leaf it lands on, an operand + // exactly the value its render produces. + held := map[node]bool{} + reader, isPath := t.bound[head] + if isPath { + for _, r := range readers { + if a := splitArm(r.name, t.refs); a.key == head { + coverPath(t.fields[head], a.tail, held) + } + } + } else { + operandDraw(t.fields[head], held) + } + if len(held) == 0 { + continue // an early out: a fixed head holds nothing to reach + } + // One seen set across the edges: a node that cannot reach the level + // cannot reach it by another route either, so it is walked once here. + seen := map[node]bool{} + for _, e := range renderEdges(t) { + if splitArm(e.label, t.refs).key == head { + continue // a token or operand reading this draw, the routes allowed + } + if renders(e.to, held, seen) { + if isPath { + return fmt.Errorf("%s: %s renders %q, which {%s} reads a path into; name the fields you want instead", path, e.reached(), head, reader) + } + return fmt.Errorf("%s: %s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", path, e.reached(), head, operandReader(t, head)) + } + } + } + return nil + }) +} + +// operandReader names the builtin whose operand holds head. +func operandReader(t *template, head string) string { + fn := "" + _ = eachToken(t.format, func(tok ftoken) error { + if tok.kind != 'b' || fn != "" { + return nil + } + if name, _, isFunc := funcCall(tok.body); isFunc { + for _, operand := range tokenOperands(tok.body) { + if splitArm(operand, t.refs).key == head { + fn = name + } + } + } + return nil + }) + return fn +} + +// coverPath collects what holding one path pins: every choice level the path +// passes through, whole, and the leaf it renders. +func coverPath(n node, tail []string, into map[node]bool) { + _ = walkPath(n, tail, pathWalk{ + choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil }, + leaf: func(n node) error { cover(n, into, false); return nil }, + }) +} + +// cover collects a level and everything contained in it. A fixed string outside a +// choice is left out — it cannot disagree with itself — but inside one each +// variant carries its own, so there it counts. +func cover(n node, into map[node]bool, inChoice bool) { + if isFixed(n) && !inChoice { + return + } + into[n] = true + _, isChoice := n.(*choice) + for _, c := range contained(n) { + cover(c.node, into, inChoice || isChoice) + } +} + +// operandDraw collects what one held draw of an operand answers for: the operand +// and what rendering it settles inside itself. The builtin renders its operand +// whole, so that draw fixes every value the render produced, and a second route to +// any of them disagrees with it. +// +// The walk stops at a reference edge, which is where the operand's own value ends +// and a shared source begins: two names referencing one category are two draws, the +// same rule {word} {word} follows. +func operandDraw(n node, into map[node]bool) { + if isFixed(n) { + return + } + if into[n] { + return + } + into[n] = true + for _, e := range renderEdges(n) { + if isRef(e.label) { + continue + } + operandDraw(e.to, into) + } +} + +// isFixed is a string that varies nothing: fixed text with no fields to read into. +func isFixed(n node) bool { + t, ok := n.(*template) + return ok && t.fixed && len(t.fields) == 0 +} + +// renders reports whether rendering n can reach anything in want, following the +// same edges expand does. seen keeps a node shared by several routes from being +// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk +// ends. +func renders(n node, want, seen map[node]bool) bool { + if want[n] { + return true + } + if seen[n] { + return false + } + seen[n] = true + for _, e := range renderEdges(n) { + if renders(e.to, want, seen) { + return true + } + } + return false +} + +// arm is one alternative of a {a|b} token or one operand, split into the key +// naming the node in a template's fields (a sibling field, or the head a +// reference is bound under) and the tail of a dotted path into it. A non-empty +// tail is what makes the arm a bound draw: its head is drawn once per expansion +// (see compileOps). +type arm struct { + name string // as written, and the key a bound draw's value is held under + key string + tail []string + steps []string // key per level passed through; the head and leaf hold their own +} + +// splitArm splits one name into key and tail. refs maps a reference to what +// linkRefs bound it to; before linking, a reference is whole. +func splitArm(name string, refs map[string]refBinding) arm { + if isRef(name) { + b, bound := refs[name] + if !bound || len(b.tail) == 0 { + key := name + if bound { + key = b.key + } + return arm{name: name, key: key} + } + return pathArm(name, b.key, b.tail) + } + head, tail, dotted := strings.Cut(name, ".") + if !dotted { + return arm{name: name, key: name} + } + return pathArm(name, head, strings.Split(tail, ".")) +} + +func pathArm(name, key string, segs []string) arm { + var steps []string + for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own + steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) + } + return arm{name: name, key: key, tail: segs, steps: steps} +} + +// checkNoOverlap rejects a format that both renders a level and reads a path into +// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The path reads the +// level's held draw while rendering the level expands it afresh, so their values +// would disagree. Names are compared in sorted order, so which pair is reported +// does not depend on where the tokens sit. +func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error { + names := boundReaders(format, bound, refs) + // Stable over one format-order scan, so two readers of one name (a token and a + // calc operand both naming "p") are reported as the format writes them. + sort.SliceStable(names, func(i, j int) bool { return names[i].name < names[j].name }) + for i, level := range names { + for _, path := range names[i+1:] { + if strings.HasPrefix(path.name, level.name+".") { + return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name) + } + } + } + return nil +} + +// reader is one way a format reaches a bound field, and how to name that spelling. +type reader struct{ name, label string } + +// boundReaders lists every way a format reaches a bound field, in the order the +// format writes them. An operand renders its field, so it names a level exactly +// as a token does; one scan finds both, which is what puts them in one order. +func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader { + var names []reader + _ = eachToken(format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + a := splitArm(operand, refs) + if _, isBound := bound[a.key]; isBound { + names = append(names, reader{a.name, fmt.Sprintf("%s operand %q", fn, operand)}) + } + } + return nil + } + for _, a := range splitArms(t.body, refs) { + if _, isBound := bound[a.key]; isBound { + names = append(names, reader{a.name, "token {" + a.name + "}"}) + } + } + return nil + }) + return names +} + +// splitArms splits a token body's '|' alternatives. +func splitArms(body string, refs map[string]refBinding) []arm { + parts := strings.Split(body, "|") + arms := make([]arm, len(parts)) + for i, p := range parts { + arms[i] = splitArm(p, refs) + } + return arms +} + +// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside +// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The +// error names the single-token spelling. +func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error { + count := map[string]int{} + return eachToken(format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if _, _, isFunc := funcCall(t.body); isFunc { + return nil + } + for _, a := range splitArms(t.body, refs) { + if len(a.tail) > 0 || !c.held[a.key] { + continue + } + if count[a.key]++; count[a.key] > 1 { + return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name) + } + } + return nil + }) +} + +// draws is what an expansion has already drawn for its held names: the variant each +// was drawn as, so every path under it reads one row, and the value each read, so +// the same name read twice reads one value. +type draws struct { + variant map[string]node + value map[string]string +} + +// readField renders one arm of a token. An arm's key is a sibling field or a +// reference linkRefs bound into fields. A name the expansion holds — a level some +// token addresses by dotted path, or a field an operand reads — is drawn once and +// kept, so {place.postal-code} and {place.locality} read one row, either read twice +// gives one value, and a shown operand is the operand computed. Every other name is +// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and +// linkRefs prove every step, so the walk cannot fail. +func readField(s *session, t *template, held *draws, a arm) string { + if !t.held[a.key] { + return render(s, t.fields[a.key]) + } + if v, read := held.value[a.name]; read { + return v + } + var v string + _ = walkPath(t.fields[a.key], a.tail, pathWalk{ + // Hold the draw at every level passed through, so two paths sharing a + // prefix share it. + choice: func(c *choice, rest []string) ([]node, error) { + key := a.key + if consumed := len(a.tail) - len(rest); consumed > 0 { + key = a.steps[consumed-1] + } + n, drew := held.variant[key] + if !drew { + n = drawn(s, c) + held.variant[key] = n + } + return []node{n}, nil + }, + leaf: func(n node) error { v = render(s, n); return nil }, + }) + held.value[a.name] = v + return v +} + +// drawn resolves a choice to one variant, so a bound head is a concrete node the +// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not +// another set to pick from. +func drawn(s *session, n node) node { + for c, ok := n.(*choice); ok; c, ok = n.(*choice) { + n = pick(s, c) + } + return n +} diff --git a/node.go b/node.go index 84da64c..6cea876 100644 --- a/node.go +++ b/node.go @@ -149,8 +149,8 @@ func compileChoice(items []any) (node, error) { } c.cum = cum } - // Safe to precompute: a choice's items come from one file, so no group can - // appear inside one, and neither mergeChildren nor linkRefs can reach in. + // Computed before linkRefs binds references into the items: a binding is keyed + // by a reference sigil, which paths skips, so the set is the same after. c.shared = sharedPaths(c.items) return c, nil } @@ -229,43 +229,6 @@ func compileTemplate(m map[string]any) (node, error) { return t, nil } -// checkPath reports whether a token's dotted tail can address a node whichever way -// the draw goes, by the reachability rule descend applies — a choice must carry -// the whole remaining path in the set every variant shares — plus the -// rules a held draw adds, which descend has no need of: a level a path reads may -// not carry a repeat, and each variant answers for that itself. So a path that -// validates here resolves on every render, and a typo is a New-time error. -func checkPath(n node, tail []string, level string) error { - if len(tail) == 0 { - return nil - } - switch n := n.(type) { - case *template: - if n.repeat > 1 { - return fmt.Errorf("the level %q carries a repeat, which a path reading one draw of it cannot apply", level) - } - child, ok := n.fields[tail[0]] - if !ok { - return fmt.Errorf("no field %q", tail[0]) - } - return checkPath(child, tail[1:], level+"."+tail[0]) - case *choice: - if want := strings.Join(tail, "."); !n.shared[want] { - return unreachableInChoice(n, want) - } - // Reachability is settled; each variant still answers for itself, so a - // rule about the level (its repeat) holds behind a choice as in front. - for _, item := range n.items { - if err := checkPath(item, tail, level); err != nil { - return err - } - } - return nil - default: - return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) - } -} - // repeatOf reads a template's "repeat" (default 1): how many times its format // is rendered and concatenated. A present one must be an integer above 1. func repeatOf(m map[string]any) (int, error) { diff --git a/path.go b/path.go new file mode 100644 index 0000000..40e7461 --- /dev/null +++ b/path.go @@ -0,0 +1,109 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// pathWalk is what one walk of a dotted path does at each kind of level: choice +// returns the variants to continue into (none stops the walk); level runs at each +// template a segment descends into; leaf runs where the tail ends. A nil action +// is skipped. +type pathWalk struct { + choice func(c *choice, rest []string) ([]node, error) + level func(t *template, rest []string) error + leaf func(n node) error +} + +// walkPath descends tail from n: a group or template by its next segment, a +// choice by w.choice, which consumes no segment. A missing segment is an error, +// so no walk reaches past what the data holds. +func walkPath(n node, tail []string, w pathWalk) error { + if len(tail) == 0 { + if w.leaf != nil { + return w.leaf(n) + } + return nil + } + switch n := n.(type) { + case *group: + child, ok := n.children[tail[0]] + if !ok { + return fmt.Errorf("no entry %q", tail[0]) + } + return walkPath(child, tail[1:], w) + case *template: + if w.level != nil { + if err := w.level(n, tail); err != nil { + return err + } + } + child, ok := n.fields[tail[0]] + if !ok { + return fmt.Errorf("no field %q", tail[0]) + } + return walkPath(child, tail[1:], w) + case *choice: + if w.choice == nil { + return nil + } + next, err := w.choice(n, tail) + if err != nil { + return err + } + for _, item := range next { + if err := walkPath(item, tail, w); err != nil { + return err + } + } + return nil + } + return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) +} + +// carriedByAll is the choice rule a path that must resolve on every call obeys: +// the rest of the tail must be one every variant carries. +func carriedByAll(c *choice, rest []string) error { + if want := strings.Join(rest, "."); !c.shared[want] { + return unreachableInChoice(c, want) + } + return nil +} + +// unreachableInChoice reports that a path cannot step through this choice, listing +// what every variant does carry. It reads the precomputed set, so a failing path +// costs no more than a rendering one. +func unreachableInChoice(c *choice, want string) error { + if len(c.shared) == 0 { + return fmt.Errorf("no variant of this %d-way choice carries %q", len(c.items), want) + } + offered := make([]string, 0, len(c.shared)) + for p := range c.shared { + offered = append(offered, p) + } + sort.Strings(offered) + return fmt.Errorf("not every variant of this %d-way choice carries %q; all carry %v", len(c.items), want, offered) +} + +// checkPath proves a dotted tail resolves whichever way the draws go — a choice +// must carry the rest of the path in the set every variant shares — and that no +// level a path reads carries a repeat, which one draw of it could not apply. So a +// path that validates here resolves on every render, and a typo is a New-time +// error. +func checkPath(n node, tail []string, level string) error { + return walkPath(n, tail, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if err := carriedByAll(c, rest); err != nil { + return nil, err + } + return c.items, nil + }, + level: func(t *template, rest []string) error { + if t.repeat > 1 { + return fmt.Errorf("the level %q carries a repeat, which a path reading one draw of it cannot apply", join(level, strings.Join(tail[:len(tail)-len(rest)], "."))) + } + return nil + }, + }) +} diff --git a/reference.go b/reference.go index 0d59f23..6d685e4 100644 --- a/reference.go +++ b/reference.go @@ -2,7 +2,6 @@ package fejkdata import ( "fmt" - "sort" "strings" ) @@ -137,248 +136,6 @@ func eachTemplate(root map[string]node, fn func(folder []string, path string, t return inFolder(nil, root) } -// checkBoundLevelsHeld rejects every route to a held name except the ones that read -// its draw. An expansion holds one draw of that name; anything else that renders it -// draws again, and the two disagree. checkNoOverlap settles the spellings within one -// format (a token, an operand); this settles the rest — a reference, whether it -// sits in that format or in anything the format renders, however deep. -// -// It runs after checkNoCycles, whose guarantee is what lets the walk terminate. -func checkBoundLevelsHeld(root map[string]node) error { - return walkNodes(root, func(path string, n node) error { - t, ok := n.(*template) - if !ok || len(t.held) == 0 { - return nil - } - heads := make([]string, 0, len(t.held)) - for head := range t.held { - heads = append(heads, head) - } - // Operand heads first, then paths, each in name order: a level read both - // ways is reported by the operand's fence, and which overlap is reported - // does not vary. - sort.Slice(heads, func(i, j int) bool { - _, pi := t.bound[heads[i]] - _, pj := t.bound[heads[j]] - if pi != pj { - return !pi - } - return heads[i] < heads[j] - }) - readers := boundReaders(t.format, t.bound, t.refs) - for _, head := range heads { - // What one draw answers for depends on how the draw is read: a path pins - // the levels it passes through and the leaf it lands on, an operand - // exactly the value its render produces. - held := map[node]bool{} - reader, isPath := t.bound[head] - if isPath { - for _, r := range readers { - if a := splitArm(r.name, t.refs); a.key == head { - coverPath(t.fields[head], a.tail, held) - } - } - } else { - operandDraw(t.fields[head], held) - } - if len(held) == 0 { - continue // an early out: a fixed head holds nothing to reach - } - // One seen set across the edges: a node that cannot reach the level - // cannot reach it by another route either, so it is walked once here. - seen := map[node]bool{} - for _, e := range renderEdges(t) { - if splitArm(e.label, t.refs).key == head { - continue // a token or operand reading this draw, the routes allowed - } - if renders(e.to, held, seen) { - if isPath { - return fmt.Errorf("%s: %s renders %q, which {%s} reads a path into; name the fields you want instead", path, e.reached(), head, reader) - } - return fmt.Errorf("%s: %s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", path, e.reached(), head, operandReader(t, head)) - } - } - } - return nil - }) -} - -// operandReader names the builtin whose operand holds head. -func operandReader(t *template, head string) string { - fn := "" - _ = eachToken(t.format, func(tok ftoken) error { - if tok.kind != 'b' || fn != "" { - return nil - } - if name, _, isFunc := funcCall(tok.body); isFunc { - for _, operand := range tokenOperands(tok.body) { - if splitArm(operand, t.refs).key == head { - fn = name - } - } - } - return nil - }) - return fn -} - -// coverPath collects what holding one path pins: every choice level the path -// passes through, whole, and the leaf it renders. -func coverPath(n node, tail []string, into map[node]bool) { - if _, isChoice := n.(*choice); isChoice || len(tail) == 0 { - cover(n, into, false) - return - } - t, ok := n.(*template) - if !ok { - return - } - if child, ok := t.fields[tail[0]]; ok { - coverPath(child, tail[1:], into) - } -} - -// cover collects a level and everything contained in it. A fixed string outside a -// choice is left out — it cannot disagree with itself — but inside one each -// variant carries its own, so there it counts. -func cover(n node, into map[node]bool, inChoice bool) { - if isFixed(n) && !inChoice { - return - } - into[n] = true - _, isChoice := n.(*choice) - for _, c := range contained(n) { - cover(c.node, into, inChoice || isChoice) - } -} - -// operandDraw collects what one held draw of an operand answers for: the operand -// and what rendering it settles inside itself. The builtin renders its operand -// whole, so that draw fixes every value the render produced, and a second route to -// any of them disagrees with it. -// -// The walk stops at a reference edge, which is where the operand's own value ends -// and a shared source begins: two names referencing one category are two draws, the -// same rule {word} {word} follows. -func operandDraw(n node, into map[node]bool) { - if isFixed(n) { - return - } - if into[n] { - return - } - into[n] = true - for _, e := range renderEdges(n) { - if isRef(e.label) { - continue - } - operandDraw(e.to, into) - } -} - -// isFixed is a string that varies nothing: fixed text with no fields to read into. -func isFixed(n node) bool { - t, ok := n.(*template) - return ok && t.fixed && len(t.fields) == 0 -} - -// renders reports whether rendering n can reach anything in want, following the -// same edges expand does. seen keeps a node shared by several routes from being -// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk -// ends. -func renders(n node, want, seen map[node]bool) bool { - if want[n] { - return true - } - if seen[n] { - return false - } - seen[n] = true - for _, e := range renderEdges(n) { - if renders(e.to, want, seen) { - return true - } - } - return false -} - -// walkNodes calls fn once per contained node, passing the dot path that reaches it, -// visiting keys in sorted order so which of several broken nodes gets reported does -// not depend on map iteration. -func walkNodes(root map[string]node, fn func(path string, n node) error) error { - seen := map[node]bool{} - var visit func(string, node) error - visit = func(path string, n node) error { - if n == nil || seen[n] { - return nil - } - seen[n] = true - if err := fn(path, n); err != nil { - return err - } - for _, c := range contained(n) { - if err := visit(join(path, c.name), c.node); err != nil { - return err - } - } - return nil - } - for _, name := range sortedNames(root) { - if err := visit(name, root[name]); err != nil { - return err - } - } - return nil -} - -// namedNode is a contained child and the segment reaching it; a choice's items carry -// no segment, matching how a dot path steps over a choice. -type namedNode struct { - name string - node node -} - -func contained(n node) []namedNode { - switch n := n.(type) { - case *group: - return named(n.children) - case *choice: - out := make([]namedNode, len(n.items)) - for i, it := range n.items { - out[i] = namedNode{node: it} - } - return out - case *template: - return named(n.fields) - default: - return nil - } -} - -// named skips a bound {/path} key: it is a render edge, not containment, so using -// it as a path segment would report a node under a path that does not reach it. Only -// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a -// group's children never carry the prefix — so this one skip serves both. -func named(m map[string]node) []namedNode { - out := make([]namedNode, 0, len(m)) - for _, name := range sortedNames(m) { - if isRef(name) { - continue - } - out = append(out, namedNode{name: name, node: m[name]}) - } - return out -} - -func sortedNames(m map[string]node) []string { - names := make([]string, 0, len(m)) - for name := range m { - names = append(names, name) - } - sort.Strings(names) - return names -} - // resolveRef walks a reference path through the folders to the category it names, // returning that head, the node, and the tail left to read into it. func resolveRef(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { @@ -413,148 +170,3 @@ func refTokens(format string) []string { } return refs } - -// renderEdge is a child a node renders into, labelled by what reaches it (a field -// name, reference, or choice index) for a readable cycle report. operand names -// the builtin when the label is its operand rather than a token, so an error can -// name it the way the author wrote it. -type renderEdge struct { - to node - label string - operand string -} - -// reached names an edge as the author spelled it, the vocabulary boundReaders uses -// for the sibling fence. -func (e renderEdge) reached() string { - if e.operand != "" { - return fmt.Sprintf("%s operand %q", e.operand, e.label) - } - return "{" + e.label + "}" -} - -// renderEdges lists the children rendering n recurses into, mirroring expand: a -// choice's items, and a template's field/reference tokens plus its operands. A -// group renders nothing, so it has no edges. -func renderEdges(n node) []renderEdge { - switch n := n.(type) { - case *choice: - es := make([]renderEdge, len(n.items)) - for i, it := range n.items { - es[i] = renderEdge{to: it, label: fmt.Sprintf("[%d]", i)} - } - return es - case *template: - var es []renderEdge - add := func(name, operand string) { - a := splitArm(name, n.refs) - c, ok := n.fields[a.key] - if !ok { - return - } - for _, leaf := range pathLeaves(c, a.tail) { - es = append(es, renderEdge{leaf, name, operand}) - } - } - _ = eachToken(n.format, func(t ftoken) error { - if t.kind != 'b' { - return nil - } - if fn, _, isFunc := funcCall(t.body); isFunc { - for _, operand := range tokenOperands(t.body) { - add(operand, fn) - } - return nil - } - for _, name := range strings.Split(t.body, "|") { - add(name, "") - } - return nil - }) - return es - default: - return nil - } -} - -// pathLeaves lists what a token's dotted tail renders. A path draws the levels it -// passes through but renders only what it lands on, so the leaf is the edge — a -// bare token, whose tail is empty, lands on the field itself. A choice on the way -// contributes every variant, since any of them may be the one drawn. checkPath has -// already proved the tail resolves in every variant, so the walk drops nothing. -func pathLeaves(n node, tail []string) []node { - if len(tail) == 0 { - return []node{n} - } - if c, ok := n.(*choice); ok { - var out []node - for _, it := range c.items { - out = append(out, pathLeaves(it, tail)...) - } - return out - } - return pathLeaves(child(n, tail[0]), tail[1:]) -} - -// checkRepeatReach bounds the renders a repeat multiplies to along any root-to-leaf -// path, so nested repeats cannot build what one repeat may not. It runs after -// checkNoCycles, whose guarantee is what lets the walk terminate. -func checkRepeatReach(root map[string]node) error { - reach := map[node]int{} - var of func(n node) int - of = func(n node) int { - if r, done := reach[n]; done { - return r - } - r := 1 - for _, e := range renderEdges(n) { - if c := of(e.to); c > r { - r = c - } - } - if t, ok := n.(*template); ok { - r *= t.repeat - } - reach[n] = r - return r - } - return walkNodes(root, func(path string, n node) error { - if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > maxLen { - return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), maxLen) - } - return nil - }) -} - -// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself -// — directly, mutually, or through a chain — never terminates, so it must fail at -// New rather than stack-overflow at render. It is a depth-first walk of the render -// graph (renderEdges); grey marks nodes on the current path so a back-edge to one -// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped. -// Every node is a root: a field its parent's format never renders is still reachable -// by dot path, so a cycle in one would otherwise reach render and be fatal there. -func checkNoCycles(root map[string]node) error { - const ( - grey = 1 - black = 2 - ) - color := map[node]int{} - var visit func(n node, path string) error - visit = func(n node, path string) error { - switch color[n] { - case grey: - return fmt.Errorf("reference cycle: %s", path) - case black: - return nil - } - color[n] = grey - for _, e := range renderEdges(n) { - if err := visit(e.to, path+" -> "+e.label); err != nil { - return err - } - } - color[n] = black - return nil - } - return walkNodes(root, func(path string, n node) error { return visit(n, path) }) -} diff --git a/render.go b/render.go index 937874e..2d998fb 100644 --- a/render.go +++ b/render.go @@ -33,48 +33,20 @@ func (f *Generator) Fake(path string) (string, error) { // descend walks named fields to the node a path names. It is the one render-side // step that can fail, because the path comes from the caller and may name a field // that does not exist. A choice consumes no segment, so the rest of the path must -// be one every variant carries (the set compile stored) before a variant is picked -// — a path that resolves at all resolves on every call. -func descend(s *session, n node, segments []string) (node, error) { - if len(segments) == 0 { - return n, nil - } - switch n := n.(type) { - case *group: - child, ok := n.children[segments[0]] - if !ok { - return nil, fmt.Errorf("no entry %q", segments[0]) - } - return descend(s, child, segments[1:]) - case *template: - child, ok := n.fields[segments[0]] - if !ok { - return nil, fmt.Errorf("no field %q", segments[0]) - } - return descend(s, child, segments[1:]) - case *choice: - if want := strings.Join(segments, "."); !n.shared[want] { - return nil, unreachableInChoice(n, want) - } - return descend(s, pick(s, n), segments) - default: - return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[0]) - } -} - -// unreachableInChoice reports that a path cannot step through this choice, listing -// what every variant does carry. It reads the precomputed set, so a failing path -// costs no more than a rendering one. -func unreachableInChoice(c *choice, want string) error { - if len(c.shared) == 0 { - return fmt.Errorf("no variant of this %d-way choice carries %q", len(c.items), want) - } - offered := make([]string, 0, len(c.shared)) - for p := range c.shared { - offered = append(offered, p) - } - sort.Strings(offered) - return fmt.Errorf("not every variant of this %d-way choice carries %q; all carry %v", len(c.items), want, offered) +// be one every variant carries before a variant is picked — a path that resolves +// at all resolves on every call. +func descend(s *session, root node, segments []string) (node, error) { + var found node + err := walkPath(root, segments, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if err := carriedByAll(c, rest); err != nil { + return nil, err + } + return []node{pick(s, c)}, nil + }, + leaf: func(n node) error { found = n; return nil }, + }) + return found, err } // render evaluates a compiled node to a string. compile validates every node up @@ -152,76 +124,3 @@ func expand(s *session, t *template) string { } return b.String() } - -// draws is what an expansion has already drawn for its held names: the variant each -// was drawn as, so every path under it reads one row, and the value each read, so -// the same name read twice reads one value. -type draws struct { - variant map[string]node - value map[string]string -} - -// readField renders one arm of a token. An arm's key is a sibling field or a -// {/path} reference, which linkRefs bound into fields too. A name the expansion -// holds — a level some token addresses by dotted path, or a sibling a {calc()} -// reads — is drawn once and kept, so {place.postal-code} and {place.locality} read -// one row, either read twice gives one value, and a shown operand is the operand -// computed. Every other name is drawn afresh, so {word} {word} still draws twice. -// checkTokens, checkPath and linkRefs prove every step, so this cannot fail. -func readField(s *session, t *template, held *draws, a arm) string { - if !t.held[a.key] { - return render(s, t.fields[a.key]) - } - if v, read := held.value[a.name]; read { - return v - } - n, drew := held.variant[a.key] - if !drew { - n = drawn(s, t.fields[a.key]) - held.variant[a.key] = n - } - // Hold the draw at every level passed through, so two paths sharing a prefix - // share it. - for i, seg := range a.tail { - if i < len(a.steps) { - step, drew := held.variant[a.steps[i]] - if !drew { - step = drawn(s, child(n, seg)) - held.variant[a.steps[i]] = step - } - n = step - continue - } - n = child(n, seg) - } - v := render(s, n) - held.value[a.name] = v - return v -} - -// child is the node one path segment names below an already-drawn node. It holds -// while checkPath and the set a choice shares (see sharedPaths) agree with this -// walk: both prove the segment exists and that drawn leaves a template here. Each -// way that can break panics naming the segment, so a slip in that agreement -// reports where it happened rather than surfacing a nil node a level later. -func child(n node, seg string) node { - t, ok := n.(*template) - if !ok { - panic(fmt.Sprintf("fejkdata: %q under %T, which carries no fields", seg, n)) - } - c, ok := t.fields[seg] - if !ok { - panic(fmt.Sprintf("fejkdata: no field %q under a drawn level", seg)) - } - return c -} - -// drawn resolves a choice to one variant, so a bound head is a concrete node the -// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not -// another set to pick from. -func drawn(s *session, n node) node { - for c, ok := n.(*choice); ok; c, ok = n.(*choice) { - n = pick(s, c) - } - return n -} diff --git a/template.go b/template.go index bb337cd..03c057e 100644 --- a/template.go +++ b/template.go @@ -2,7 +2,6 @@ package fejkdata import ( "fmt" - "sort" "strings" ) @@ -229,98 +228,6 @@ func fieldTokens(format string) []string { return names } -// arm is one alternative of a {a|b} token or one operand, split into the key -// naming the node in a template's fields (a sibling field, or the head a -// reference is bound under) and the tail of a dotted path into it. A non-empty -// tail is what makes the arm a bound draw: its head is drawn once per expansion -// (see compileOps). -type arm struct { - name string // as written, and the key a bound draw's value is held under - key string - tail []string - steps []string // key per level passed through; the head and leaf hold their own -} - -// splitArm splits one name into key and tail. refs maps a reference to what -// linkRefs bound it to; before linking, a reference is whole. -func splitArm(name string, refs map[string]refBinding) arm { - if isRef(name) { - b, bound := refs[name] - if !bound || len(b.tail) == 0 { - key := name - if bound { - key = b.key - } - return arm{name: name, key: key} - } - return pathArm(name, b.key, b.tail) - } - head, tail, dotted := strings.Cut(name, ".") - if !dotted { - return arm{name: name, key: name} - } - return pathArm(name, head, strings.Split(tail, ".")) -} - -func pathArm(name, key string, segs []string) arm { - var steps []string - for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own - steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) - } - return arm{name: name, key: key, tail: segs, steps: steps} -} - -// checkNoOverlap rejects a format that both renders a level and reads a path into -// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The path reads the -// level's held draw while rendering the level expands it afresh, so their values -// would disagree. Names are compared in sorted order, so which pair is reported -// does not depend on where the tokens sit. -func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error { - names := boundReaders(format, bound, refs) - // Stable over one format-order scan, so two readers of one name (a token and a - // calc operand both naming "p") are reported as the format writes them. - sort.SliceStable(names, func(i, j int) bool { return names[i].name < names[j].name }) - for i, level := range names { - for _, path := range names[i+1:] { - if strings.HasPrefix(path.name, level.name+".") { - return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name) - } - } - } - return nil -} - -// reader is one way a format reaches a bound field, and how to name that spelling. -type reader struct{ name, label string } - -// boundReaders lists every way a format reaches a bound field, in the order the -// format writes them. An operand renders its field, so it names a level exactly -// as a token does; one scan finds both, which is what puts them in one order. -func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader { - var names []reader - _ = eachToken(format, func(t ftoken) error { - if t.kind != 'b' { - return nil - } - if fn, _, isFunc := funcCall(t.body); isFunc { - for _, operand := range tokenOperands(t.body) { - a := splitArm(operand, refs) - if _, isBound := bound[a.key]; isBound { - names = append(names, reader{a.name, fmt.Sprintf("%s operand %q", fn, operand)}) - } - } - return nil - } - for _, a := range splitArms(t.body, refs) { - if _, isBound := bound[a.key]; isBound { - names = append(names, reader{a.name, "token {" + a.name + "}"}) - } - } - return nil - }) - return names -} - // checkSegments rejects an unfinished path: "{a.}", "{.b}" and "{a..b}" each have // a segment naming nothing. A field really named "" would otherwise make them // resolve, so a typo would read as a path that worked. @@ -339,16 +246,6 @@ func checkSegments(a arm) error { return nil } -// splitArms splits a token body's '|' alternatives. -func splitArms(body string, refs map[string]refBinding) []arm { - parts := strings.Split(body, "|") - arms := make([]arm, len(parts)) - for i, p := range parts { - arms[i] = splitArm(p, refs) - } - return arms -} - // callFn is a builtin bound to one call site: its args already parsed. It reads the // output emitted so far in the current expansion (a derivation's payload) and the // values of the operands it named, which expand read for it. @@ -450,27 +347,3 @@ func compileOps(format string, refs map[string]refBinding) formatOps { flush() return c } - -// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside -// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The -// error names the single-token spelling. -func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error { - count := map[string]int{} - return eachToken(format, func(t ftoken) error { - if t.kind != 'b' { - return nil - } - if _, _, isFunc := funcCall(t.body); isFunc { - return nil - } - for _, a := range splitArms(t.body, refs) { - if len(a.tail) > 0 || !c.held[a.key] { - continue - } - if count[a.key]++; count[a.key] > 1 { - return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name) - } - } - return nil - }) -} -- 2.52.0 From f9647ff6bbb1d615b8679606c3bc23e06abe67b9 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:31:38 +0200 Subject: [PATCH 26/38] Decompose compileTemplate, checkBoundLevelsHeld and loadDir; gate cyclomatic complexity at 14 --- Dockerfile | 1 + README.md | 3 +- calc.go | 1 + compose.yaml | 4 ++ data.go | 77 +++++++++++++++++++++--------------- hold.go | 107 +++++++++++++++++++++++++++++---------------------- node.go | 74 ++++++++++++++++++++++++----------- template.go | 3 +- 8 files changed, 169 insertions(+), 101 deletions(-) diff --git a/Dockerfile b/Dockerfile index 227d6da..8f04a4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN go mod download COPY . . RUN go vet ./... && \ + go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \ { unformatted="$(gofmt -l .)"; test -z "$unformatted" || \ { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \ go test -race ./... diff --git a/README.md b/README.md index 773d875..e775105 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,7 @@ docker compose run --rm cover # tests with coverage docker compose run --rm bench # benchmarks docker compose run --rm build # compile the library docker compose run --rm vet # go vet +docker compose run --rm cyclo # cyclomatic complexity over 14 (test files excluded) docker compose run --rm dev # interactive shell ``` @@ -391,7 +392,7 @@ docker compose run --rm --user "$(id -u):$(id -g)" tidy # go mod tidy Every pull request runs `docker build .` against both the latest and the lowest supported Go, and must pass before it can be merged. That build is the whole -gate — vet, format check and tests — so run it locally before pushing: +gate — vet, complexity, format check and tests — so run it locally before pushing: ```sh docker build . # latest diff --git a/calc.go b/calc.go index a9a8653..140b84d 100644 --- a/calc.go +++ b/calc.go @@ -242,6 +242,7 @@ func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNod } } +// factor is a table-shaped scanner, one case per token kind, kept whole on purpose. func (p *calcParser) factor() (calcNode, error) { p.space() if p.pos >= len(p.rs) { diff --git a/compose.yaml b/compose.yaml index fda93dd..bb47873 100644 --- a/compose.yaml +++ b/compose.yaml @@ -42,6 +42,10 @@ services: <<: *go command: go vet ./... + cyclo: + <<: *go + command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . + build: <<: *go command: go build ./... diff --git a/data.go b/data.go index fe78367..3d59bb6 100644 --- a/data.go +++ b/data.go @@ -87,44 +87,59 @@ func loadDir(src dataSource, dir string) (*group, error) { continue } full := path.Join(dir, e.Name()) + load := loadFile if e.IsDir() { - child, err := loadDir(src, full) - if err != nil { - return nil, err - } - if len(child.children) == 0 { - continue - } - if err := checkName(e.Name()); err != nil { - return nil, fmt.Errorf("%s: folder %w", src.name(full), err) - } - g.children[e.Name()] = child - continue + load = loadFolder } - if !strings.HasSuffix(e.Name(), ".json") { - continue + if err := load(src, g, full, e.Name()); err != nil { + return nil, err } - name := strings.TrimSuffix(e.Name(), ".json") - if err := checkName(name); err != nil { - return nil, fmt.Errorf("%s: category %w", src.name(full), err) - } - b, err := fs.ReadFile(src.fsys, full) - if err != nil { - return nil, fmt.Errorf("%s: %w", src.name(full), err) - } - var raw any - if err := json.Unmarshal(b, &raw); err != nil { - return nil, fmt.Errorf("%s: %w", src.name(full), err) - } - n, err := compile(raw) - if err != nil { - return nil, fmt.Errorf("%s: %w", src.name(full), err) - } - g.children[name] = n } return g, nil } +// loadFolder adds a subdirectory as a nested group, unless nothing under it is data. +func loadFolder(src dataSource, g *group, full, name string) error { + child, err := loadDir(src, full) + if err != nil { + return err + } + if len(child.children) == 0 { + return nil + } + if err := checkName(name); err != nil { + return fmt.Errorf("%s: folder %w", src.name(full), err) + } + g.children[name] = child + return nil +} + +// loadFile compiles a *.json file into a category named after it; any other file +// is skipped. +func loadFile(src dataSource, g *group, full, file string) error { + if !strings.HasSuffix(file, ".json") { + return nil + } + name := strings.TrimSuffix(file, ".json") + if err := checkName(name); err != nil { + return fmt.Errorf("%s: category %w", src.name(full), err) + } + b, err := fs.ReadFile(src.fsys, full) + if err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + var raw any + if err := json.Unmarshal(b, &raw); err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + n, err := compile(raw) + if err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + g.children[name] = n + return nil +} + // mergeChildren overlays src onto dst. Two groups under the same key merge // recursively (so locales/categories from several paths combine); every other // key is replaced, making the last-loaded directory win on a conflict. diff --git a/hold.go b/hold.go index 965dce2..def4091 100644 --- a/hold.go +++ b/hold.go @@ -19,59 +19,74 @@ func checkBoundLevelsHeld(root map[string]node) error { if !ok || len(t.held) == 0 { return nil } - heads := make([]string, 0, len(t.held)) - for head := range t.held { - heads = append(heads, head) - } - // Operand heads first, then paths, each in name order: a level read both - // ways is reported by the operand's fence, and which overlap is reported - // does not vary. - sort.Slice(heads, func(i, j int) bool { - _, pi := t.bound[heads[i]] - _, pj := t.bound[heads[j]] - if pi != pj { - return !pi - } - return heads[i] < heads[j] - }) readers := boundReaders(t.format, t.bound, t.refs) - for _, head := range heads { - // What one draw answers for depends on how the draw is read: a path pins - // the levels it passes through and the leaf it lands on, an operand - // exactly the value its render produces. - held := map[node]bool{} - reader, isPath := t.bound[head] - if isPath { - for _, r := range readers { - if a := splitArm(r.name, t.refs); a.key == head { - coverPath(t.fields[head], a.tail, held) - } - } - } else { - operandDraw(t.fields[head], held) - } - if len(held) == 0 { - continue // an early out: a fixed head holds nothing to reach - } - // One seen set across the edges: a node that cannot reach the level - // cannot reach it by another route either, so it is walked once here. - seen := map[node]bool{} - for _, e := range renderEdges(t) { - if splitArm(e.label, t.refs).key == head { - continue // a token or operand reading this draw, the routes allowed - } - if renders(e.to, held, seen) { - if isPath { - return fmt.Errorf("%s: %s renders %q, which {%s} reads a path into; name the fields you want instead", path, e.reached(), head, reader) - } - return fmt.Errorf("%s: %s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", path, e.reached(), head, operandReader(t, head)) - } + for _, head := range heldHeads(t) { + if err := checkHeadHeld(t, head, readers); err != nil { + return fmt.Errorf("%s: %w", path, err) } } return nil }) } +// heldHeads lists a template's held names, operand heads first, then paths, each +// in name order: a level read both ways is reported by the operand's fence, and +// which overlap is reported does not vary. +func heldHeads(t *template) []string { + heads := make([]string, 0, len(t.held)) + for head := range t.held { + heads = append(heads, head) + } + sort.Slice(heads, func(i, j int) bool { + _, pi := t.bound[heads[i]] + _, pj := t.bound[heads[j]] + if pi != pj { + return !pi + } + return heads[i] < heads[j] + }) + return heads +} + +// pinned collects what the hold of head answers for: a path pins the levels it +// passes through and the leaf it lands on, an operand exactly the value its render +// produces. +func pinned(t *template, head string, readers []reader) map[node]bool { + held := map[node]bool{} + if _, isPath := t.bound[head]; !isPath { + operandDraw(t.fields[head], held) + return held + } + for _, r := range readers { + if a := splitArm(r.name, t.refs); a.key == head { + coverPath(t.fields[head], a.tail, held) + } + } + return held +} + +// checkHeadHeld rejects every route to what head's hold pins except the readers +// holding it. One seen set across the edges: a node that cannot reach the level +// cannot reach it by another route either, so it is walked once. +func checkHeadHeld(t *template, head string, readers []reader) error { + held := pinned(t, head, readers) + if len(held) == 0 { + return nil // a fixed head holds nothing to reach + } + reader, isPath := t.bound[head] + seen := map[node]bool{} + for _, e := range renderEdges(t) { + if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) { + continue + } + if isPath { + return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader) + } + return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head)) + } + return nil +} + // operandReader names the builtin whose operand holds head. func operandReader(t *template, head string) string { fn := "" diff --git a/node.go b/node.go index 6cea876..54b44a3 100644 --- a/node.go +++ b/node.go @@ -181,29 +181,68 @@ func checkNoRepeatedItem(items []any) error { } func compileTemplate(m map[string]any) (node, error) { - format, ok := m["format"].(string) - if !ok { - return nil, fmt.Errorf("template object missing string \"format\"") - } - repeat, err := repeatOf(m) + o, err := readOptions(m) if err != nil { return nil, err } - sep := "" + fields, err := compileFields(m) + if err != nil { + return nil, err + } + if len(fields) == 0 && o.repeat == 1 && !o.weighted { + return nil, fmt.Errorf("an object holding only a format is a string; write %q", o.format) + } + if err := checkTokens(o.format, fields); err != nil { + return nil, err + } + t := &template{format: o.format, fields: fields, repeat: o.repeat, separator: o.separator} + if err := t.compileFormat(); err != nil { + return nil, err + } + return t, nil +} + +// templateOptions is what a template object's option keys say. +type templateOptions struct { + format string + repeat int + separator string + weighted bool +} + +func readOptions(m map[string]any) (templateOptions, error) { + var o templateOptions + format, ok := m["format"].(string) + if !ok { + return o, fmt.Errorf("template object missing string \"format\"") + } + o.format = format + repeat, err := repeatOf(m) + if err != nil { + return o, err + } + o.repeat = repeat if sv, ok := m["separator"]; ok { - if sep, ok = sv.(string); !ok { - return nil, fmt.Errorf("separator must be a string, got %T", sv) + if o.separator, ok = sv.(string); !ok { + return o, fmt.Errorf("separator must be a string, got %T", sv) } if repeat == 1 { - return nil, fmt.Errorf("separator joins repeated renders, so it has no effect without a repeat above 1") + return o, fmt.Errorf("separator joins repeated renders, so it has no effect without a repeat above 1") } } - t := &template{format: format, fields: make(map[string]node, len(m)), repeat: repeat, separator: sep} + _, o.weighted = m["weight"] + return o, nil +} + +// compileFields compiles every non-option key of a template object, in name order +// so which of several bad fields is reported does not vary. +func compileFields(m map[string]any) (map[string]node, error) { + fields := make(map[string]node, len(m)) keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } - sort.Strings(keys) // so which of several bad fields is reported does not vary + sort.Strings(keys) for _, k := range keys { if isOption(k) { continue @@ -215,18 +254,9 @@ func compileTemplate(m map[string]any) (node, error) { if err != nil { return nil, fmt.Errorf("field %q: %w", k, err) } - t.fields[k] = n + fields[k] = n } - if _, weighted := m["weight"]; len(t.fields) == 0 && repeat == 1 && !weighted { - return nil, fmt.Errorf("an object holding only a format is a string; write %q", format) - } - if err := checkTokens(format, t.fields); err != nil { - return nil, err - } - if err := t.compileFormat(); err != nil { - return nil, err - } - return t, nil + return fields, nil } // repeatOf reads a template's "repeat" (default 1): how many times its format diff --git a/template.go b/template.go index 03c057e..87f2ff9 100644 --- a/template.go +++ b/template.go @@ -15,7 +15,8 @@ type ftoken struct { // eachToken scans a format string once and calls fn for each unit, the single // source of truth for how braces are read: "{{" and "}}" are literal braces, a "{" -// opens a token that must reach its "}", and a lone "}" is an error. +// opens a token that must reach its "}", and a lone "}" is an error. A table-shaped +// scanner, one case per rune kind, kept whole on purpose. func eachToken(format string, fn func(ftoken) error) error { rs := []rune(format) for i := 0; i < len(rs); i++ { -- 2.52.0 From 1b8768e431e7c3f8794c955640981bda30f64e01 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:32:11 +0200 Subject: [PATCH 27/38] Record the compatibility policy, the override consequence and the deferred data de-duplication --- AGENTS.md | 8 ++++++++ README.md | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 87d73c9..daf3121 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,3 +7,11 @@ - One spelling per result: reject the other at `New`, and let the error name the spelling to use. - A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. + +# Deferred + +- **Shipped-data de-duplication (2026-09-02).** `email.json`'s `local` is a + drifted copy of `username.json`, and no shipped file yet uses a held path, an + operand or a reference. Fixed in the data fill before the first tag, when the + shipped set is rewritten anyway; premise: nothing depends on the shipped data's + shape until then. Not raised in review before that. diff --git a/README.md b/README.md index e775105..9849ef3 100644 --- a/README.md +++ b/README.md @@ -364,6 +364,21 @@ tokens add cost in proportion to the output. folder, `..` the folder above — what those spellings already mean to anyone who has typed a path. A locale's files reach each other without naming the locale, so a folder renames and copies without editing its references. +- **After the first tag, a new fence is a major version.** Data files are the + public API, and one spelling per result grows by tightening, so every fence + invalidates some file. Each such release names the rejected spelling and its + replacement in the changelog and in the load error, and that is the whole + migration: a fence rejects one spelling with one replacement, so the fix is + local to each site. A fence that would need a non-local rewrite ships a + converter with its release instead. Before the first tag there is no + compatibility promise. +- **A `--data-path` override rebinds every reference to the category it + replaces.** References bind against the merged tree, so once shipped data uses + `{.person}`, a consumer's `sv_SE/person.json` is what every shipped reference + into `person` reads, and `New` fails on shipped data the consumer never wrote + when that file lacks a field those references read. Accepted: overriding is the + point of layering, the error names the reference and the field, and the fix is + the consumer's file carrying the fields the shipped tree reads. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. -- 2.52.0 From 512696722d5cd885e8431b8c84a8391094a7aadb Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:45:21 +0200 Subject: [PATCH 28/38] Tests for two spellings of one reference being one level, and a slash after a sigil --- reference_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/reference_test.go b/reference_test.go index a87e68a..65494d5 100644 --- a/reference_test.go +++ b/reference_test.go @@ -312,3 +312,37 @@ func TestReferenceSigilErrors(t *testing.T) { } } } + +func TestSpellingsOfOneReferenceAreOneLevel(t *testing.T) { + person := `[{"format":"{first} {last}","first":"Ada","last":"Byron"},{"format":"{first} {last}","first":"Bo","last":"Ek"}]` + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "sv_SE/person": person, + "sv_SE/mail": `"{.person} <{/sv_SE.person.first}>"`, + }))) + if err == nil || !strings.Contains(err.Error(), "reads a path into") || !strings.Contains(err.Error(), "{.person}") { + t.Errorf("New = %v, want the bare spelling rejected beside the path spelling", err) + } + dir := writeData(t, map[string]string{ + "sv_SE/word": `["alpha","beta","gamma"]`, + "sv_SE/loud": `"{/sv_SE.word} {uppercase(.word)}"`, + }) + f := newGenerator(t, dir, WithSeed(2)) + for i := 0; i < 50; i++ { + got := strings.Fields(fake(t, f, "sv_SE.loud")) + if len(got) != 2 || strings.ToUpper(got[0]) != got[1] { + t.Fatalf("loud = %q, want one draw under both spellings", got) + } + } +} + +func TestSlashAfterAReferenceSigilIsRejected(t *testing.T) { + for name, files := range map[string]map[string]string{ + "after ..": {"sv_SE/person": `"Ada"`, "sv_SE/deep/a": `"{../person}"`}, + "after .": {"sv_SE/person": `"Ada"`, "sv_SE/a": `"{./person}"`}, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))) + if err == nil || !strings.Contains(err.Error(), "person}") || !strings.Contains(err.Error(), "write {") { + t.Errorf("%s: New = %v, want the slash rejected naming the spelling", name, err) + } + } +} -- 2.52.0 From fdbbf94d2cf6c289663678ddfc71662c78a74988 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 18:45:31 +0200 Subject: [PATCH 29/38] Compare and cache a read by its one spelling, so two spellings of one reference are one level; reject a slash after a sigil naming the spelling --- AGENTS.md | 1 + hold.go | 81 +++++++++++----------------------------------------- path.go | 3 +- reference.go | 7 ++++- template.go | 56 ++++++++++++++++++++++++++++++++++-- 5 files changed, 79 insertions(+), 69 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index daf3121..75d511e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ - One spelling per result: reject the other at `New`, and let the error name the spelling to use. - A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. +- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`eachToken`, `calc.factor`, `walkPath`) sit at 13–14 and stay whole; anything else that reaches 14 is decomposed. # Deferred diff --git a/hold.go b/hold.go index def4091..a900a7c 100644 --- a/hold.go +++ b/hold.go @@ -179,60 +179,20 @@ func renders(n node, want, seen map[node]bool) bool { return false } -// arm is one alternative of a {a|b} token or one operand, split into the key -// naming the node in a template's fields (a sibling field, or the head a -// reference is bound under) and the tail of a dotted path into it. A non-empty -// tail is what makes the arm a bound draw: its head is drawn once per expansion -// (see compileOps). -type arm struct { - name string // as written, and the key a bound draw's value is held under - key string - tail []string - steps []string // key per level passed through; the head and leaf hold their own -} - -// splitArm splits one name into key and tail. refs maps a reference to what -// linkRefs bound it to; before linking, a reference is whole. -func splitArm(name string, refs map[string]refBinding) arm { - if isRef(name) { - b, bound := refs[name] - if !bound || len(b.tail) == 0 { - key := name - if bound { - key = b.key - } - return arm{name: name, key: key} - } - return pathArm(name, b.key, b.tail) - } - head, tail, dotted := strings.Cut(name, ".") - if !dotted { - return arm{name: name, key: name} - } - return pathArm(name, head, strings.Split(tail, ".")) -} - -func pathArm(name, key string, segs []string) arm { - var steps []string - for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own - steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) - } - return arm{name: name, key: key, tail: segs, steps: steps} -} - // checkNoOverlap rejects a format that both renders a level and reads a path into -// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The path reads the -// level's held draw while rendering the level expands it afresh, so their values -// would disagree. Names are compared in sorted order, so which pair is reported -// does not depend on where the tokens sit. +// it — {p} beside {p.first}, {p.addr} beside {p.addr.city}, {.p} beside +// {/sv_SE.p.first}. The path reads the level's held draw while rendering the level +// expands it afresh, so their values would disagree. Reads are compared by their +// one spelling, in sorted order, so which pair is reported depends neither on how +// a reference was written nor on where the tokens sit. func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error { names := boundReaders(format, bound, refs) // Stable over one format-order scan, so two readers of one name (a token and a // calc operand both naming "p") are reported as the format writes them. - sort.SliceStable(names, func(i, j int) bool { return names[i].name < names[j].name }) + sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path }) for i, level := range names { for _, path := range names[i+1:] { - if strings.HasPrefix(path.name, level.name+".") { + if strings.HasPrefix(path.path, level.path+".") { return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name) } } @@ -240,8 +200,9 @@ func checkNoOverlap(format string, bound map[string]string, refs map[string]refB return nil } -// reader is one way a format reaches a bound field, and how to name that spelling. -type reader struct{ name, label string } +// reader is one way a format reaches a bound field: as written, by its one +// spelling, and how to name it. +type reader struct{ name, path, label string } // boundReaders lists every way a format reaches a bound field, in the order the // format writes them. An operand renders its field, so it names a level exactly @@ -256,14 +217,14 @@ func boundReaders(format string, bound map[string]string, refs map[string]refBin for _, operand := range tokenOperands(t.body) { a := splitArm(operand, refs) if _, isBound := bound[a.key]; isBound { - names = append(names, reader{a.name, fmt.Sprintf("%s operand %q", fn, operand)}) + names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)}) } } return nil } for _, a := range splitArms(t.body, refs) { if _, isBound := bound[a.key]; isBound { - names = append(names, reader{a.name, "token {" + a.name + "}"}) + names = append(names, reader{a.name, a.path, "token {" + a.name + "}"}) } } return nil @@ -271,16 +232,6 @@ func boundReaders(format string, bound map[string]string, refs map[string]refBin return names } -// splitArms splits a token body's '|' alternatives. -func splitArms(body string, refs map[string]refBinding) []arm { - parts := strings.Split(body, "|") - arms := make([]arm, len(parts)) - for i, p := range parts { - arms[i] = splitArm(p, refs) - } - return arms -} - // checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside // {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The // error names the single-token spelling. @@ -306,8 +257,8 @@ func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) } // draws is what an expansion has already drawn for its held names: the variant each -// was drawn as, so every path under it reads one row, and the value each read, so -// the same name read twice reads one value. +// was drawn as, so every path under it reads one row, and the value each read, by +// its one spelling, so the same read written twice reads one value. type draws struct { variant map[string]node value map[string]string @@ -324,7 +275,7 @@ func readField(s *session, t *template, held *draws, a arm) string { if !t.held[a.key] { return render(s, t.fields[a.key]) } - if v, read := held.value[a.name]; read { + if v, read := held.value[a.path]; read { return v } var v string @@ -345,7 +296,7 @@ func readField(s *session, t *template, held *draws, a arm) string { }, leaf: func(n node) error { v = render(s, n); return nil }, }) - held.value[a.name] = v + held.value[a.path] = v return v } diff --git a/path.go b/path.go index 40e7461..cc3edff 100644 --- a/path.go +++ b/path.go @@ -18,7 +18,8 @@ type pathWalk struct { // walkPath descends tail from n: a group or template by its next segment, a // choice by w.choice, which consumes no segment. A missing segment is an error, -// so no walk reaches past what the data holds. +// so no walk reaches past what the data holds. A table-shaped dispatch, one case +// per node kind, kept whole on purpose. func walkPath(n node, tail []string, w pathWalk) error { if len(tail) == 0 { if w.leaf != nil { diff --git a/reference.go b/reference.go index 6d685e4..664521b 100644 --- a/reference.go +++ b/reference.go @@ -29,6 +29,9 @@ func refShape(name string) (sigil, rest string, err error) { if rest == "" { return "", "", fmt.Errorf("reference has no path") } + if strings.HasPrefix(rest, "/") { + return "", "", fmt.Errorf("the path after %s starts at a name, not a /; write {%s%s}", sigil, sigil, rest[1:]) + } if strings.HasPrefix(rest, ".") { return "", "", fmt.Errorf("a reference starts with / (the root), . (this folder) or .. (the folder above)") } @@ -137,7 +140,9 @@ func eachTemplate(root map[string]node, fn func(folder []string, path string, t } // resolveRef walks a reference path through the folders to the category it names, -// returning that head, the node, and the tail left to read into it. +// returning that head, the node, and the tail left to read into it. A descent of +// its own rather than a walkPath: it walks groups only and returns where they end, +// not a leaf. func resolveRef(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { var n node = &group{children: root} i := 0 diff --git a/template.go b/template.go index 87f2ff9..32f5f38 100644 --- a/template.go +++ b/template.go @@ -229,8 +229,60 @@ func fieldTokens(format string) []string { return names } -// checkSegments rejects an unfinished path: "{a.}", "{.b}" and "{a..b}" each have -// a segment naming nothing. A field really named "" would otherwise make them +// arm is one alternative of a {a|b} token or one operand, split into the key +// naming the node in a template's fields (a sibling field, or the head a +// reference is bound under) and the tail of a dotted path into it. A non-empty +// tail is what makes the arm a bound draw: its head is drawn once per expansion +// (see compileOps). +type arm struct { + name string // as written, for messages + key string + tail []string + steps []string // key per level passed through; the head and leaf hold their own + path string // key and tail, the one spelling every way of writing this read shares +} + +// splitArm splits one name into key and tail. refs maps a reference to what +// linkRefs bound it to; before linking, a reference is whole. +func splitArm(name string, refs map[string]refBinding) arm { + if isRef(name) { + b, bound := refs[name] + if !bound || len(b.tail) == 0 { + key := name + if bound { + key = b.key + } + return arm{name: name, key: key, path: key} + } + return pathArm(name, b.key, b.tail) + } + head, tail, dotted := strings.Cut(name, ".") + if !dotted { + return arm{name: name, key: name, path: name} + } + return pathArm(name, head, strings.Split(tail, ".")) +} + +func pathArm(name, key string, segs []string) arm { + var steps []string + for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own + steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) + } + return arm{name: name, key: key, tail: segs, steps: steps, path: key + "." + strings.Join(segs, ".")} +} + +// splitArms splits a token body's '|' alternatives. +func splitArms(body string, refs map[string]refBinding) []arm { + parts := strings.Split(body, "|") + arms := make([]arm, len(parts)) + for i, p := range parts { + arms[i] = splitArm(p, refs) + } + return arms +} + +// checkSegments rejects an unfinished path: "{a.}" and "{a..b}" each have a +// segment naming nothing. A field really named "" would otherwise make them // resolve, so a typo would read as a path that worked. func checkSegments(a arm) error { if len(a.tail) == 0 { -- 2.52.0 From 5a75df8034f8ece0b6955d96827306eb4bed1d74 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:05:54 +0200 Subject: [PATCH 30/38] Tests for a bounded, streamed --repeat, rune-named flags, and --list with inert flags --- cmd/fejkdata/main_test.go | 41 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index d8bc864..afe2a62 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "os" + "path/filepath" "strings" "testing" ) @@ -90,7 +92,7 @@ func TestRunShortFlagValues(t *testing.T) { {[]string{"-s=42", "-d", svSE, "address"}, "-s42"}, {[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"}, {[]string{"-d=" + svSE, "address"}, "--data-path="}, - {[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs a positive integer, got "d"`}, + {[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`}, {[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`}, } { code, out, errb := runOut(c.args...) @@ -316,3 +318,40 @@ func TestRunNoShippedData(t *testing.T) { t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb) } } + +func TestRunRepeatIsBounded(t *testing.T) { + for _, args := range [][]string{ + {"--repeat", "9223372036854775807", "sv_SE.word"}, + {"--repeat", "1048577", "sv_SE.word"}, + {"-n", "99999999999", "sv_SE.word"}, + } { + code, out, errb := runOut(args...) + if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") { + t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb) + } + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil { + t.Fatal(err) + } + code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x") + if code != 0 || len(out) != 1048576+1 { + t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb) + } +} + +func TestRunListTakesNoRepeatOrSeparator(t *testing.T) { + for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} { + code, _, errb := runOut(args...) + if code != 2 || !strings.Contains(errb, "--list takes no") { + t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb) + } + } +} + +func TestRunUnknownFlagIsNamedByRune(t *testing.T) { + code, _, errb := runOut("-ä", "sv_SE.word") + if code != 2 || !strings.Contains(errb, "unknown flag -ä") { + t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb) + } +} -- 2.52.0 From 3d3efbb642439589115cf28aa2fc75722c7c6294 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:06:20 +0200 Subject: [PATCH 31/38] Bound --repeat at MaxRepeat and stream the renders; name an unknown flag by its rune; --list rejects an inert --repeat or --separator --- README.md | 2 +- builtins.go | 2 +- cmd/fejkdata/main.go | 66 +++++++++++++++++++++++++------------------- fejkdata.go | 3 ++ 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9849ef3..0ac460c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ level — folders, then the category (a JSON file), then fields. | `-d`, `--data-path D` | a directory to layer over the shipped data; repeatable, the last wins a name clash | | `--no-shipped-data` | load only the `--data-path` directories | | `-s`, `--seed N` | reproducible output | -| `-n`, `--repeat N` | render the path N times, each an independent draw | +| `-n`, `--repeat N` | render the path N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | diff --git a/builtins.go b/builtins.go index dfb6ce6..c121eb7 100644 --- a/builtins.go +++ b/builtins.go @@ -14,7 +14,7 @@ import ( // fat-fingered or overflowing argument fails at New instead of trying to allocate // gigabytes — or panicking — at render. const ( - maxLen = 1 << 20 // 1,048,576 chars/bytes + maxLen = MaxRepeat maxDecimals = 1024 ) diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 0b335a1..a5514c0 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -8,6 +8,7 @@ package main import ( + "bufio" "errors" "fmt" "io" @@ -27,7 +28,7 @@ const usage = `Usage: fejkdata [flags] -h, --help print this help, then exit --list list the paths the data offers, then exit --no-shipped-data load only the --data-path directories - -n, --repeat N render the path N times (default 1) + -n, --repeat N render the path N times, 1..1048576 (default 1) -s, --seed N seed for reproducible output --separator S string between repeated values (default newline) --version print the version, then exit @@ -37,16 +38,18 @@ attaches or follows (-n3, -n 3); short flags bundle (-hn 3). ` type invocation struct { - dirs []string - help bool - list bool - noShipped bool - paths []string - repeat int - seed uint64 - seeded bool - separator string - version bool + dirs []string + help bool + list bool + noShipped bool + paths []string + repeat int + repeatSet bool + seed uint64 + seeded bool + separator string + separatorSet bool + version bool } type flagDef struct { @@ -63,10 +66,10 @@ var flagDefs = []flagDef{ {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, {"repeat", "n", true, func(in *invocation, v string) error { n, err := strconv.Atoi(v) - if err != nil || n < 1 { - return fmt.Errorf("--repeat needs a positive integer, got %q", v) + if err != nil || n < 1 || n > fejkdata.MaxRepeat { + return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v) } - in.repeat = n + in.repeat, in.repeatSet = n, true return nil }}, {"seed", "s", true, func(in *invocation, v string) error { @@ -77,7 +80,7 @@ var flagDefs = []flagDef{ in.seed, in.seeded = n, true return nil }}, - {"separator", "", true, func(in *invocation, v string) error { in.separator = v; return nil }}, + {"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }}, {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, } @@ -124,8 +127,8 @@ func splitFlags(arg string) ([]flagArg, error) { } var flags []flagArg letters := arg[1:] - for i := 0; i < len(letters); i++ { - letter := letters[i : i+1] + for i, r := range letters { + letter := string(r) def := flagByShort(letter) if def == nil { return nil, fmt.Errorf("unknown flag -%s", letter) @@ -134,7 +137,7 @@ func splitFlags(arg string) ([]flagArg, error) { flags = append(flags, flagArg{def: def}) continue } - rest := letters[i+1:] + rest := letters[i+len(letter):] if strings.HasPrefix(rest, "=") { return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s", letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:]) @@ -188,6 +191,9 @@ func (in invocation) check() error { if in.list && len(in.paths) > 0 { return errors.New("--list takes no path") } + if in.list && (in.repeatSet || in.separatorSet) { + return errors.New("--list takes no --repeat or --separator") + } if !in.list && len(in.paths) != 1 { return fmt.Errorf("expected one path, got %d", len(in.paths)) } @@ -208,17 +214,23 @@ func (in invocation) options() []fejkdata.Option { return opts } -// values renders the path repeat times, joined by the separator. -func (in invocation) values(f *fejkdata.Generator) (string, error) { - vals := make([]string, in.repeat) - for i := range vals { +// write streams the path's renders to w, repeat of them joined by the separator +// and ended by a newline. A path that renders once renders every time, so the only +// failure comes before anything is written. +func (in invocation) write(f *fejkdata.Generator, w io.Writer) error { + out := bufio.NewWriter(w) + for i := 0; i < in.repeat; i++ { v, err := f.Fake(in.paths[0]) if err != nil { - return "", err + return err } - vals[i] = v + if i > 0 { + out.WriteString(in.separator) + } + out.WriteString(v) } - return strings.Join(vals, in.separator), nil + out.WriteString("\n") + return out.Flush() } func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } @@ -254,12 +266,10 @@ func run(args []string, stdout, stderr io.Writer) int { } return 0 } - out, err := in.values(f) - if err != nil { + if err := in.write(f, stdout); err != nil { fmt.Fprintln(stderr, err) return 1 } - fmt.Fprintln(stdout, out) return 0 } diff --git a/fejkdata.go b/fejkdata.go index ff7c052..0af610a 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -28,6 +28,9 @@ import ( //go:embed data var shippedFS embed.FS +// MaxRepeat caps a repeat, and the renders nested repeats multiply to along any path. +const MaxRepeat = 1 << 20 + // ErrNoData is returned by New when no source is loaded at all. var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") -- 2.52.0 From 7b28dc36d9d2204962e8f23f9cd0739f19284cca Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:07:49 +0200 Subject: [PATCH 32/38] Tests for finite float bounds, plain integer args, constant samples and divisors, the default separator, binding keys in paths, DEL in ascii, and an unheld path --- calc_test.go | 15 +++++++++++++++ hold_test.go | 10 ++++++++++ one_spelling_test.go | 11 ++++++----- path_test.go | 14 ++++++++++++++ template_test.go | 34 +++++++++++++++++++++++----------- transform_test.go | 6 ++++++ 6 files changed, 74 insertions(+), 16 deletions(-) diff --git a/calc_test.go b/calc_test.go index b24696c..2efdf89 100644 --- a/calc_test.go +++ b/calc_test.go @@ -245,3 +245,18 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { } } } + +func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { + for _, bad := range []string{`"{calc(1/0)}"`, `"{calc(2/(1-1))}"`, `{"format":"{calc(x/y)}","x":"1","y":"0"}`, `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`} { + if _, err := compile(parse(t, bad)); err == nil || !strings.Contains(err.Error(), "zero") { + t.Errorf("compile(%s) = %v, want the constant zero divisor rejected", bad, err) + } + } + f := engine(1) + for i := 0; i < 50; i++ { + if got := mustRender(t, f, `{"format":"{calc(x/y)}","x":"1","y":["0","1"]}`); got == "+Inf" { + return + } + } + t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws") +} diff --git a/hold_test.go b/hold_test.go index 76f2053..3827d55 100644 --- a/hold_test.go +++ b/hold_test.go @@ -711,3 +711,13 @@ func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) { } } } + +func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) { + tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template) + if !ok { + t.Fatal("not a template") + } + mustPanic(t, "unheld arm with a path", func() { + readField(engine(1).rand, tm, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"}) + }) +} diff --git a/one_spelling_test.go b/one_spelling_test.go index e2d0fe1..f866609 100644 --- a/one_spelling_test.go +++ b/one_spelling_test.go @@ -30,11 +30,12 @@ func TestRepeatedChoiceItemIsRejected(t *testing.T) { func TestInertObjectIsRejected(t *testing.T) { for src, want := range map[string]string{ - `{"format":"Malmö"}`: `write "Malmö"`, - `{"format":"{digits(3)}"}`: `write "{digits(3)}"`, - `[{"format":"a","weight":1},"b"]`: "weight 1", - `{"format":"{x}","x":"v","repeat":1}`: "repeat 1", - `{"format":"{x}","x":"v","separator":","}`: "separator", + `{"format":"Malmö"}`: `write "Malmö"`, + `{"format":"{digits(3)}"}`: `write "{digits(3)}"`, + `[{"format":"a","weight":1},"b"]`: "weight 1", + `{"format":"{x}","x":"v","repeat":1}`: "repeat 1", + `{"format":"{x}","x":"v","separator":","}`: "separator", + `{"format":"{x}","x":"v","repeat":2,"separator":""}`: "default", } { if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) diff --git a/path_test.go b/path_test.go index 57a385f..b784793 100644 --- a/path_test.go +++ b/path_test.go @@ -136,3 +136,17 @@ func TestMissingFieldNamesItself(t *testing.T) { t.Errorf("Fake(person.typo) = %v, want it to name the missing field", err) } } + +func TestBindingKeyIsNotAPathSegment(t *testing.T) { + dir := writeData(t, map[string]string{ + "color": `["red","blue"]`, + "name": `{"format":"{/color} {w}","w":"x"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + if slices.Contains(f.List(), "name./color") { + t.Error("List() advertises a binding key") + } + if _, err := f.Fake("name./color"); err == nil || !strings.Contains(err.Error(), `no field "/color"`) { + t.Errorf("Fake(name./color) = %v, want a no-field error", err) + } +} diff --git a/template_test.go b/template_test.go index f1ddcb8..0ce3237 100644 --- a/template_test.go +++ b/template_test.go @@ -227,17 +227,29 @@ func TestCompileErrors(t *testing.T) { `{"format":"x","repeat":1.5}`, // non-integer repeat `{"format":"x","repeat":"two"}`, // non-numeric repeat `{"format":"x","repeat":2,"separator":5}`, // non-string separator - `"{nope()}"`, // unknown function - `"{luhn(x)}"`, // function given args it takes none of - `"{luhn(}"`, // malformed function token - `"{int(1)}"`, // wrong arity - `"{int(a,b)}"`, // non-integer args - `"{int(5,1)}"`, // min > max - `"{hex(0)}"`, // count must be positive - `"{nanoid(-1)}"`, // negative count - `"{base64(0)}"`, // count must be positive - `"{float(1,2)}"`, // wrong arity - `"{float(1,2,-1)}"`, // negative decimals + `"{nope()}"`, // unknown function + `"{luhn(x)}"`, // function given args it takes none of + `"{luhn(}"`, // malformed function token + `"{int(1)}"`, // wrong arity + `"{int(a,b)}"`, // non-integer args + `"{int(5,1)}"`, // min > max + `"{hex(0)}"`, // count must be positive + `"{nanoid(-1)}"`, // negative count + `"{base64(0)}"`, // count must be positive + `"{float(1,2)}"`, // wrong arity + `"{float(1,2,-1)}"`, // negative decimals + `"{float(NaN,NaN,2)}"`, // bounds must be finite + `"{float(Inf,Inf,2)}"`, // same-sign infinities + `"{float(1,NaN,2)}"`, // one NaN bound + `"{float(-Inf,1,2)}"`, // one infinite bound + `"{digits(+5)}"`, // a count is a plain integer + `"{digits(05)}"`, // no leading zero + `"{int(+1,5)}"`, // a bound is a plain integer + `"{int(5,5)}"`, // a constant is written as text + `"{float(1,1,2)}"`, // a constant is written as text + `{"format":"{x}","x":"v","repeat":2,"separator":""}`, // separator "" is the default + `"{calc(1/0)}"`, // a constant zero divisor + `{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor `"{iban(US)}"`, // unsupported country `"{seq(a,b)}"`, // seq takes at most one name `"{calc()}"`, // calc needs an expression diff --git a/transform_test.go b/transform_test.go index d201427..0272856 100644 --- a/transform_test.go +++ b/transform_test.go @@ -51,3 +51,9 @@ func TestTransformArgs(t *testing.T) { } } } + +func TestAsciiKeepsDEL(t *testing.T) { + if got := mustRender(t, engine(1), `{"format":"{ascii(x)}","x":"a\u007fb"}`); got != "a\u007fb" { + t.Errorf("ascii over DEL = %q, want it kept: DEL is ASCII", got) + } +} -- 2.52.0 From 4f9285ab5cdca9382d3295a50c22481c827a95b7 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:08:32 +0200 Subject: [PATCH 33/38] Reject non-finite float bounds, unplain integer args, constant samples, a constant zero divisor and the default separator; a binding key is no path segment; ascii keeps DEL; an unheld path panics --- README.md | 19 ++++++++------- builtins.go | 48 +++++++++++++++++++++++++++---------- calc.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++-- hold.go | 3 +++ node.go | 12 ++++++++++ path.go | 2 +- 6 files changed, 129 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 0ac460c..dc7e1fa 100644 --- a/README.md +++ b/README.md @@ -180,9 +180,9 @@ many times — each an independent draw — joined by `separator` (default `""`) { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -Renders e.g. `bar foo baz`. A `separator` without a `repeat` is rejected, and so -is a `repeat` that multiplies to more than 1 048 576 renders along any path of -nested repeats. +Renders e.g. `bar foo baz`. Rejected at load: a `separator` without a `repeat`, +a `separator` of `""` (the default), and a `repeat` that multiplies to more than +1 048 576 renders along any path of nested repeats. ### Options and fields @@ -194,9 +194,11 @@ choice naming its item. ### Functions A `{name(args)}` token calls a builtin. Arguments are checked at `New`: a bad -count, range, country or expression fails fast, and a length, count or decimal -place beyond a sane maximum is rejected, so a fat-fingered `hex(2000000000)` -never tries to allocate gigabytes. Every builtin draws only from the seed — a +count, range, country or expression fails fast; an integer is written plain +(`5`, not `+5` or `05`); bounds are finite; a sample that could only ever emit one +value (`int(5,5)`, `float(1,1,2)`) is rejected naming the text to write instead; +and a length, count or decimal place beyond a sane maximum is rejected, so a +fat-fingered `hex(2000000000)` never tries to allocate gigabytes. Every builtin draws only from the seed — a time-based id takes its timestamp from the rng, not the clock — so seeded output stays reproducible. @@ -243,8 +245,9 @@ hyphenated field can't be an operand. ``` Renders e.g. `19.99 x 3 = 59.97`. An operand that can never be a number (`"abc"`, -or a choice of such) is rejected at load; one that sometimes is not yields `NaN`, -and a division by zero `Inf` — both print rather than fail. +or a choice of such) is rejected at load, as is a division by a constant zero +(`1/0`, or a fixed `"0"` field); an operand that sometimes is not a number yields +`NaN`, and a division by a sometimes-zero one `Inf` — both print rather than fail. ### Transforms diff --git a/builtins.go b/builtins.go index c121eb7..c592d6e 100644 --- a/builtins.go +++ b/builtins.go @@ -176,7 +176,7 @@ var asciiFolds = map[rune]string{ func asciiFold(s string) string { var b strings.Builder for _, r := range s { - if r < unicode.MaxASCII { + if r <= unicode.MaxASCII { b.WriteRune(r) } else { b.WriteString(asciiFolds[r]) @@ -221,10 +221,25 @@ func randChars(r rng, n int, alphabet string) string { return string(b) } +// plainInt parses an integer arg written the one way: no sign, no leading zero. +func plainInt(s string) (int, error) { + n, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("%q is not an integer", s) + } + if strconv.Itoa(n) != s { + return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n) + } + return n, nil +} + func posIntArg(_ map[string]node, a []string) error { - n, err := strconv.Atoi(a[0]) - if err != nil || n < 1 { - return fmt.Errorf("count %q must be a positive integer", a[0]) + n, err := plainInt(a[0]) + if err != nil { + return fmt.Errorf("count %w", err) + } + if n < 1 { + return fmt.Errorf("count %q must be positive", a[0]) } if n > maxLen { return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen) @@ -233,14 +248,17 @@ func posIntArg(_ map[string]node, a []string) error { } func intRangeArgs(_ map[string]node, a []string) error { - lo, e1 := strconv.Atoi(a[0]) - hi, e2 := strconv.Atoi(a[1]) + lo, e1 := plainInt(a[0]) + hi, e2 := plainInt(a[1]) if e1 != nil || e2 != nil { - return fmt.Errorf("int(min,max) needs integer args, got %q,%q", a[0], a[1]) + return fmt.Errorf("int(min,max) needs plain integers, got %q,%q", a[0], a[1]) } if lo > hi { return fmt.Errorf("int(min,max): min %d > max %d", lo, hi) } + if lo == hi { + return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo) + } if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi) } @@ -250,19 +268,25 @@ func intRangeArgs(_ map[string]node, a []string) error { func floatArgs(_ map[string]node, a []string) error { lo, e1 := strconv.ParseFloat(a[0], 64) hi, e2 := strconv.ParseFloat(a[1], 64) - dp, e3 := strconv.Atoi(a[2]) + dp, e3 := plainInt(a[2]) if e1 != nil || e2 != nil || e3 != nil { - return fmt.Errorf("float(min,max,dp) needs numeric args, got %q,%q,%q", a[0], a[1], a[2]) + return fmt.Errorf("float(min,max,dp) needs numeric bounds and a plain decimals count, got %q,%q,%q", a[0], a[1], a[2]) + } + if math.IsNaN(lo) || math.IsNaN(hi) || math.IsInf(lo, 0) || math.IsInf(hi, 0) { + return fmt.Errorf("float(min,max,dp) needs finite bounds, got %q,%q", a[0], a[1]) } if lo > hi { return fmt.Errorf("float(min,max,dp): min %v > max %v", lo, hi) } - if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" - return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) - } if dp < 0 || dp > maxDecimals { return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals) } + if lo == hi { + return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64)) + } + if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" + return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) + } return nil } diff --git a/calc.go b/calc.go index 140b84d..212466f 100644 --- a/calc.go +++ b/calc.go @@ -78,14 +78,78 @@ func checkCalc(fields map[string]node, args []string) error { return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text) } } + if divisor, zero := constantZeroDivisor(expr, fields); zero { + return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor) + } if len(args) == 2 { - if dp, err := strconv.Atoi(args[1]); err != nil || dp < 0 || dp > maxDecimals { - return fmt.Errorf("calc decimals %q must be an integer in 0..%d", args[1], maxDecimals) + if dp, err := plainInt(args[1]); err != nil || dp < 0 || dp > maxDecimals { + return fmt.Errorf("calc decimals %q must be a plain integer in 0..%d", args[1], maxDecimals) } } return nil } +// constantZeroDivisor finds a division whose right side is a constant zero: number +// literals and fixed operands folded, anything that varies left unknown. +func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) { + switch n := n.(type) { + case calcNeg: + return constantZeroDivisor(n.x, fields) + case calcBin: + if n.op == '/' { + if v, known := constantValue(n.r, fields); known && v == 0 { + return calcText(n.r), true + } + } + if d, zero := constantZeroDivisor(n.l, fields); zero { + return d, true + } + return constantZeroDivisor(n.r, fields) + } + return "", false +} + +// constantValue evaluates an expression whose every operand is fixed. +func constantValue(n calcNode, fields map[string]node) (float64, bool) { + switch n := n.(type) { + case calcNum: + return float64(n), true + case calcVar: + t, ok := fields[string(n)].(*template) + if !ok || !t.fixed || t.repeat > 1 { + return 0, false + } + v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64) + return v, err == nil + case calcNeg: + v, ok := constantValue(n.x, fields) + return -v, ok + case calcBin: + l, lok := constantValue(n.l, fields) + r, rok := constantValue(n.r, fields) + if !lok || !rok { + return 0, false + } + return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true + } + return 0, false +} + +// calcText spells an expression node the way an author would read it. +func calcText(n calcNode) string { + switch n := n.(type) { + case calcNum: + return strconv.FormatFloat(float64(n), 'f', -1, 64) + case calcVar: + return string(n) + case calcNeg: + return "-" + calcText(n.x) + case calcBin: + return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")" + } + return "?" +} + // neverNumeric reports a node no render of which is a number: fixed text that does // not parse, or a choice of only such items. text is one such render. func neverNumeric(n node) (text string, never bool) { diff --git a/hold.go b/hold.go index a900a7c..bb5c3fe 100644 --- a/hold.go +++ b/hold.go @@ -273,6 +273,9 @@ type draws struct { // linkRefs prove every step, so the walk cannot fail. func readField(s *session, t *template, held *draws, a arm) string { if !t.held[a.key] { + if len(a.tail) > 0 { + panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key)) + } return render(s, t.fields[a.key]) } if v, read := held.value[a.path]; read { diff --git a/node.go b/node.go index 54b44a3..bbe80dd 100644 --- a/node.go +++ b/node.go @@ -57,6 +57,15 @@ type template struct { func (*template) isNode() {} +// field is the node a path segment names; a binding is a render edge, not a field. +func (t *template) field(seg string) (node, bool) { + if isRef(seg) { + return nil, false + } + n, ok := t.fields[seg] + return n, ok +} + // compile converts parsed JSON into a node tree, validating structure up front. // Only a choice's items carry a weight, so one here would be inert whatever its type. func compile(v any) (node, error) { @@ -229,6 +238,9 @@ func readOptions(m map[string]any) (templateOptions, error) { if repeat == 1 { return o, fmt.Errorf("separator joins repeated renders, so it has no effect without a repeat above 1") } + if o.separator == "" { + return o, fmt.Errorf("separator \"\" is the default, so it has no effect; drop it") + } } _, o.weighted = m["weight"] return o, nil diff --git a/path.go b/path.go index cc3edff..ee9b1c5 100644 --- a/path.go +++ b/path.go @@ -40,7 +40,7 @@ func walkPath(n node, tail []string, w pathWalk) error { return err } } - child, ok := n.fields[tail[0]] + child, ok := n.field(tail[0]) if !ok { return fmt.Errorf("no field %q", tail[0]) } -- 2.52.0 From ae0527a26e8b78f44e7452f746a86df3e3aa7867 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:09:19 +0200 Subject: [PATCH 34/38] Tests for the real data-path error, an empty data path, and a crypto/rand failure --- cmd/fejkdata/main_test.go | 7 +++++++ fejkdata_test.go | 24 ++++++++++++++++++++++-- template_test.go | 8 +++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index afe2a62..cb08507 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -355,3 +355,10 @@ func TestRunUnknownFlagIsNamedByRune(t *testing.T) { t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb) } } + +func TestRunEmptyDataPathIsNamed(t *testing.T) { + code, _, errb := runOut("--data-path=", "sv_SE.word") + if code != 1 || !strings.Contains(errb, "empty") { + t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb) + } +} diff --git a/fejkdata_test.go b/fejkdata_test.go index d85401f..51e8e24 100644 --- a/fejkdata_test.go +++ b/fejkdata_test.go @@ -1,8 +1,11 @@ package fejkdata import ( + "errors" + "io/fs" "strings" "testing" + "testing/fstest" ) // newGenerator creates a generator over a single data directory, failing on @@ -38,8 +41,25 @@ func fake(t *testing.T, f *Generator, path string) string { func TestNewMissingDirectory(t *testing.T) { _, err := New(WithoutShippedData(), WithDataPath("data/de_DE")) - if err == nil || !strings.Contains(err.Error(), "de_DE") { - t.Fatalf("New(missing) error = %v, want it to name the path", err) + if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("New(missing) error = %v, want the real error, naming the path", err) + } +} + +func TestNewRejectsAnEmptyDataPath(t *testing.T) { + _, err := New(WithoutShippedData(), WithDataPath("")) + if err == nil || !strings.Contains(err.Error(), "empty") { + t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err) + } +} + +func TestNewReportsAnEntropyFailure(t *testing.T) { + saved := randomBytes + randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") } + defer func() { randomBytes = saved }() + _, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}})) + if err == nil || !strings.Contains(err.Error(), "no entropy") { + t.Fatalf("New() without entropy = %v, want the failure reported", err) } } diff --git a/template_test.go b/template_test.go index 0ce3237..d5aa2ad 100644 --- a/template_test.go +++ b/template_test.go @@ -8,7 +8,13 @@ import ( ) // engine builds a seeded generator with no loaded categories, for rendering tests. -func engine(seed uint64) *Generator { return &Generator{rand: newRand(seed, true)} } +func engine(seed uint64) *Generator { + s, err := newRand(seed, true) + if err != nil { + panic(err) + } + return &Generator{rand: s} +} // parse unmarshals a JSON template fragment into its dynamic form. func parse(t *testing.T, s string) any { -- 2.52.0 From 08c7ef9b7f8b4de3b509dec8b36b94bac6e009bb Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:09:29 +0200 Subject: [PATCH 35/38] Keep the real data-path error, reject an empty data path, and report a crypto/rand failure instead of seeding zero --- data.go | 20 ++++++++++++-------- fejkdata.go | 25 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/data.go b/data.go index 3d59bb6..fd2b3f1 100644 --- a/data.go +++ b/data.go @@ -10,13 +10,14 @@ import ( ) // dataSource is one tree to load: an fs.FS and the directory in it to start from. -// label prefixes file names in errors; path, when set, is a directory on disk that -// must exist. +// label prefixes file names in errors; onDisk marks path as a directory that must +// exist. type dataSource struct { - fsys fs.FS - label string - path string - root string + fsys fs.FS + label string + onDisk bool + path string + root string } func (s dataSource) name(p string) string { @@ -36,10 +37,13 @@ func (s dataSource) name(p string) string { func loadData(sources []dataSource) (map[string]node, error) { root := map[string]node{} for _, src := range sources { - if src.path != "" { + if src.onDisk { + if src.path == "" { + return nil, fmt.Errorf("a data path is empty") + } info, err := os.Stat(src.path) if err != nil { - return nil, fmt.Errorf("%s: no such directory", src.path) + return nil, err } if !info.IsDir() { return nil, fmt.Errorf("%s is not a directory", src.path) diff --git a/fejkdata.go b/fejkdata.go index 0af610a..599a73d 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -75,7 +75,7 @@ func WithSeed(seed uint64) Option { // layer several; the last wins a name clash. func WithDataPath(dir string) Option { return func(c *config) { - c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, path: dir}) + c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir}) } } @@ -112,7 +112,11 @@ func New(opts ...Option) (*Generator, error) { if err != nil { return nil, fmt.Errorf("fejkdata: %w", err) } - return &Generator{rand: newRand(c.seed, c.seeded), categories: cats}, nil + rng, err := newRand(c.seed, c.seeded) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + return &Generator{rand: rng, categories: cats}, nil } // List returns the sorted dotted paths Fake can render: every category, the dotted @@ -203,12 +207,19 @@ func join(prefix, name string) string { return prefix + "." + name } -func newRand(seed uint64, seeded bool) *session { - r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) - if !seeded { +// randomBytes seeds an unseeded generator; a failure is an error, never a fixed seed. +var randomBytes = crand.Read + +func newRand(seed uint64, seeded bool) (*session, error) { + var r *rand.Rand + if seeded { + r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) + } else { var b [16]byte - _, _ = crand.Read(b[:]) + if _, err := randomBytes(b[:]); err != nil { + return nil, fmt.Errorf("seeding from crypto/rand: %w", err) + } r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:]))) } - return &session{Rand: r, counters: map[string]uint64{}} + return &session{Rand: r, counters: map[string]uint64{}}, nil } -- 2.52.0 From c5304bb6e78105e0a5234d862120b383f50954e0 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:09:50 +0200 Subject: [PATCH 36/38] Record the repeat cap's reach, 64-bit only, the constant-divisor rule and the rejected default spellings; correct the repeat cap comment --- README.md | 16 ++++++++++++++++ node.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dc7e1fa..fe7515e 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,22 @@ tokens add cost in proportion to the output. when that file lacks a field those references read. Accepted: overriding is the point of layering, the error names the reference and the field, and the fix is the consumer's file carrying the fields the shipped tree reads. +- **The repeat cap bounds renders, not bytes.** A repeat, alone or nested, may + ask for at most 1 048 576 renders; how large each render is stays what the data + asked for, so `{hex(1048576)}` repeated to the cap is a terabyte, loaded without + complaint. A byte estimate would need every builtin to declare a width to fence + a shape no data comes near, and the harm lands on the author who wrote it. +- **64-bit targets only.** The gate builds amd64, and the buffer sizing a render + pre-computes (renders × bytes) assumes a 64-bit int; on a 32-bit target it could + overflow and panic. +- **A constant zero divisor is a load error; a sometimes-zero one prints `Inf`.** + `1/0` and a fixed `"0"` field are decidable, so they join the never-numeric + operand as a load error; a divisor that is only sometimes zero is data, and + `Inf` printing visibly is the never-fail rule. +- **A default written out, and a constant spelled as a sample, are load errors.** + `weight: 1`, `repeat: 1`, `separator: ""`, `int(5,5)`, `float(1,1,2)`, `+5` + and `05` each spell what a shorter form already spells, so each is rejected + naming that form. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. diff --git a/node.go b/node.go index bbe80dd..c1ecf04 100644 --- a/node.go +++ b/node.go @@ -288,7 +288,7 @@ func repeatOf(m map[string]any) (int, error) { if r == 1 { return 0, fmt.Errorf("repeat 1 is the default, so it has no effect; drop it") } - if r > maxLen { // cap so a fat-fingered repeat can't build a multi-GB string + if r > maxLen { // caps the renders one repeat asks for; checkRepeatReach bounds what nested ones multiply to return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, maxLen) } return int(r), nil -- 2.52.0 From a9552ed96647fe88b6b5237520019f5bc45ef7e8 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:24:46 +0200 Subject: [PATCH 37/38] Tests for every branch of the constant-divisor fold, arg errors naming the spelling, and shell numbers at the CLI --- calc_test.go | 16 +++++++++++++--- cmd/fejkdata/main_test.go | 8 ++++++++ template_test.go | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/calc_test.go b/calc_test.go index 2efdf89..77adf77 100644 --- a/calc_test.go +++ b/calc_test.go @@ -247,11 +247,21 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { } func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { - for _, bad := range []string{`"{calc(1/0)}"`, `"{calc(2/(1-1))}"`, `{"format":"{calc(x/y)}","x":"1","y":"0"}`, `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`} { - if _, err := compile(parse(t, bad)); err == nil || !strings.Contains(err.Error(), "zero") { - t.Errorf("compile(%s) = %v, want the constant zero divisor rejected", bad, err) + for src, want := range map[string]string{ + `"{calc(1/0)}"`: "divides by 0", + `"{calc(2/(1-1))}"`: "divides by (1 - 1)", + `{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y", + `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)", + `{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0", + `{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want) } } + if _, err := compile(parse(t, `{"format":"{calc(a/(b*c))}","a":"1","b":"0","c":["1","2"]}`)); err != nil { + t.Errorf("compile(a/(b*c)) = %v, want a divisor that varies accepted", err) + } f := engine(1) for i := 0; i < 50; i++ { if got := mustRender(t, f, `{"format":"{calc(x/y)}","x":"1","y":["0","1"]}`); got == "+Inf" { diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index cb08507..acc5a23 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -362,3 +362,11 @@ func TestRunEmptyDataPathIsNamed(t *testing.T) { t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb) } } + +func TestRunNumbersFollowTheShell(t *testing.T) { + _, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word") + code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word") + if code != 0 || got != want { + t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want) + } +} diff --git a/template_test.go b/template_test.go index d5aa2ad..74dbc30 100644 --- a/template_test.go +++ b/template_test.go @@ -365,3 +365,17 @@ func TestAlternationThreeWay(t *testing.T) { t.Fatalf("3-way alternation produced %v, want A, B and C", seen) } } + +func TestArgErrorsNameTheSpelling(t *testing.T) { + for src, want := range map[string]string{ + `"{float(1,2,02)}"`: "write 2", + `{"format":"{calc(a,02)}","a":"1"}`: "write 2", + `"{digits(+5)}"`: "write 5", + `"{hex(99999999999999999999)}"`: "exceeds the maximum", + `"{int(007,9)}"`: "write 7", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error saying %q", src, err, want) + } + } +} -- 2.52.0 From addba9abf70d32f4b471ebdd14cd40052123b270 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 2 Sep 2026 19:25:19 +0200 Subject: [PATCH 38/38] Give maxLen its own literal and repeats MaxRepeat; arg errors name the spelling and the range; write's comment says what Flush reports; assert 64-bit at compile time --- README.md | 21 ++++++++++++--------- builtins.go | 37 +++++++++++++++++++++++++------------ calc.go | 8 ++++++-- cmd/fejkdata/main.go | 5 +++-- fejkdata.go | 7 +++++-- graph.go | 4 ++-- node.go | 4 ++-- 7 files changed, 55 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index fe7515e..7cd0dbc 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,8 @@ hyphenated field can't be an operand. Renders e.g. `19.99 x 3 = 59.97`. An operand that can never be a number (`"abc"`, or a choice of such) is rejected at load, as is a division by a constant zero (`1/0`, or a fixed `"0"` field); an operand that sometimes is not a number yields -`NaN`, and a division by a sometimes-zero one `Inf` — both print rather than fail. +`NaN`, and a division by one that is not constant `Inf` — both print rather than +fail. ### Transforms @@ -390,14 +391,16 @@ tokens add cost in proportion to the output. - **64-bit targets only.** The gate builds amd64, and the buffer sizing a render pre-computes (renders × bytes) assumes a 64-bit int; on a 32-bit target it could overflow and panic. -- **A constant zero divisor is a load error; a sometimes-zero one prints `Inf`.** - `1/0` and a fixed `"0"` field are decidable, so they join the never-numeric - operand as a load error; a divisor that is only sometimes zero is data, and - `Inf` printing visibly is the never-fail rule. -- **A default written out, and a constant spelled as a sample, are load errors.** - `weight: 1`, `repeat: 1`, `separator: ""`, `int(5,5)`, `float(1,1,2)`, `+5` - and `05` each spell what a shorter form already spells, so each is rejected - naming that form. +- **A constant zero divisor is a load error; a divisor that is not constant prints + `Inf`.** `1/0` and a fixed `"0"` field are decidable, so they join the + never-numeric operand as a load error; the fold stops where an operand varies, + so `a/(b*c)` with `b` fixed at `0` and `c` varying loads and prints `Inf` every + draw — catching it needs zero-absorbing algebra for a shape nobody writes. +- **In data, a default written out and a constant spelled as a sample are load + errors.** `weight: 1`, `repeat: 1`, `separator: ""`, `int(5,5)`, `float(1,1,2)`, + `+5` and `05` each spell what a shorter form already spells, so each is rejected + naming that form. The CLI's numbers follow the shell instead: `--seed 007` and + `--repeat +3` are 7 and 3, as every command line reads them. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. diff --git a/builtins.go b/builtins.go index c592d6e..d72ce2e 100644 --- a/builtins.go +++ b/builtins.go @@ -2,6 +2,7 @@ package fejkdata import ( "encoding/base64" + "errors" "fmt" "math" "strconv" @@ -9,12 +10,12 @@ import ( "unicode" ) -// maxLen caps sample output lengths (hex, nanoid, base64) and the renders a repeat -// multiplies to along any path; maxDecimals caps float/calc decimal places. So a -// fat-fingered or overflowing argument fails at New instead of trying to allocate -// gigabytes — or panicking — at render. +// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower) +// and maxDecimals float/calc decimal places, so a fat-fingered or overflowing +// argument fails at New instead of trying to allocate gigabytes — or panicking — +// at render. const ( - maxLen = MaxRepeat + maxLen = 1 << 20 maxDecimals = 1024 ) @@ -224,6 +225,9 @@ func randChars(r rng, n int, alphabet string) string { // plainInt parses an integer arg written the one way: no sign, no leading zero. func plainInt(s string) (int, error) { n, err := strconv.Atoi(s) + if errors.Is(err, strconv.ErrRange) { + return 0, fmt.Errorf("%q is past the integer range: %w", s, err) + } if err != nil { return 0, fmt.Errorf("%q is not an integer", s) } @@ -235,6 +239,9 @@ func plainInt(s string) (int, error) { func posIntArg(_ map[string]node, a []string) error { n, err := plainInt(a[0]) + if errors.Is(err, strconv.ErrRange) { + return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen) + } if err != nil { return fmt.Errorf("count %w", err) } @@ -248,10 +255,13 @@ func posIntArg(_ map[string]node, a []string) error { } func intRangeArgs(_ map[string]node, a []string) error { - lo, e1 := plainInt(a[0]) - hi, e2 := plainInt(a[1]) - if e1 != nil || e2 != nil { - return fmt.Errorf("int(min,max) needs plain integers, got %q,%q", a[0], a[1]) + lo, err := plainInt(a[0]) + if err != nil { + return fmt.Errorf("int(min,max): min %w", err) + } + hi, err := plainInt(a[1]) + if err != nil { + return fmt.Errorf("int(min,max): max %w", err) } if lo > hi { return fmt.Errorf("int(min,max): min %d > max %d", lo, hi) @@ -268,9 +278,12 @@ func intRangeArgs(_ map[string]node, a []string) error { func floatArgs(_ map[string]node, a []string) error { lo, e1 := strconv.ParseFloat(a[0], 64) hi, e2 := strconv.ParseFloat(a[1], 64) - dp, e3 := plainInt(a[2]) - if e1 != nil || e2 != nil || e3 != nil { - return fmt.Errorf("float(min,max,dp) needs numeric bounds and a plain decimals count, got %q,%q,%q", a[0], a[1], a[2]) + if e1 != nil || e2 != nil { + return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1]) + } + dp, err := plainInt(a[2]) + if err != nil { + return fmt.Errorf("float(min,max,dp): decimals %w", err) } if math.IsNaN(lo) || math.IsNaN(hi) || math.IsInf(lo, 0) || math.IsInf(hi, 0) { return fmt.Errorf("float(min,max,dp) needs finite bounds, got %q,%q", a[0], a[1]) diff --git a/calc.go b/calc.go index 212466f..cd776b7 100644 --- a/calc.go +++ b/calc.go @@ -82,8 +82,12 @@ func checkCalc(fields map[string]node, args []string) error { return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor) } if len(args) == 2 { - if dp, err := plainInt(args[1]); err != nil || dp < 0 || dp > maxDecimals { - return fmt.Errorf("calc decimals %q must be a plain integer in 0..%d", args[1], maxDecimals) + dp, err := plainInt(args[1]) + if err != nil { + return fmt.Errorf("calc decimals %w", err) + } + if dp < 0 || dp > maxDecimals { + return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals) } } return nil diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index a5514c0..d6b2d05 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -215,8 +215,9 @@ func (in invocation) options() []fejkdata.Option { } // write streams the path's renders to w, repeat of them joined by the separator -// and ended by a newline. A path that renders once renders every time, so the only -// failure comes before anything is written. +// and ended by a newline. A path that renders once renders every time, so a Fake +// failure comes before anything is written; a write failure surfaces from Flush, +// bufio keeping the first one. func (in invocation) write(f *fejkdata.Generator, w io.Writer) error { out := bufio.NewWriter(w) for i := 0; i < in.repeat; i++ { diff --git a/fejkdata.go b/fejkdata.go index 599a73d..6cfdc01 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -28,9 +28,12 @@ import ( //go:embed data var shippedFS embed.FS -// MaxRepeat caps a repeat, and the renders nested repeats multiply to along any path. +// MaxRepeat caps a repeat, and the renders nested repeats multiply to along any +// path; the CLI's --repeat shares it. const MaxRepeat = 1 << 20 +var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions + // ErrNoData is returned by New when no source is loaded at all. var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") @@ -207,7 +210,7 @@ func join(prefix, name string) string { return prefix + "." + name } -// randomBytes seeds an unseeded generator; a failure is an error, never a fixed seed. +// randomBytes seeds an unseeded generator. var randomBytes = crand.Read func newRand(seed uint64, seeded bool) (*session, error) { diff --git a/graph.go b/graph.go index 4e2d09a..887c0ec 100644 --- a/graph.go +++ b/graph.go @@ -181,8 +181,8 @@ func checkRepeatReach(root map[string]node) error { return r } return walkNodes(root, func(path string, n node) error { - if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > maxLen { - return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), maxLen) + if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > MaxRepeat { + return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), MaxRepeat) } return nil }) diff --git a/node.go b/node.go index c1ecf04..fb6aee9 100644 --- a/node.go +++ b/node.go @@ -288,8 +288,8 @@ func repeatOf(m map[string]any) (int, error) { if r == 1 { return 0, fmt.Errorf("repeat 1 is the default, so it has no effect; drop it") } - if r > maxLen { // caps the renders one repeat asks for; checkRepeatReach bounds what nested ones multiply to - return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, maxLen) + if r > MaxRepeat { // caps the renders one repeat asks for; checkRepeatReach bounds what nested ones multiply to + return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, MaxRepeat) } return int(r), nil } -- 2.52.0