diff --git a/.gitignore b/.gitignore index 71fd48f..28f4f54 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .claude *.out +__pycache__/ todo.md \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..75d511e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,18 @@ +# Rules + +- Go runs only through `docker compose run --rm `; the merge gate is `docker build .`. +- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output is its own commit. +- One-line commit messages: no ticket prefix, no repo name, no authorship trailers. +- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble. +- One spelling per result: reject the other at `New`, and let the error name the spelling to use. +- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. +- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. +- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`eachToken`, `calc.factor`, `walkPath`) sit at 13–14 and stay whole; anything else that reaches 14 is decomposed. + +# Deferred + +- **Shipped-data de-duplication (2026-09-02).** `email.json`'s `local` is a + drifted copy of `username.json`, and no shipped file yet uses a held path, an + operand or a reference. Fixed in the data fill before the first tag, when the + shipped set is rewritten anyway; premise: nothing depends on the shipped data's + shape until then. Not raised in review before that. diff --git a/Dockerfile b/Dockerfile index 63be7a8..8f04a4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN go mod download COPY . . RUN go vet ./... && \ + go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \ { unformatted="$(gofmt -l .)"; test -z "$unformatted" || \ { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \ - go test ./... + go test -race ./... diff --git a/README.md b/README.md index 13aeb6f..7cd0dbc 100644 --- a/README.md +++ b/README.md @@ -1,68 +1,66 @@ # fejkdata +A Go library and CLI for generating locale-aware fake data from JSON templates. Forked from [github.com/Timewave-AB/fakes](https://github.com/Timewave-AB/fakes). -A Go library and CLI for generating fake data, built for -**internationalization**. It exists because the existing Go fake libraries -lacked the locale coverage and format control we needed. +## Goals -- **Standards first** — formats, names and structures follow international and - local standards first and foremost. -- **Locale-aware** — names, addresses, postal codes and phone numbers follow - per-locale data and formats. The shipped data is organised by full locale tag - (`sv_SE`), but the engine treats folders as plain namespaces — name yours - anything. -- **Data lives in JSON** — all source data is recursive JSON on disk, read when - you create a generator and then served from memory. Add or change data without - touching the library. Behavior belongs in data too: the engine grows a - built-in function only for what data can't express (a checksum, a time-based - id), never for what character classes and choices already do. -- **Composable** — templates nest without limit: weighted choices, character - classes and sub-templates combine to model any format. -- **Reproducible** — seed a generator and it emits the same sequence every time, - for a given version of the data: changing how a value is composed shifts the - stream for that value and for everything drawn after it in the same generator. - Every built-in draws only from that seed — no wall-clock, no `crypto/rand` — - so determinism holds end to end. -- **Zero dependencies** — standard library only. +1. **Valid by construction** — every value passes the check its real consumer + applies; facts that belong together come from one draw, within a value and + across categories. +2. **Text means what it says** — a format renders as written; only `{…}` varies, + random characters included (`{digits(3)}`). One spelling per result; the wrong + one is a load error naming the right one. +3. **Every mistake is a load error** — `New` rejects; `Fake` on a loaded generator + fails only for an unknown path. +4. **Zero to a value in one command** — `go install`, then `fejkdata sv_SE.person`: + no checkout, no flag. Flags are GNU-form (`--seed 42`, `-n 3`) in any position; + the first custom template needs no escape and no option. +5. **Data lives in JSON** — a builtin only for what data can't express. +6. **Reproducible** — seed in, same stream out; no builtin reads a clock. +7. **Zero dependencies** — standard library only. +8. **Docs index the grammar** — every syntax feature is a heading; every example + runs under test and shows its output; a rule is stated once. ## CLI -Install the `fejkdata` command, then give it one or more `-data-path` directories -and a path — it prints one value to stdout. Each dot segment descends one level: -folders, then the category (a JSON file), then fields inside it. - ```sh go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest -fejkdata -data-path ./data/sv_SE person # Sara Eriksson -fejkdata -data-path ./data/sv_SE person.last # Eriksson (dotted path into a category) -fejkdata -data-path ./data sv_SE.person # point at the tree; the folder is a segment -fejkdata -data-path ./data/sv_SE -data-path ./mydata word # layer dirs; the last wins a name clash -fejkdata -seed 42 -data-path ./data/sv_SE address -fejkdata -repeat 3 -data-path ./data/sv_SE person # three values, one per line -fejkdata -repeat 3 -separator ', ' -data-path ./data/sv_SE word # nät, barn, sol -fejkdata -data-path ./data/sv_SE -list # every path this data offers +fejkdata sv_SE.person # Sara Eriksson +fejkdata sv_SE.person.last # Eriksson +fejkdata --seed 42 sv_SE.address # the same address every run +fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol +fejkdata --list # every path the data offers +fejkdata --data-path ./mydata sv_SE.word # layer a directory over the shipped data +fejkdata --no-shipped-data -d ./mydata --list # only your data ``` -`-data-path` is repeatable (last wins a name clash) and the path comes last — -all flags must precede it. `-repeat N` renders the path N times — each an -independent draw — joined by `-separator` (default a newline, so values land one -per line). Not sure what a data set offers? `-list` prints every path you can ask -for; `-version` prints the build version. +A path names a category, or a field inside one: each dot segment descends one +level — folders, then the category (a JSON file), then fields. -Without installing, run it from a checkout with `go run ./cmd/fejkdata …`. Exit -codes: `0` success (including `-list`, `-version`, `-h`), `1` runtime error -(missing dir, unknown path), `2` misuse. +| Flag | | +|------|--| +| `-d`, `--data-path D` | a directory to layer over the shipped data; repeatable, the last wins a name clash | +| `--no-shipped-data` | load only the `--data-path` directories | +| `-s`, `--seed N` | reproducible output | +| `-n`, `--repeat N` | render the path N times (up to 1048576), each an independent draw, streamed | +| `--separator S` | between repeated values (default a newline) | +| `--list` | print every path, then exit | +| `--version`, `-h`, `--help` | print, then exit | -### Generating a file from a custom template +`--name value` and `--name=value` both work, a short flag's value attaches or +follows (`-n3`, `-n 3`) and short flags bundle (`-hn 3`) — see +[Decisions](#decisions); flags go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing +dir, unknown path), `2` misuse. From a checkout: `go run ./cmd/fejkdata …`. -A category is just a JSON file in a data directory, so you can drop in your -own and render it — no code change. Save this as `data/sv_SE/sql.json`: +### Your own data + +A category is a JSON file in a directory. Save this as `mydata/sql.json`: ```json { - "format": "INSERT INTO users V#ALUES({sql-username});", + "format": "INSERT INTO users VALUES({sql-username});", "sql-username": { "format": "'{username}'", "repeat": 3, @@ -72,408 +70,340 @@ own and render it — no code change. Save this as `data/sv_SE/sql.json`: } ``` -`sql-username` renders `'{username}'` `repeat` times and joins the results with -the `),(` separator; the outer `V#ALUES(…)` wraps that into one valid row list. -(`#A` escapes the literal `A`, which a format string would otherwise read as a -letter token — see [Data format](#data-format).) - ```sh -fejkdata -seed 1 -data-path ./data/sv_SE sql +fejkdata --seed 1 --data-path ./mydata sql # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); -``` - -Raise the template's `repeat` for more rows per statement; use the CLI's -`-repeat` for more statements — together they build a whole seed file: - -```sh -fejkdata -repeat 100 -data-path ./data/sv_SE sql > seed.sql +fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` ## Library ```sh -go get gitea.larvit.se/larvit/fejkdata # requires Go 1.22+ (for math/rand/v2) +go get gitea.larvit.se/larvit/fejkdata # Go 1.22+ ``` -Point `New` at one or more data directories, then generate values by path with -`Fake`. Each dot segment descends one level: folders, then the category (a JSON -file), then fields inside it. - ```go -package main - -import ( - "fmt" - "log" - - "gitea.larvit.se/larvit/fejkdata" -) - -func main() { - f, err := fejkdata.New([]string{"./data/sv_SE"}) - if err != nil { - log.Fatal(err) - } - - for _, path := range []string{"person", "address", "phone", "address.locality"} { - v, err := f.Fake(path) - if err != nil { - log.Fatal(err) - } - fmt.Printf("%-18s %s\n", path, v) - } +f, err := fejkdata.New(fejkdata.WithSeed(42)) +if err != nil { + log.Fatal(err) } +v, err := f.Fake("sv_SE.address") // "Kungsvägen 68\n379 17 Stockholm" +paths := f.List() // every path Fake accepts, sorted ``` -``` -person Sara Eriksson -address Kungsvägen 68 - 379 17 Stockholm -phone 072-402 91 67 -address.locality Linköping -``` +| Option | | +|--------|--| +| `WithSeed(n)` | same seed, same data: identical sequence | +| `WithDataPath(dir)` | layer a directory; repeat to layer several, the last wins a clash | +| `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | +| `WithoutShippedData()` | load only what you give | -Seed a generator for reproducible output — same seed + locale yields an identical -sequence, handy for stable tests: - -```go -a, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -b, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -av, _ := a.Fake("person") -bv, _ := b.Fake("person") -av == bv // true -``` - -`f.List()` returns the sorted paths the loaded data offers — the categories, their -dotted fields and folder segments (what the CLI's `-list` prints). Every path it -lists renders. - -A `*Generator` is **not** safe for concurrent use — create one per goroutine. +A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only +when drawn from one goroutine. Changing how a value is composed shifts the seeded +stream for that value and everything drawn after it. ## Data -The library ships a ready-to-use set under [`data/`](data): one folder per locale -(`en_US`, `sv_SE`) plus a locale-neutral `misc` folder. Point either tool at the -whole tree, a single folder, a copy, or your own directory — anywhere on disk. A -category or folder name must not use `.`, `|`, `(` or `}` (see [Data -format](#data-format)), and dot-prefixed entries are skipped, so a data directory -can also be a checkout. +The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) +plus a locale-neutral `misc` folder — is embedded, so the CLI and the library +work with no data on disk. A directory is a namespace: each JSON file is a +category named after the file, each subdirectory a dot-path segment, so +`mydata/sv_SE/person.json` is `sv_SE.person` and replaces the shipped one. +Sources merge in order; matching folders combine, any other clash is won by the +last loaded. Names may not use `.`, `|`, `(`, `{`, `}` or `/`; dot-prefixed entries +are skipped, so a data directory can also be a checkout. -A directory is just a namespace. Each JSON file is a category named after the -file; each subdirectory is a dot-path segment — folders nest exactly like JSON -objects do. So `data/sv_SE/person.json` is `Fake("person")` when you point at -`data/sv_SE`, or `Fake("sv_SE.person")` when you point at `data`. - -Pass several directories and they merge, left to right: matching folders combine -by their children, and any other clash is won by the last directory loaded. That -lets you layer your own data over the built-ins without copying them: - -```go -fejkdata.New([]string{"./data/sv_SE", "./mydata"}) // mydata overrides on a clash -``` - -Each shipped locale carries these categories, formatted per locale (e.g. `date` -is `MM/DD/YYYY` in `en_US`, `YYYY-MM-DD` in `sv_SE`; `ssn` is a US SSN vs a -Swedish personnummer): `address`, `color`, `company`, `date`, `email`, `ip`, +Each locale carries `address`, `color`, `company`, `date`, `email`, `ip`, `person`, `phone`, `price`, `sentence`, `ssn`, `time`, `url`, `username`, -`version`, `word`. - -`data/misc` carries locale-neutral categories — universal data that isn't tied to -a language or region: - -- ids & networking: `uuid` (a proper random v4), `mac`, `objectid` (24 hex chars, - MongoDB ObjectID-shaped — the leading bytes are random, not a real timestamp), - `creditcard` (per-network numbers ending in a valid `{luhn()}` digit) -- reference codes: `currency` (ISO 4217), `country` (ISO 3166), `language` (ISO - 639), `timezone` (IANA) -- web & systems: `mimetype`, `httpstatus`, `useragent` -- misc: `coordinate` (a lat/long point), `emoji`, `car` - -Many carry dotted sub-fields — `currency.symbol`, `country.alpha2`, -`mimetype.ext`, `httpstatus.code`, `car.maker`. A time-ordered v7 UUID can't be -expressed as data, so it's the `{uuid()}` builtin instead (see **Functions**). +`version` and `word`, formatted per locale. `misc` carries `car`, `coordinate`, +`country` (ISO 3166), `creditcard` (Luhn-valid), `currency` (ISO 4217), `emoji`, +`httpstatus`, `language` (ISO 639), `mac`, `mimetype`, `objectid`, `timezone` +(IANA), `useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, +`misc.country.alpha2`, `misc.httpstatus.code` — which `--list` shows. ## Data format -Each JSON file in a data directory is a **category** named after the file -(`address.json` → `address`), rendered by `Fake("address")`. Drop in a new file -or folder — no code change, no recompile. +Every value is a **node**, nestable without limit: -Every value is a **node**, one of three shapes, nestable without limit: - -| Node | JSON | Meaning | +| Node | JSON | Renders | |------|------|---------| -| literal | `"Malmö"` | emitted verbatim — never formatted | -| choice | `["a", "b", …]` | one element, picked at random | -| template | `{"format": "…", …}` | a format string plus the named sub-nodes it references | +| string | `"Malmö"` | its text, with any `{…}` tokens expanded | +| choice | `["a", "b", …]` | one item, picked at random | +| template | `{"format": "…", …}` | its format, with `{name}` tokens rendering the named fields | -**Weight.** A template node may carry a `weight` (default `1`) to skew its odds -within a choice: +### Format string + +Every character is literal except a `{…}` token: + +| Token | Renders | +|-------|---------| +| `{name}` | the sibling field `name` | +| `{name.field}` | `field` of one draw of `name` ([Correlated fields](#correlated-fields)) | +| `{a\|b}` | one of the named fields, even odds | +| `{fn(args)}` | a builtin ([Functions](#functions)) | +| `{/path}`, `{.path}`, `{..path}` | a node reached from the data root, this file's folder, or the folder above ([References](#references)) | +| `{{`, `}}` | a literal `{` or `}` | + +```json +"100 Main St, Apt {int(1,9)}{upper(1)} — tel {digits(3)}-{digits(4)}" +``` + +Renders e.g. `100 Main St, Apt 4B — tel 555-0199`. Random characters come from +the sample functions, so a phone pattern is `070-{digits(3)} {digits(2)} {digits(2)}`. +A lone `}` is a load error naming `}}`; the arms of `{a|b}` must differ, a +repeated arm being a second spelling of [weight](#weight). + +### Weight + +An item of a choice may carry a `weight` (default `1`) to skew its odds: ```json [ - { "format": "#070-000 00 00", "weight": 10 }, - { "format": "#01-000 00 00" }, - { "format": "#010-000 00 00" } + { "format": "070-{digits(3)} {digits(2)} {digits(2)}", "weight": 10 }, + "08-{digits(3)} {digits(2)} {digits(2)}", + { "format": "010-{digits(3)} {digits(2)} {digits(2)}", "weight": 2 } ] ``` -Only template (object) nodes carry `weight` — a bare string or nested array in a -choice always counts as `1`. Weights are checked when you create the generator: a -negative, non-numeric, or all-zero set is rejected at `New`, so a typo fails -fast instead of silently skewing output. +Renders `070-412 38 91` ten times as often as `08-…`. A string item is weighted by +writing it as `{ "format": "AB", "weight": 3 }`. Rejected at load: a weight that +is negative, non-numeric, `0` (never drawn — remove the item), `1` (the default) +or outside a choice, and a repeated item — `["a", "a", "b"]` is `[{ "format": "a", "weight": 2 }, "b"]`, +and the error says so. -**Repeat.** A template node may carry a `repeat` (default `1`) to render its -`format` that many times — each render an independent pick — joined by -`separator` (default `""`): +### Repeat + +A template may carry `repeat` (an integer above `1`) to render its format that +many times — each an independent draw — joined by `separator` (default `""`): ```json { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -This yields e.g. `bar foo baz`. `repeat` must be a positive integer and -`separator` a string, both checked at `New`. +Renders e.g. `bar foo baz`. Rejected at load: a `separator` without a `repeat`, +a `separator` of `""` (the default), and a `repeat` that multiplies to more than +1 048 576 renders along any path of nested repeats. -`format`, `weight`, `repeat` and `separator` are the only options; **any other key -is a field**. So write `seperator` and you get a field by that name while the -option stays unset. `New` rejects an option that cannot take effect — a -`separator` without a `repeat` above 1, a `weight` outside a choice — and a name -using a character the grammars reserve: `.` separates the segments of a path, `|` -the arms of a token, `(` opens a function call and `}` ends the token, so a name -carrying one is a name no format could ever spell. An empty name goes the same -way — it is no path segment at all, so `List` never offers it. That holds for a -category, a folder and a field alike. +### Options and fields -**Functions.** A `{name()}` token calls a built-in function instead of rendering -a field. `{luhn()}` appends a Luhn check digit over the digits emitted **so far** -in the current format (non-digits skipped but kept); unknown functions or wrong -argument counts are rejected at `New`. This is what makes a generated Swedish -personnummer valid — its last digit is a Luhn checksum over the nine before it: +`format`, `weight`, `repeat` and `separator` are the only options; **any other +key is a field** (see [Decisions](#decisions)). An object that does nothing a +string can't — only a `format` — is rejected naming the string, as is a one-item +choice naming its item. -```json -{ "format": "00{mmdd}-000{luhn()}", "mmdd": [ … ] } -``` +### Functions -renders e.g. `811218-987`, then `{luhn()}` appends `6` → `811218-9876`. Place it -after its payload (it reads what is to its left). The buffer it reads is -per-expansion, so nesting keeps fixed parts out of the sum — e.g. a 12-digit -form prefixes the century outside the checksummed core: +A `{name(args)}` token calls a builtin. Arguments are checked at `New`: a bad +count, range, country or expression fails fast; an integer is written plain +(`5`, not `+5` or `05`); bounds are finite; a sample that could only ever emit one +value (`int(5,5)`, `float(1,1,2)`) is rejected naming the text to write instead; +and a length, count or decimal place beyond a sane maximum is rejected, so a +fat-fingered `hex(2000000000)` never tries to allocate gigabytes. Every builtin draws only from the seed — a +time-based id takes its timestamp from the rng, not the clock — so seeded output +stays reproducible. + +| Function | Kind | Renders | +|----------|------|---------| +| `{luhn()}` | derivation | Luhn check digit over the digits emitted so far in this expansion | +| `{mod11()}` | derivation | weighted mod-11 check char (weights 2–7 from the right); `X` for 10 | +| `{ean()}` | derivation | EAN-13 / UPC-A / ISBN-13 / GTIN check digit | +| `{digits(n)}` | sample | `n` digits 0–9 | +| `{upper(n)}`, `{lower(n)}` | sample | `n` letters A–Z, a–z | +| `{int(min,max)}` | sample | uniform integer in `[min, max]` | +| `{float(min,max,dp)}` | sample | number in `[min, max]` with `dp` decimals | +| `{hex(n)}` | sample | `n` lowercase hex digits | +| `{base64(n)}` | sample | `n` random bytes, base64 | +| `{uuid()}` | sample | UUID v7 (`misc.uuid` ships v4 as data) | +| `{ulid()}` | sample | ULID, 26 Crockford base32 chars | +| `{nanoid(n)}` | sample | URL-safe Nano ID, `n` chars | +| `{iban(CC)}` | sample | length- and mod-97-valid IBAN for BE, DE, DK, ES, FI, NO or SE | +| `{seq()}`, `{seq(name)}` | counter | next integer from 1 in this generator; `name` selects an independent counter | +| `{calc(expr)}`, `{calc(expr,dp)}` | computation | an arithmetic expression over sibling fields ([Computation](#computation)) | +| `{lowercase(x)}`, `{uppercase(x)}`, `{ascii(x)}` | transform | a field's value rewritten ([Transforms](#transforms)) | + +A derivation reads what is to its left, so place it after its payload; the +buffer is per expansion, so a nested template keeps fixed parts out of the sum. A +Swedish personnummer is a Luhn checksum over the nine digits before it: ```json { "format": "{century}{core}", "century": ["19", "20"], - "core": { "format": "00{mmdd}-000{luhn()}", "mmdd": [ … ] } } + "core": { "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", "mmdd": ["0115", "0704", "1218"] } } ``` -A function must be deterministic (no wall-clock), so a seeded generator stays -reproducible. A time-based id (UUID v7, ULID) therefore draws its timestamp from -the rng, not the clock — the result is a valid, reproducible value, not a real -point in time. +Renders e.g. `19811218-9876`. `{seq()}` spans `Fake` calls and `repeat`, resets +with a new generator, and is the natural primary key for the SQL example above. -There are four kinds. **Derivations** read the digits emitted so far, so put them -after their payload; **samples** read only the rng, so they stand alone; one -**session counter** (`seq`) advances state held on the generator; and one -**computation** (`calc`) evaluates arithmetic over sibling fields. Arguments are -validated at `New` (a bad count, range, country, or expression fails fast); a -length, count or decimal place beyond a sane maximum is rejected there too, so a -fat-fingered `hex(2000000000)` can't try to allocate gigabytes at render. +### Computation -| Function | Kind | Emits | -|----------|------|-------| -| `{luhn()}` | derivation | Luhn check digit (mod-10) over preceding digits | -| `{mod11()}` | derivation | weighted mod-11 check char (weights 2–7 from the right); `X` when it would be 10 | -| `{ean()}` | derivation | EAN-13 / UPC-A / ISBN-13 / GTIN check digit | -| `{uuid()}` | sample | UUID v7 (v4 ships as data — see [Data](#data)) | -| `{ulid()}` | sample | ULID, 26-char Crockford base32 | -| `{nanoid(n)}` | sample | URL-safe Nano ID, `n` chars | -| `{hex(n)}` | sample | `n` lowercase hex digits | -| `{base64(n)}` | sample | `n` random bytes, base64 | -| `{int(min,max)}` | sample | uniform integer in `[min, max]` | -| `{float(min,max,dp)}` | sample | number in `[min, max]` with `dp` decimals | -| `{iban(CC)}` | sample | a length- and mod-97-valid IBAN for country `CC` (BE, DE, DK, ES, FI, NO, SE) | -| `{seq()}`, `{seq(name)}` | session counter | next integer (from 1) in this generator's sequence; `name` selects an independent counter | -| `{calc(expr)}`, `{calc(expr,dp)}` | computation | value of an arithmetic expression over number literals and sibling fields; `dp` rounds | - -`{ean()}` is also the ISBN-13 check (an ISBN-13 *is* an EAN-13 — build the 978/979 -prefix in data and call `{ean()}`). `{iban()}` is a sample, not a derivation: -an IBAN's check digits sit *before* the account number, which a left-to-right -reader can't reach, so it emits the whole value (a generic numeric BBAN — valid -length and checksum, not real bank routing). - -`{seq()}`'s counter lives on the generator, so it spans `Fake` calls (and `repeat`) -and resets when you build a new generator — `seq` is reproducible by being ordered, -not random. It's the natural fit for a primary-key column in the SQL example above. - -**Computation.** `{calc(expr)}` evaluates an arithmetic expression — `+ - * /`, -parentheses and unary minus, the usual precedence — and emits the result. Operands -are number literals and **sibling field names**, each rendered then read as a -number; an optional second arg rounds to that many decimals (`{calc(net * qty, 2)}`), -otherwise the value prints in minimal form. A hyphen is always subtraction, so a -hyphenated field name can't be an operand. The expression is checked at `New` -(parse, and that every name is a real field): +`{calc(expr)}` evaluates `+ - * /`, parentheses and unary minus over number +literals and sibling field names, each rendered then read as a number; a second +argument rounds to that many decimals. A hyphen is always subtraction, so a +hyphenated field can't be an operand. ```json -{ "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99"], "qty": ["3"] } +{ "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } ``` -renders `19.99 x 3 = 59.97`. A field a `calc` reads is drawn **once per -expansion** and held, so the operand shown is the operand computed — give `net` -three prices and the line still multiplies the one it printed. The hold covers -every reading of that name in the format, so `{w} {w} {calc(w)}` is one value -three times; a name no `calc` reads is unaffected, and `{word} {word}` still draws -twice. A field that doesn't render to a number yields `NaN`, and a division by -zero yields `Inf`; both print rather than failing the render. +Renders e.g. `19.99 x 3 = 59.97`. An operand that can never be a number (`"abc"`, +or a choice of such) is rejected at load, as is a division by a constant zero +(`1/0`, or a fixed `"0"` field); an operand that sometimes is not a number yields +`NaN`, and a division by one that is not constant `Inf` — both print rather than +fail. -**References.** A `{..path}` token renders a node from the **data root** instead -of a sibling field — the dot path is the one `Fake` takes, resolved across every -loaded directory. One category can borrow another, even across folders or layered -data dirs: +### Transforms + +`{lowercase(x)}`, `{uppercase(x)}` and `{ascii(x)}` rewrite the value of `x` — a +field, a path or a `..path` — and nest. `ascii` folds Latin letters (`Åsa Öberg` +→ `Asa Oberg`) and drops any other non-ASCII rune. `x` is held +([One draw, one spelling](#one-draw-one-spelling)), so an email built from a name +matches the name beside it: ```json -{ "format": "Hej, {..en_US.person}!" } -``` - -renders e.g. `Hej, Pat Smith!`. References are bound when you create the generator, so -a path that is unknown, names a folder, or steps through a multi-variant choice -fails at `New`. A reference that leads back to its own value (directly, mutually, -or through a chain) is a cycle that would never finish rendering, so it too is -rejected at `New`. - -**Correlated fields.** A `{name.field}` token addresses a **path** into a sibling, -and a sibling addressed that way is drawn **once per expansion** — so several -tokens read one row. That is how two facts that belong together, such as a -locality and the postal code that really covers it, stay together: - -```json -{ "format": "{street} {number}\n{place.postal-code} {place.locality}", - "place": [ - { "format": "{locality}", "locality": "Stockholm", "postal-code": { "format": "#100 00" }, "weight": 975 }, - { "format": "{locality}", "locality": "Tranås", "postal-code": { "format": "#5#7#3 00" }, "weight": 18 } +{ "format": "{p.first} {p.last} <{lowercase(ascii(p.first))}.{lowercase(ascii(p.last))}@example.com>", + "p": [ + { "format": "{first} {last}", "first": "Åsa", "last": "Öberg" }, + { "format": "{first} {last}", "first": "Bo", "last": "Ek" } ] } ``` -renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm postal code -beside Tranås. Each row carries its own `weight`, so how often a place appears is data -too. The rule holds both ways: two tails of one head come from the same row, and -one path read twice reads one value (`{p.first} … {p.first}@…` gives one name). +Renders `Åsa Öberg ` or `Bo Ek `, never +a mix. -**One draw, one spelling.** The hold above is what a dotted token reads, and a -`{calc()}` operand reads its sibling the same way (see **Computation**). A format -may not both *render* a level and *read a path into* it — `{p}` beside -`{p.first}`, or `{place}` beside `{place.locality}`. -Rendering a level expands it afresh while a path reads the level's held draw, so -the two would disagree; naming the fields you want is the one spelling that -always agrees, and the other is a load error. This covers every way a level can -be rendered: a token, a `{calc()}` operand, and a `{..path}` reference — wherever -the reference sits, including in a field the format renders. +### References -A `{calc()}` operand is held on its own terms too, so the same fence guards it: -`{..cat.net} x 2 = {calc(net * 2, 2)}` names one field two ways and is a load -error, with no path token anywhere. A calc renders its operand whole, so the hold -pins what that render settled, following the operand's plain `{field}` tokens — -`{net.v}` and `{..cat.net.v}` are rejected alike. It stops at a `{..path}`, where -the operand's own value ends and a shared source begins: two names drawing from -one referenced category are two draws, as `{word} {word}` is, so two dice over -one `{..die}` are fine. +A reference renders a node from elsewhere in the data — the path `Fake` takes, +across every loaded source — so one category borrows another. The sigil says +where the path starts, as in a filesystem: `{/en_US.person}` from the data root, +`{.username}` from the folder this file sits in, `{..username}` from the folder +above. `data/sv_SE/email.json` can therefore read its own locale's `username` +without naming `sv_SE`: + +```json +"Hej, {/en_US.person}!" +``` + +Renders e.g. `Hej, Pat Smith!`. A reference into a category is held like a +[correlated](#correlated-fields) path — `{.person.first} {.person.last}` name one +person, `{lowercase(.person.first)}` reads that same draw, and `{.person.first}` +beside `{/sv_SE.person.last}` in `sv_SE` is one person too — while a bare +`{/misc.uuid} {/misc.uuid}` is two draws. Rejected at `New`: a path that is +unknown, names a folder, has no folder above, or reads a field not every variant +of a choice carries, and a reference that leads back to its own value, directly, +mutually or through a chain. + +### Correlated fields + +`{name.field}` reads a path into a sibling, which holds the sibling to one draw +([One draw, one spelling](#one-draw-one-spelling)), so several tokens read one +row — a locality and the postal code that really covers it: + +```json +{ "format": "{street} {int(1,99)}\n{place.postal-code} {place.locality}", + "street": ["Kungsgatan", "Storgatan"], + "place": [ + { "format": "{locality}", "locality": "Stockholm", "postal-code": "1{digits(2)} {digits(2)}", "weight": 975 }, + { "format": "{locality}", "locality": "Tranås", "postal-code": "573 {digits(2)}", "weight": 18 } + ] } +``` + +Renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm code beside +Tranås; each row's `weight` says how often it appears. A path is held at every +level it passes through: `{p.geo.town.name} {p.geo.town.zip}` share the town. +Every variant of a choice on the path must carry the rest of it, so a row missing +a field is named at load: + +```text +token {place.postal-code}: field "place": not every variant of this 2-way choice carries "postal-code"; all carry [locality] +``` + +The sub-fields stay addressable — `Fake("address.place.locality")` renders, and +`List` advertises it. A path may not read into a level carrying a `repeat`. + +### One draw, one spelling + +A name any token reads as a path (`{p.first}`) or as an operand (`{calc(net * 2)}`, +`{uppercase(w)}`) is drawn **once per expansion**, and every other route to it — +a bare `{p}`, a second bare `{w}`, `{/cat.net}`, a nested template rendering +`{/cat.p.last}`, at any depth — is a load error naming the spelling to use. A +name nothing reads that way is drawn each time: `{word} {word}` differs. An +expansion is one render of one format, so each `repeat` iteration and each nested +template draws again. ```text token {p} renders a level that {p.first} reads a path into; name the fields you want instead +token {w} is repeated, and uppercase operand "w" holds "w" to one draw per expansion; write {w} once ``` -For the same reason a path may not read into a level carrying a `repeat`: it -reads one draw, so the repeat could never apply. - -A path is held at **every level it passes through**, not just the first, so the -facts can nest as deeply as they belong: - -```json -{ "format": "{p.geo.town.name} {p.geo.town.zip}, {p.geo.region}", "p": [ … ] } -``` - -renders `Kiruna 98100, Norrbotten` — the town, the zip that covers it and the -region it sits in all come from one draw. Two paths part company exactly where -they diverge: `{p.a.v}` and `{p.b.v}` share the row and nothing below it. - -The binding lasts for one expansion, so each `repeat` iteration draws again and a -nested template keeps its own. A field no dotted token addresses is unaffected: -`{word} {word}` still draws twice. - -`New` checks a path the way `Fake` resolves one: every variant of a multi-variant -choice must carry the whole path, so a row missing a field is named at load: - -```text -token {place.postal-code}: field "place": not every variant of this 2-way choice -carries "postal-code"; all carry [locality] -``` - -The sub-fields stay addressable on their own — `Fake("address.place.locality")` -renders, and `List` advertises it. - -**Format string.** Every character is literal except: - -| Token | Expands to | -|-------|-----------| -| `0` | digit 0–9 | -| `1` | digit 1–9 | -| `A` | letter A–Z | -| `a` | letter a–z | -| `#` | escape — the next char is literal (`#0` → `0`, `##` → `#`) | -| `{name}` | render the sibling field `name` | -| `{name.field}` | render `field` of one draw of the sibling `name` (see **Correlated fields**) | -| `{name()}` | call a built-in function (see **Functions**) | -| `{..path}` | render the node at a dot path from the data root (see **References**) | - -`{a|b}` renders one of the sibling fields `a` or `b`, chosen at random; an arm -may be a `{..path}` reference too (`{name|..en_US.person}`). The arms are picked -evenly and must differ — `{a|a|b}` would skew the odds, which is what `weight` is -for, so a repeated arm is a load error. - -Inside a `format`, `0 1 A a` are **always** character classes — so a fixed `0`, -`1`, `A` or `a` must be escaped (`#1`, `#A`) or it becomes random. A format of -`100 Main St` renders e.g. `506 Mdin St` — the `1`, `0`, `0` and `a` were random, -the `M`, `in` and `St` were not. A half-fixed string is the trap: `555-0000` -keeps `555-` and randomises the last four digits. For a value with no -tokens at all, use a bare string node (`"100 Main St"`), emitted verbatim. - -**Putting it together** (`person.json`): - -```json -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Anna", "Astrid", "Elin"], - "malefirst": ["Anders", "Erik", "Gustav"], - "last": [ - { "format": "{first}sson", "first": ["Ander", "Erik", "Karl"] }, - ["Berg", "von Flemming"] - ], - "prefix": [ - "", - { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 } - ] - } -] -``` - -This yields e.g. `Anna Eriksson`, `Erik Berg`, or rarely `dr Astrid von Flemming`. -Any field is reachable by dotted path — `Fake("person.last")` renders just a -surname; choices along the path are resolved at random. A path may continue -*through* a choice only where every variant carries the rest of it — every -`currency` variant carries `symbol`, so `currency.symbol` resolves — which keeps a -path from rendering on one call and failing on the next. - ### Performance -Each file is parsed, validated and weight-indexed once, in `New`. After that a -`Fake` call costs about what its output costs — it scans the chosen format and -renders nested tokens, independent of how large your lists are: +Each file is parsed, validated and weight-indexed once, in `New`. A `Fake` call +then costs about what its output costs: an unweighted pick is O(1) whatever the +list's length, a weighted one O(log n), and long formats, deep nesting and many +tokens add cost in proportion to the output. -- Picking from a list is **O(1)** whatever its length — a 10-name list and a - 100 000-name list cost the same. -- Giving entries a `weight` makes that list's pick **O(log n)** instead (a - search over cumulative weights). Still tiny, but an unweighted list is the - cheapest — only add `weight` where you actually want skew. -- Long `format` strings, deep nesting and many `{tokens}` add cost in - proportion to the output produced. +## Decisions + +- **Options and fields share one namespace.** `format`, `weight`, `repeat` and + `separator` are reserved; every other key is a field. Nesting fields under a + key, or prefixing options, would tax every template to guard against a + misspelt option. +- **`{a|b}` stays beside nested choices.** `[[…], […]]` picks the same way, but + its arms are anonymous; `{femalefirst|malefirst}` keeps `person.femalefirst` + addressable. +- **Flags follow getopt_long.** `--name value` and `--name=value` both work; a + short flag's value attaches or follows (`-s42`, `-s 42`) and short flags bundle + (`-hn 3`), as every shell user expects. A single-dash long flag is rejected + naming the double-dash spelling, and `-s=42` is rejected naming both short + spellings: `=` belongs to the long form, and reading `=42` as the value would + make `-d=./x` a directory named `=./x`. +- **The shipped data is embedded, not discovered.** A directory a machine happens + to have would make `--seed 42` machine-dependent. Data still lives in `data/` + as JSON; `--data-path` layers over it. +- **A bare reference draws each time; a reference path is held.** `{/p} {/p}` + is two draws, as `{word} {word}` is, while `{/p.first}` beside a nested template + rendering `{/p.first}` is a load error: a bare token is by contract an + independent draw, a path pins its level, and any route into a pinned level from + another expansion could show another row. +- **Reference sigils follow the filesystem.** `/` is the root, `.` this file's + folder, `..` the folder above — what those spellings already mean to anyone who + has typed a path. A locale's files reach each other without naming the locale, + so a folder renames and copies without editing its references. +- **After the first tag, a new fence is a major version.** Data files are the + public API, and one spelling per result grows by tightening, so every fence + invalidates some file. Each such release names the rejected spelling and its + replacement in the changelog and in the load error, and that is the whole + migration: a fence rejects one spelling with one replacement, so the fix is + local to each site. A fence that would need a non-local rewrite ships a + converter with its release instead. Before the first tag there is no + compatibility promise. +- **A `--data-path` override rebinds every reference to the category it + replaces.** References bind against the merged tree, so once shipped data uses + `{.person}`, a consumer's `sv_SE/person.json` is what every shipped reference + into `person` reads, and `New` fails on shipped data the consumer never wrote + when that file lacks a field those references read. Accepted: overriding is the + point of layering, the error names the reference and the field, and the fix is + the consumer's file carrying the fields the shipped tree reads. +- **The repeat cap bounds renders, not bytes.** A repeat, alone or nested, may + ask for at most 1 048 576 renders; how large each render is stays what the data + asked for, so `{hex(1048576)}` repeated to the cap is a terabyte, loaded without + complaint. A byte estimate would need every builtin to declare a width to fence + a shape no data comes near, and the harm lands on the author who wrote it. +- **64-bit targets only.** The gate builds amd64, and the buffer sizing a render + pre-computes (renders × bytes) assumes a 64-bit int; on a 32-bit target it could + overflow and panic. +- **A constant zero divisor is a load error; a divisor that is not constant prints + `Inf`.** `1/0` and a fixed `"0"` field are decidable, so they join the + never-numeric operand as a load error; the fold stops where an operand varies, + so `a/(b*c)` with `b` fixed at `0` and `c` varying loads and prints `Inf` every + draw — catching it needs zero-absorbing algebra for a shape nobody writes. +- **In data, a default written out and a constant spelled as a sample are load + errors.** `weight: 1`, `repeat: 1`, `separator: ""`, `int(5,5)`, `float(1,1,2)`, + `+5` and `05` each spell what a shorter form already spells, so each is rejected + naming that form. The CLI's numbers follow the shell instead: `--seed 007` and + `--repeat +3` are 7 and 3, as every command line reads them. +- **Samples say what they emit, transforms what they do.** `{upper(2)}` is two + letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on + whether the argument looks like a number. ## Development @@ -481,11 +411,12 @@ Everything runs in Docker — **no local tooling beyond Docker is needed**. Source is bind-mounted; build caches persist in the `gocache` volume. ```sh -docker compose run --rm test # run tests +docker compose run --rm test # go test -race docker compose run --rm cover # tests with coverage docker compose run --rm bench # benchmarks docker compose run --rm build # compile the library docker compose run --rm vet # go vet +docker compose run --rm cyclo # cyclomatic complexity over 14 (test files excluded) docker compose run --rm dev # interactive shell ``` @@ -498,39 +429,32 @@ docker compose run --rm --user "$(id -u):$(id -g)" tidy # go mod tidy Every pull request runs `docker build .` against both the latest and the lowest supported Go, and must pass before it can be merged. That build is the whole -gate — vet, format check and tests — so run it locally before pushing: +gate — vet, complexity, format check and tests — so run it locally before pushing: ```sh docker build . # latest docker build --build-arg GO_VERSION=1.22.12 . # lowest supported -``` - -Tests run against the latest Go by default. Set `GO_VERSION` to check the lowest -supported version too: - -```sh -GO_VERSION=1.22.12 docker compose run --rm test # lowest supported -docker compose run --rm test # latest +GO_VERSION=1.22.12 docker compose run --rm test # the same tests, without the image build ``` ## Layout ``` -fejkdata.go Generator, New, List, options, seeding +fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation -render.go Fake and the recursive renderer (choices, format strings, paths, bound draws) -template.go the {token} grammar: scanning, function and path tokens, validation -reference.go {..path} binding across the tree, the render graph, and the walks over it +path.go the dotted-path walk, and proving a path resolves +render.go Fake and the recursive renderer (choices, format strings, expansions) +template.go the {token} grammar: scanning, tokens, operands, validation, compiling a format +hold.go the hold: one draw per expansion for paths and operands, and its fences +reference.go reference sigils, and binding references across the tree +graph.go the render graph: edges, cycles, the repeat bound, tree walks builtins.go the {name()} function registry and its implementations calc.go the {calc()} arithmetic evaluator: parser, eval, validation -data.go data loading: folders/files -> namespace tree, multi-path merge -cmd/fejkdata/ the `fejkdata` CLI (New + Fake/List over stdout) -data/ shipped data (JSON): locale folders + a misc folder +data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge +cmd/fejkdata/ the fejkdata CLI +data/ shipped data (JSON), embedded at build: locale folders + a misc folder ``` -To add a category, drop a JSON file into a data directory; to add a locale, add -a subdirectory of JSON files. - ## License MIT — see [LICENSE](LICENSE). diff --git a/bench_test.go b/bench_test.go index 9c69865..d3bc76f 100644 --- a/bench_test.go +++ b/bench_test.go @@ -8,7 +8,7 @@ import ( func benchGenerator(b *testing.B, dir string) *Generator { b.Helper() - f, err := New([]string{dir}, WithSeed(1)) + f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1)) if err != nil { b.Fatal(err) } @@ -43,12 +43,12 @@ func tmpData(b *testing.B, name, body string) string { } func BenchmarkCalc(b *testing.B) { - dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`) + dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`) benchPath(b, dir, "inv") } func BenchmarkLongLiteral(b *testing.B) { - dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) V#ALUES (#1#2#3, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) + dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) benchPath(b, dir, "sql") } @@ -61,24 +61,19 @@ func BenchmarkRepeat(b *testing.B) { // what a correlated pair costs against BenchmarkUnbound, the same output drawn from // two independent fields. func BenchmarkBound(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"#100 00"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"#5#7#3 00"}}]}`) + dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`) benchPath(b, dir, "addr") } func BenchmarkUnbound(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}", - "postal-code":[{"format":"#100 00"},{"format":"#5#7#3 00"}], - "locality":["Stockholm","Tranås"]}`) + dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`) benchPath(b, dir, "addr") } // BenchmarkBoundDeep reads two paths through an intermediate level, the shape the // depth question is about. func BenchmarkBoundDeep(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":[ - {"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":{"format":"#100 00"}}}]}`) + dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`) benchPath(b, dir, "addr") } @@ -91,11 +86,10 @@ func BenchmarkBoundWide(b *testing.B) { benchPath(b, dir, "row") } -// BenchmarkNew measures load+compile+validate of the whole shipped tree. func BenchmarkNew(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - if _, err := New([]string{"data"}, WithSeed(1)); err != nil { + if _, err := New(WithSeed(1)); err != nil { b.Fatal(err) } } diff --git a/builtins.go b/builtins.go index 5482665..d72ce2e 100644 --- a/builtins.go +++ b/builtins.go @@ -2,41 +2,38 @@ package fejkdata import ( "encoding/base64" + "errors" "fmt" "math" "strconv" "strings" + "unicode" ) -// maxLen caps sample output lengths (hex, nanoid, base64) and maxDecimals caps -// float/calc decimal places, so a fat-fingered or overflowing argument fails at -// New instead of trying to allocate gigabytes — or panicking — at render. +// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower) +// and maxDecimals float/calc decimal places, so a fat-fingered or overflowing +// argument fails at New instead of trying to allocate gigabytes — or panicking — +// at render. const ( - maxLen = 1 << 20 // 1,048,576 chars/bytes + maxLen = 1 << 20 maxDecimals = 1024 ) -// builtins is the registry of {name(args)} functions. Derivations read the -// digits emitted so far in the current expansion (luhn, mod11, ean — place them -// after their payload); samples read only the rng (uuid, ulid, ...). All -// must stay pure over (rng, emitted, args) so seeded output is reproducible — a -// time-based id (uuid v7, ulid) draws its timestamp from the rng, not the wall -// clock. Add a builtin only for what data can't express: a random v4 UUID and a -// 24-hex ObjectID both ship as data, so the uuid builtin is v7. +// builtins is the registry of {name(args)} functions. Derivations read the digits +// emitted so far in the current expansion (place them after their payload); +// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp +// from the rng, not the wall clock, so seeded output stays reproducible. var builtins = map[string]builtin{ - "luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })}, - "mod11": {arity: 0, prep: derive(mod11Check)}, - "ean": {arity: 0, prep: derive(eanCheck)}, - "uuid": {arity: 0, prep: sample(uuidV7)}, - "ulid": {arity: 0, prep: sample(ulid)}, - "nanoid": {arity: 1, check: posIntArg, prep: func(a []string) callFn { - n := atoi(a[0]) - return func(s *session, _ string, _ []string) string { return nanoid(s, n) } - }}, - "hex": {arity: 1, check: posIntArg, prep: func(a []string) callFn { - n := atoi(a[0]) - return func(s *session, _ string, _ []string) string { return randHex(s, n) } - }}, + "luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })}, + "mod11": {arity: 0, prep: derive(mod11Check)}, + "ean": {arity: 0, prep: derive(eanCheck)}, + "uuid": {arity: 0, prep: sample(uuidV7)}, + "ulid": {arity: 0, prep: sample(ulid)}, + "nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)}, + "hex": {arity: 1, check: posIntArg, prep: chars(hexDigits)}, + "digits": {arity: 1, check: posIntArg, prep: chars("0123456789")}, + "upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")}, + "lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")}, "base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn { n := atoi(a[0]) return func(s *session, _ string, _ []string) string { @@ -57,9 +54,10 @@ var builtins = map[string]builtin{ cc := a[0] return func(s *session, _ string, _ []string) string { return iban(s, cc) } }}, - // calc is the one builtin that names operands (expand reads them for it); - // every other ignores them. 1 or 2 args: the expression and an optional decimals. - "calc": {arity: -1, check: checkCalc, prep: calcPrep}, + "calc": {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands}, + "lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand}, + "uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand}, + "ascii": {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand}, // seq is the one stateful builtin: a per-session counter from 1, advancing on // each call. An optional name selects an independent counter; no name uses the // default one. Deterministic by construction, so seeded output stays stable. @@ -74,9 +72,9 @@ var builtins = map[string]builtin{ }}, } -// derive and sample are the two argument-free builtin shapes the README names: a -// derivation reads the output emitted so far, a sample reads only the rng. Each -// lifts that one function into the prep every registry entry supplies. +// derive and sample are the two argument-free builtin shapes: a derivation reads +// the output emitted so far, a sample reads only the rng. chars is the shape of a +// sample of n characters drawn from an alphabet. func derive(f func(emitted string) string) func([]string) callFn { return func([]string) callFn { return func(_ *session, emitted string, _ []string) string { return f(emitted) } @@ -89,8 +87,105 @@ func sample(f func(rng) string) func([]string) callFn { } } +func chars(alphabet string) func([]string) callFn { + return func(a []string) callFn { + n := atoi(a[0]) + return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) } + } +} + const hexDigits = "0123456789abcdef" +// transforms are the builtins that rewrite one operand's value; they nest, so +// {lowercase(ascii(x))} folds then lowers. +var transforms = map[string]func(string) string{ + "ascii": asciiFold, + "lowercase": strings.ToLower, + "uppercase": strings.ToUpper, +} + +// unwrapTransform peels nested transform calls off an operand arg, returning the +// field it finally names and the transforms to apply, innermost last. +func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) { + for { + name, args, isCall := funcCall(arg) + if !isCall { + return arg, chain, nil + } + fn, isTransform := transforms[name] + if !isTransform { + return "", nil, fmt.Errorf("%s(%s) is not a transform, so it cannot be an operand", name, strings.Join(args, ",")) + } + if len(args) != 1 { + return "", nil, fmt.Errorf("%s takes 1 arg, got %d", name, len(args)) + } + chain = append(chain, fn) + arg = args[0] + } +} + +func transformArg(fields map[string]node, a []string) error { + leaf, _, err := unwrapTransform(a[0]) + if err != nil { + return err + } + if isRef(leaf) { + _, _, err := refShape(leaf) + return err + } + return checkArm(leaf, fields) +} + +func transformOperand(a []string) []string { + leaf, _, err := unwrapTransform(a[0]) + if err != nil { + return nil + } + return []string{leaf} +} + +func transformPrep(outer func(string) string) func([]string) callFn { + return func(a []string) callFn { + _, chain, err := unwrapTransform(a[0]) + if err != nil { + panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err)) + } + return func(_ *session, _ string, operands []string) string { + v := operands[0] + for i := len(chain) - 1; i >= 0; i-- { + v = chain[i](v) + } + return outer(v) + } + } +} + +// asciiFolds maps the Latin letters with diacritics or ligatures to ASCII. +var asciiFolds = map[rune]string{ + 'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", 'Å': "A", 'Æ': "AE", 'Ç': "C", + 'È': "E", 'É': "E", 'Ê': "E", 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", + 'Ð': "D", 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", 'Ø': "O", + 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", 'Þ': "Th", 'ß': "ss", 'Œ': "OE", + 'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", + 'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", + 'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", + 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'þ': "th", 'ÿ': "y", 'œ': "oe", +} + +// asciiFold rewrites s to ASCII: folded Latin letters stay, any other non-ASCII +// rune is dropped. +func asciiFold(s string) string { + var b strings.Builder + for _, r := range s { + if r <= unicode.MaxASCII { + b.WriteRune(r) + } else { + b.WriteString(asciiFolds[r]) + } + } + return b.String() +} + // atoi parses an arg a builtin's check already validated. It panics rather than // returning zero, so a check that stops covering its own args is a stack trace and // not a silently wrong length, range or decimal count. @@ -119,18 +214,39 @@ func randBytes(r rng, n int) []byte { return b } -func randHex(r rng, n int) string { +func randChars(r rng, n int, alphabet string) string { b := make([]byte, n) for i := range b { - b[i] = hexDigits[r.IntN(16)] + b[i] = alphabet[r.IntN(len(alphabet))] } return string(b) } +// plainInt parses an integer arg written the one way: no sign, no leading zero. +func plainInt(s string) (int, error) { + n, err := strconv.Atoi(s) + if errors.Is(err, strconv.ErrRange) { + return 0, fmt.Errorf("%q is past the integer range: %w", s, err) + } + if err != nil { + return 0, fmt.Errorf("%q is not an integer", s) + } + if strconv.Itoa(n) != s { + return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n) + } + return n, nil +} + func posIntArg(_ map[string]node, a []string) error { - n, err := strconv.Atoi(a[0]) - if err != nil || n < 1 { - return fmt.Errorf("count %q must be a positive integer", a[0]) + n, err := plainInt(a[0]) + if errors.Is(err, strconv.ErrRange) { + return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen) + } + if err != nil { + return fmt.Errorf("count %w", err) + } + if n < 1 { + return fmt.Errorf("count %q must be positive", a[0]) } if n > maxLen { return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen) @@ -139,14 +255,20 @@ func posIntArg(_ map[string]node, a []string) error { } func intRangeArgs(_ map[string]node, a []string) error { - lo, e1 := strconv.Atoi(a[0]) - hi, e2 := strconv.Atoi(a[1]) - if e1 != nil || e2 != nil { - return fmt.Errorf("int(min,max) needs integer args, got %q,%q", a[0], a[1]) + lo, err := plainInt(a[0]) + if err != nil { + return fmt.Errorf("int(min,max): min %w", err) + } + hi, err := plainInt(a[1]) + if err != nil { + return fmt.Errorf("int(min,max): max %w", err) } if lo > hi { return fmt.Errorf("int(min,max): min %d > max %d", lo, hi) } + if lo == hi { + return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo) + } if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi) } @@ -156,19 +278,28 @@ func intRangeArgs(_ map[string]node, a []string) error { func floatArgs(_ map[string]node, a []string) error { lo, e1 := strconv.ParseFloat(a[0], 64) hi, e2 := strconv.ParseFloat(a[1], 64) - dp, e3 := strconv.Atoi(a[2]) - if e1 != nil || e2 != nil || e3 != nil { - return fmt.Errorf("float(min,max,dp) needs numeric args, got %q,%q,%q", a[0], a[1], a[2]) + if e1 != nil || e2 != nil { + return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1]) + } + dp, err := plainInt(a[2]) + if err != nil { + return fmt.Errorf("float(min,max,dp): decimals %w", err) + } + if math.IsNaN(lo) || math.IsNaN(hi) || math.IsInf(lo, 0) || math.IsInf(hi, 0) { + return fmt.Errorf("float(min,max,dp) needs finite bounds, got %q,%q", a[0], a[1]) } if lo > hi { return fmt.Errorf("float(min,max,dp): min %v > max %v", lo, hi) } - if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" - return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) - } if dp < 0 || dp > maxDecimals { return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals) } + if lo == hi { + return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64)) + } + if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" + return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) + } return nil } @@ -182,18 +313,9 @@ func seqArg(_ map[string]node, a []string) error { return nil } -// nanoidAlphabet is the 64-char URL-safe set Nano IDs use (order is irrelevant -// to the uniform pick). +// nanoidAlphabet is the 64-char URL-safe set Nano IDs use. const nanoidAlphabet = "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -func nanoid(r rng, n int) string { - b := make([]byte, n) - for i := range b { - b[i] = nanoidAlphabet[r.IntN(len(nanoidAlphabet))] - } - return string(b) -} - // uuidV7 builds an RFC 9562 v7 UUID. The 48-bit timestamp field is drawn from // the rng (not the clock) to stay reproducible, then the version (7) and variant // (10) bits are forced; the rest is random. diff --git a/builtins_test.go b/builtins_test.go index fcac338..7a63509 100644 --- a/builtins_test.go +++ b/builtins_test.go @@ -13,10 +13,10 @@ func TestBuiltinIDGenerators(t *testing.T) { tmpl string re *regexp.Regexp }{ - {"uuid v7", `{"format":"{uuid()}"}`, regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)}, - {"ulid", `{"format":"{ulid()}"}`, regexp.MustCompile(`^[0-7][0-9A-HJKMNP-TV-Z]{25}$`)}, - {"nanoid", `{"format":"{nanoid(21)}"}`, regexp.MustCompile(`^[A-Za-z0-9_-]{21}$`)}, - {"hex", `{"format":"{hex(16)}"}`, regexp.MustCompile(`^[0-9a-f]{16}$`)}, + {"uuid v7", `"{uuid()}"`, regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)}, + {"ulid", `"{ulid()}"`, regexp.MustCompile(`^[0-7][0-9A-HJKMNP-TV-Z]{25}$`)}, + {"nanoid", `"{nanoid(21)}"`, regexp.MustCompile(`^[A-Za-z0-9_-]{21}$`)}, + {"hex", `"{hex(16)}"`, regexp.MustCompile(`^[0-9a-f]{16}$`)}, } f := engine(1) for _, c := range cases { @@ -38,9 +38,9 @@ func TestBuiltinIDGenerators(t *testing.T) { // sample draws only from the seeded rng, so same seed -> same value. func TestBuiltinSamplesReproducible(t *testing.T) { for _, tmpl := range []string{ - `{"format":"{uuid()}"}`, `{"format":"{ulid()}"}`, - `{"format":"{nanoid(12)}"}`, `{"format":"{int(1,1000000)}"}`, - `{"format":"{float(0,1,6)}"}`, `{"format":"{base64(12)}"}`, `{"format":"{iban(SE)}"}`, + `"{uuid()}"`, `"{ulid()}"`, + `"{nanoid(12)}"`, `"{int(1,1000000)}"`, + `"{float(0,1,6)}"`, `"{base64(12)}"`, `"{iban(SE)}"`, } { if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b { t.Fatalf("%s not reproducible: %q != %q", tmpl, a, b) @@ -52,7 +52,7 @@ func TestBuiltinIntInclusiveRange(t *testing.T) { f := engine(1) lo, hi := false, false for i := 0; i < 1000; i++ { - n, err := strconv.Atoi(mustRender(t, f, `{"format":"{int(3,7)}"}`)) + n, err := strconv.Atoi(mustRender(t, f, `"{int(3,7)}"`)) if err != nil || n < 3 || n > 7 { t.Fatalf("int(3,7) = %d (err %v), out of range", n, err) } @@ -66,14 +66,14 @@ func TestBuiltinIntInclusiveRange(t *testing.T) { func TestBuiltinFloat(t *testing.T) { f, re := engine(1), regexp.MustCompile(`^[12]\.\d{3}$`) for i := 0; i < 200; i++ { - if got := mustRender(t, f, `{"format":"{float(1,2,3)}"}`); !re.MatchString(got) { + if got := mustRender(t, f, `"{float(1,2,3)}"`); !re.MatchString(got) { t.Fatalf("float(1,2,3) = %q, want d.ddd in [1,2]", got) } } } func TestBuiltinBase64(t *testing.T) { - got := mustRender(t, engine(1), `{"format":"{base64(9)}"}`) + got := mustRender(t, engine(1), `"{base64(9)}"`) if b, err := base64.StdEncoding.DecodeString(got); err != nil || len(b) != 9 { t.Fatalf("base64(9) = %q decodes to %d bytes (err %v), want 9", got, len(b), err) } @@ -83,9 +83,9 @@ func TestBuiltinBase64(t *testing.T) { // hand-computed check, like the luhn test. mod-11 emits X when it would be 10. func TestBuiltinChecksums(t *testing.T) { cases := map[string]string{ - `{"format":"#1#2#3#4#5#6#7#8{mod11()}"}`: "123456785", // weights 2..7 from the right - `{"format":"#6{mod11()}"}`: "6X", // remainder 10 -> X - `{"format":"#4#0#0#6#3#8#1#3#3#3#9#3{ean()}"}`: "4006381333931", // EAN-13 (= ISBN-13) check digit + `"12345678{mod11()}"`: "123456785", // weights 2..7 from the right + `"6{mod11()}"`: "6X", // remainder 10 -> X + `"400638133393{ean()}"`: "4006381333931", // EAN-13 (= ISBN-13) check digit } f := engine(1) for tmpl, want := range cases { @@ -101,14 +101,14 @@ func TestBuiltinChecksums(t *testing.T) { func TestBuiltinSeqPerSession(t *testing.T) { f := engine(1) for i := 1; i <= 5; i++ { - if got := mustRender(t, f, `{"format":"{seq()}"}`); got != strconv.Itoa(i) { + if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) { t.Fatalf("seq call %d = %q, want %d", i, got, i) } } - if got := mustRender(t, f, `{"format":"{seq(orders)}"}`); got != "1" { + if got := mustRender(t, f, `"{seq(orders)}"`); got != "1" { t.Fatalf("named seq(orders) = %q, want its own count from 1", got) } - if got := mustRender(t, f, `{"format":"{seq()}"}`); got != "6" { + if got := mustRender(t, f, `"{seq()}"`); got != "6" { t.Fatalf("default seq after the named one = %q, want 6 (counters are independent)", got) } // repeat drives one counter forward across its renders... @@ -116,7 +116,7 @@ func TestBuiltinSeqPerSession(t *testing.T) { t.Fatalf("repeat seq = %q, want 1,2,3", got) } // ...and a fresh session restarts from 1. - if got := mustRender(t, engine(2), `{"format":"{seq()}"}`); got != "1" { + if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" { t.Fatalf("new session seq = %q, want 1", got) } } @@ -126,14 +126,14 @@ func TestBuiltinSeqPerSession(t *testing.T) { // at compile, not detonate when the template is drawn. func TestBuiltinArgLimits(t *testing.T) { bad := []string{ - `{"format":"{int(-9223372036854775808,9223372036854775807)}"}`, // span overflows int -> IntN panic - `{"format":"{hex(2000000000)}"}`, // ~2 GB string - `{"format":"{nanoid(2000000000)}"}`, - `{"format":"{base64(2000000000)}"}`, - `{"format":"{float(-1e308,1e308,2)}"}`, // span is +Inf - `{"format":"{float(0,1,100000)}"}`, // decimals -> huge string - `{"format":"{calc(1+1,100000)}"}`, // calc decimals -> huge string - `{"format":"{x}","x":["1"],"repeat":2000000000}`, // repeat -> multi-GB join + `"{int(-9223372036854775808,9223372036854775807)}"`, // span overflows int -> IntN panic + `"{hex(2000000000)}"`, // ~2 GB string + `"{nanoid(2000000000)}"`, + `"{base64(2000000000)}"`, + `"{float(-1e308,1e308,2)}"`, // span is +Inf + `"{float(0,1,100000)}"`, // decimals -> huge string + `"{calc(1+1,100000)}"`, // calc decimals -> huge string + `{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join } for _, tmpl := range bad { if _, err := compile(parse(t, tmpl)); err == nil { @@ -141,7 +141,7 @@ func TestBuiltinArgLimits(t *testing.T) { } } // Ordinary values still compile. - if _, err := compile(parse(t, `{"format":"{hex(16)} {int(-5,5)} {float(0,1,3)} {calc(1+1,2)}"}`)); err != nil { + if _, err := compile(parse(t, `"{hex(16)} {int(-5,5)} {float(0,1,3)} {calc(1+1,2)}"`)); err != nil { t.Errorf("compile of normal args failed: %v", err) } } @@ -150,7 +150,7 @@ func TestBuiltinIBAN(t *testing.T) { wantLen := map[string]int{"SE": 24, "DE": 22, "NO": 15} f := engine(1) for cc, n := range wantLen { - tmpl := `{"format":"{iban(` + cc + `)}"}` + tmpl := `"{iban(` + cc + `)}"` for i := 0; i < 100; i++ { got := mustRender(t, f, tmpl) if got[:2] != cc || len(got) != n { @@ -217,3 +217,16 @@ func mustPanic(t *testing.T, name string, call func()) { }() call() } + +func TestClassBuiltinArgs(t *testing.T) { + for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} { + if _, err := compile(parse(t, bad)); err == nil { + t.Errorf("compile(%s) = nil error, want the arg rejected", bad) + } + } + for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v", ok, err) + } + } +} diff --git a/calc.go b/calc.go index 038fc3e..cd776b7 100644 --- a/calc.go +++ b/calc.go @@ -8,16 +8,6 @@ import ( "unicode" ) -// calc is the {calc(expr[, dp])} token: an arithmetic expression over number -// literals and sibling-field names, with + - * /, unary minus and parentheses. It is -// a registry builtin like any other {name(args)} function — the one that names -// operands, which expand reads for it (once per expansion, so the value computed is -// the value the format showed) and hands over as strings. The value prints in -// minimal decimal form, or rounded to dp when given. An operand that isn't a number -// becomes NaN, which propagates and prints as "NaN" — visible, never a render error. -// The expression is parsed once at New into an AST every render shares and none -// mutates, so render cannot fail and must not carry per-render state. - // calcNode is a parsed expression node. It evaluates over the operand values expand // read, so the evaluator touches neither the rng nor the node tree. type calcNode interface { @@ -80,18 +70,114 @@ func checkCalc(fields map[string]node, args []string) error { return fmt.Errorf("calc(%q): %w", args[0], err) } for _, name := range calcVars(expr) { - if _, ok := fields[name]; !ok { + operand, ok := fields[name] + if !ok { return fmt.Errorf("calc(%q): no field %q", args[0], name) } + if text, never := neverNumeric(operand); never { + return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text) + } + } + if divisor, zero := constantZeroDivisor(expr, fields); zero { + return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor) } if len(args) == 2 { - if dp, err := strconv.Atoi(args[1]); err != nil || dp < 0 || dp > maxDecimals { - return fmt.Errorf("calc decimals %q must be an integer in 0..%d", args[1], maxDecimals) + dp, err := plainInt(args[1]) + if err != nil { + return fmt.Errorf("calc decimals %w", err) + } + if dp < 0 || dp > maxDecimals { + return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals) } } return nil } +// constantZeroDivisor finds a division whose right side is a constant zero: number +// literals and fixed operands folded, anything that varies left unknown. +func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) { + switch n := n.(type) { + case calcNeg: + return constantZeroDivisor(n.x, fields) + case calcBin: + if n.op == '/' { + if v, known := constantValue(n.r, fields); known && v == 0 { + return calcText(n.r), true + } + } + if d, zero := constantZeroDivisor(n.l, fields); zero { + return d, true + } + return constantZeroDivisor(n.r, fields) + } + return "", false +} + +// constantValue evaluates an expression whose every operand is fixed. +func constantValue(n calcNode, fields map[string]node) (float64, bool) { + switch n := n.(type) { + case calcNum: + return float64(n), true + case calcVar: + t, ok := fields[string(n)].(*template) + if !ok || !t.fixed || t.repeat > 1 { + return 0, false + } + v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64) + return v, err == nil + case calcNeg: + v, ok := constantValue(n.x, fields) + return -v, ok + case calcBin: + l, lok := constantValue(n.l, fields) + r, rok := constantValue(n.r, fields) + if !lok || !rok { + return 0, false + } + return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true + } + return 0, false +} + +// calcText spells an expression node the way an author would read it. +func calcText(n calcNode) string { + switch n := n.(type) { + case calcNum: + return strconv.FormatFloat(float64(n), 'f', -1, 64) + case calcVar: + return string(n) + case calcNeg: + return "-" + calcText(n.x) + case calcBin: + return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")" + } + return "?" +} + +// neverNumeric reports a node no render of which is a number: fixed text that does +// not parse, or a choice of only such items. text is one such render. +func neverNumeric(n node) (text string, never bool) { + switch n := n.(type) { + case *template: + if !n.fixed || n.repeat > 1 { + return "", false + } + if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil { + return n.lit, true + } + case *choice: + for _, it := range n.items { + t, itemNever := neverNumeric(it) + if !itemNever { + return "", false + } + text = t + } + return text, true + } + return "", false +} + // calcPrep parses the expression and decimals once, at compile time, and places each // operand name at the position expand will read it into. checkCalc proved both args // valid, so no step here can fail; dp -1 prints the minimal form. @@ -132,26 +218,10 @@ func indexVars(n calcNode, at map[string]int) calcNode { return n } -// calcOperands lists the sibling-field names every {calc(...)} token in a format -// reads, so cycle detection sees the field edges calc renders through (a function -// token otherwise carries no field edge). -func calcOperands(format string) []string { - var names []string - _ = eachToken(format, func(t ftoken) error { - if t.kind == 'b' { - names = append(names, calcTokenOperands(t.body)...) - } - return nil - }) - return names -} - -// calcTokenOperands lists the sibling-field names one {token} body reads, empty -// for anything that is not a {calc(...)}. checkCalc reports an expression that does -// not parse, so one that does not simply names nothing here. -func calcTokenOperands(body string) []string { - name, args, ok := funcCall(body) - if !ok || name != "calc" || len(args) == 0 { +// calcOperands lists the sibling-field names a calc's args read. checkCalc reports +// an expression that does not parse, so one that does not simply names nothing. +func calcOperands(args []string) []string { + if len(args) == 0 { return nil } expr, err := parseCalc(args[0]) @@ -240,6 +310,7 @@ func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNod } } +// factor is a table-shaped scanner, one case per token kind, kept whole on purpose. func (p *calcParser) factor() (calcNode, error) { p.space() if p.pos >= len(p.rs) { diff --git a/calc_test.go b/calc_test.go index 1b9a3a2..77adf77 100644 --- a/calc_test.go +++ b/calc_test.go @@ -12,14 +12,14 @@ import ( func TestCalcArithmetic(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(2 + 3)}"}`: "5", - `{"format":"{calc(2 * 3 + 4)}"}`: "10", // * binds tighter than + - `{"format":"{calc(2 + 3 * 4)}"}`: "14", - `{"format":"{calc((2 + 3) * 4)}"}`: "20", // parentheses override - `{"format":"{calc(10 / 4)}"}`: "2.5", - `{"format":"{calc(-2 + 5)}"}`: "3", // unary minus - `{"format":"{calc(2 - -3)}"}`: "5", - `{"format":"{calc(1.5 * 2)}"}`: "3", // whole result drops the decimals + `"{calc(2 + 3)}"`: "5", + `"{calc(2 * 3 + 4)}"`: "10", // * binds tighter than + + `"{calc(2 + 3 * 4)}"`: "14", + `"{calc((2 + 3) * 4)}"`: "20", // parentheses override + `"{calc(10 / 4)}"`: "2.5", + `"{calc(-2 + 5)}"`: "3", // unary minus + `"{calc(2 - -3)}"`: "5", + `"{calc(1.5 * 2)}"`: "3", // whole result drops the decimals } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -33,9 +33,9 @@ func TestCalcArithmetic(t *testing.T) { func TestCalcAuto(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(10 / 3)}"}`: "3.3333333333333335", - `{"format":"{calc(6 / 2)}"}`: "3", - `{"format":"{calc(1 / 4)}"}`: "0.25", + `"{calc(10 / 3)}"`: "3.3333333333333335", + `"{calc(6 / 2)}"`: "3", + `"{calc(1 / 4)}"`: "0.25", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -49,9 +49,9 @@ func TestCalcAuto(t *testing.T) { func TestCalcDecimals(t *testing.T) { f := engine(1) cases := map[string]string{ - `{"format":"{calc(10 / 3, 2)}"}`: "3.33", - `{"format":"{calc(10 / 3, 0)}"}`: "3", - `{"format":"{calc(2 * 3, 2)}"}`: "6.00", + `"{calc(10 / 3, 2)}"`: "3.33", + `"{calc(10 / 3, 0)}"`: "3", + `"{calc(2 * 3, 2)}"`: "6.00", } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -63,43 +63,47 @@ func TestCalcDecimals(t *testing.T) { // TestCalcFields pins that bare names resolve to sibling fields, rendered then // parsed as numbers. func TestCalcFields(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{calc(price * qty, 2)}","price":["19.99"],"qty":["3"]}`); got != "59.97" { + if got := mustRender(t, engine(1), `{"format":"{calc(price * qty, 2)}","price":"19.99","qty":"3"}`); got != "59.97" { t.Fatalf("calc over fields = %q, want 59.97", got) } // A field that is itself a template renders before parsing. - if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":[{"format":"{n}","n":["10"]}],"b":["3"]}`); got != "7" { + if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" { t.Fatalf("calc(a - b) = %q, want 7", got) } } -// TestCalcNonNumericIsNaN pins the never-fail rule: a field that doesn't render -// to a number becomes NaN, which propagates and prints visibly. +// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does +// not render to a number becomes NaN then, which propagates and prints visibly. func TestCalcNonNumericIsNaN(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{calc(x * 2)}","x":["abc"]}`); got != "NaN" { - t.Fatalf("calc over non-numeric field = %q, want NaN", got) + f := engine(1) + for i := 0; i < 50; i++ { + if got := mustRender(t, f, `{"format":"{calc(x * 2)}","x":["abc","1"]}`); got == "NaN" { + return + } } + t.Fatal("calc over a sometimes non-numeric field never printed NaN in 50 draws") } // TestCalcReproducible pins that a calc over a random operand stays seed-stable. func TestCalcReproducible(t *testing.T) { - tmpl := `{"format":"{calc(q * 2 + 1)}","q":[{"format":"{int(1,1000000)}"}]}` + tmpl := `{"format":"{calc(q * 2 + 1)}","q":"{int(1,1000000)}"}` if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b { t.Fatalf("calc not reproducible: %q != %q", a, b) } } -// TestCalcTokenOperandsReadsOnlyACalc pins what the helper answers for a body that +// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that // is not a calc, and for one whose expression does not parse: nothing, either way. // checkCalc is what reports a bad expression, so the callers that run after it // never meet one — but they must not have to depend on that order to be safe. -func TestCalcTokenOperandsReadsOnlyACalc(t *testing.T) { +func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) { for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} { - if got := calcTokenOperands(body); got != nil { - t.Errorf("calcTokenOperands(%q) = %v, want none", body, got) + if got := tokenOperands(body); got != nil { + t.Errorf("tokenOperands(%q) = %v, want none", body, got) } } - if got := calcTokenOperands("calc(net * qty)"); len(got) != 2 { - t.Errorf("calcTokenOperands(calc(net * qty)) = %v, want both operands", got) + if got := tokenOperands("calc(net * qty)"); len(got) != 2 { + t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got) } } @@ -140,16 +144,16 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) { } // TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the -// expansion, not to the calc, so a bare token rendering the same name reads it too. +// expansion, not to the calc, so the bare token rendering the same name reads it too. func TestCalcOperandSharesOneDraw(t *testing.T) { dir := writeData(t, map[string]string{ - "same": `{"format":"{w} {w} {calc(w)}","w":["1","2","3","4","5"]}`, + "same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`, }) f := newGenerator(t, dir, WithSeed(5)) for i := 0; i < 200; i++ { got := fake(t, f, "same") - if p := strings.Fields(got); len(p) != 3 || p[0] != p[1] || p[0] != p[2] { - t.Fatalf("same = %q, want one value three times", got) + if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] { + t.Fatalf("same = %q, want one value twice", got) } } } @@ -218,3 +222,51 @@ func TestCalcHoldIsPerTemplate(t *testing.T) { t.Fatal("the nested template never differed from its parent, want its own draw") } } + +func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"{calc(x * 2)}","x":"abc"}`: `"abc"`, + `{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`, + `{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`, + } { + _, err := compile(parse(t, src)) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") { + t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want) + } + } + for _, ok := range []string{ + `{"format":"{calc(x * 2)}","x":"3"}`, + `{"format":"{calc(x * 2)}","x":" 2.5 "}`, + `{"format":"{calc(x * 2)}","x":["1","abc"]}`, + `{"format":"{calc(x * 2)}","x":"{digits(2)}"}`, + } { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v, want it accepted", ok, err) + } + } +} + +func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `"{calc(1/0)}"`: "divides by 0", + `"{calc(2/(1-1))}"`: "divides by (1 - 1)", + `{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y", + `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)", + `{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0", + `{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want) + } + } + if _, err := compile(parse(t, `{"format":"{calc(a/(b*c))}","a":"1","b":"0","c":["1","2"]}`)); err != nil { + t.Errorf("compile(a/(b*c)) = %v, want a divisor that varies accepted", err) + } + f := engine(1) + for i := 0; i < 50; i++ { + if got := mustRender(t, f, `{"format":"{calc(x/y)}","x":"1","y":["0","1"]}`); got == "+Inf" { + return + } + } + t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws") +} diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 24ac085..d6b2d05 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -1,122 +1,285 @@ -// Command fejkdata prints one fake value from one or more data directories. +// Command fejkdata prints fake values from the shipped data and any directories +// layered over it. // -// fejkdata -data-path ./data/sv_SE person # a full person -// fejkdata -data-path ./data/sv_SE person.last # just the surname (dotted path) -// fejkdata -data-path ./data sv_SE.person # point at the tree, address by folder -// fejkdata -data-path ./data/sv_SE -data-path ./mydata person # layer custom data; last dir wins -// fejkdata -seed 42 -data-path ./data/sv_SE address -// -// It is a thin CLI over the fejkdata library: New(dirs) then Fake(path). +// fejkdata sv_SE.person # a full person +// fejkdata sv_SE.person.last # just the surname +// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins +// fejkdata --seed 42 sv_SE.address package main import ( + "bufio" "errors" - "flag" "fmt" "io" "os" "runtime/debug" + "strconv" "strings" "gitea.larvit.se/larvit/fejkdata" ) -const usage = `Usage: fejkdata -data-path D [-data-path D]... [-seed N] [-repeat N] [-separator S] +const usage = `Usage: fejkdata [flags] - -data-path D a data directory, e.g. ./data/sv_SE (repeatable; last wins on clash) - a category, or a dotted path into one (person, person.last) - -seed N seed for reproducible output - -repeat N render the path N times (default 1) - -separator S string between repeated values (default newline) - -list list the paths the data offers, then exit - -version print the version, then exit + a category, or a dotted path into one (person, person.last) -Flags must come before .` + -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) + -h, --help print this help, then exit + --list list the paths the data offers, then exit + --no-shipped-data load only the --data-path directories + -n, --repeat N render the path N times, 1..1048576 (default 1) + -s, --seed N seed for reproducible output + --separator S string between repeated values (default newline) + --version print the version, then exit -// stringList collects a repeatable string flag, preserving order. -type stringList []string +Flags may come before or after ; -- ends the flags. A short flag's value +attaches or follows (-n3, -n 3); short flags bundle (-hn 3). +` -func (s *stringList) String() string { return strings.Join(*s, ",") } +type invocation struct { + dirs []string + help bool + list bool + noShipped bool + paths []string + repeat int + repeatSet bool + seed uint64 + seeded bool + separator string + separatorSet bool + version bool +} -func (s *stringList) Set(v string) error { *s = append(*s, v); return nil } +type flagDef struct { + long string + short string + value bool + set func(*invocation, string) error +} + +var flagDefs = []flagDef{ + {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, + {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, + {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, + {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, + {"repeat", "n", true, func(in *invocation, v string) error { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > fejkdata.MaxRepeat { + return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v) + } + in.repeat, in.repeatSet = n, true + return nil + }}, + {"seed", "s", true, func(in *invocation, v string) error { + n, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return fmt.Errorf("--seed needs an unsigned integer, got %q", v) + } + in.seed, in.seeded = n, true + return nil + }}, + {"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }}, + {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, +} + +func flagByLong(name string) *flagDef { + for i := range flagDefs { + if flagDefs[i].long == name { + return &flagDefs[i] + } + } + return nil +} + +func flagByShort(name string) *flagDef { + for i := range flagDefs { + if flagDefs[i].short == name { + return &flagDefs[i] + } + } + return nil +} + +// flagArg is one flag as written: its definition and the value attached to it, +// if any. +type flagArg struct { + def *flagDef + value string + inline bool +} + +// splitFlags reads one argv element as flags: --name or --name=value, or -abc +// where each letter is a short flag and the first that takes a value takes the +// rest of the element. +func splitFlags(arg string) ([]flagArg, error) { + if strings.HasPrefix(arg, "--") { + name, value, inline := strings.Cut(arg[2:], "=") + def := flagByLong(name) + if def == nil { + return nil, fmt.Errorf("unknown flag --%s", name) + } + return []flagArg{{def, value, inline}}, nil + } + if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil { + return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name) + } + var flags []flagArg + letters := arg[1:] + for i, r := range letters { + letter := string(r) + def := flagByShort(letter) + if def == nil { + return nil, fmt.Errorf("unknown flag -%s", letter) + } + if !def.value { + flags = append(flags, flagArg{def: def}) + continue + } + rest := letters[i+len(letter):] + if strings.HasPrefix(rest, "=") { + return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s", + letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:]) + } + return append(flags, flagArg{def, rest, rest != ""}), nil + } + return flags, nil +} + +func parseArgs(argv []string) (invocation, error) { + in := invocation{repeat: 1, separator: "\n"} + for i := 0; i < len(argv); i++ { + arg := argv[i] + if arg == "--" { + in.paths = append(in.paths, argv[i+1:]...) + return in, nil + } + if len(arg) < 2 || arg[0] != '-' { + in.paths = append(in.paths, arg) + continue + } + flags, err := splitFlags(arg) + if err != nil { + return in, err + } + for _, fl := range flags { + if !fl.def.value { + if fl.inline { + return in, fmt.Errorf("--%s takes no value", fl.def.long) + } + _ = fl.def.set(&in, "") + continue + } + value := fl.value + if !fl.inline { + if i++; i >= len(argv) { + return in, fmt.Errorf("--%s needs a value", fl.def.long) + } + value = argv[i] + } + if err := fl.def.set(&in, value); err != nil { + return in, err + } + } + } + return in, nil +} + +// check rejects a flag combination that cannot run. +func (in invocation) check() error { + if in.list && len(in.paths) > 0 { + return errors.New("--list takes no path") + } + if in.list && (in.repeatSet || in.separatorSet) { + return errors.New("--list takes no --repeat or --separator") + } + if !in.list && len(in.paths) != 1 { + return fmt.Errorf("expected one path, got %d", len(in.paths)) + } + return nil +} + +func (in invocation) options() []fejkdata.Option { + var opts []fejkdata.Option + if in.noShipped { + opts = append(opts, fejkdata.WithoutShippedData()) + } + for _, dir := range in.dirs { + opts = append(opts, fejkdata.WithDataPath(dir)) + } + if in.seeded { + opts = append(opts, fejkdata.WithSeed(in.seed)) + } + return opts +} + +// write streams the path's renders to w, repeat of them joined by the separator +// and ended by a newline. A path that renders once renders every time, so a Fake +// failure comes before anything is written; a write failure surfaces from Flush, +// bufio keeping the first one. +func (in invocation) write(f *fejkdata.Generator, w io.Writer) error { + out := bufio.NewWriter(w) + for i := 0; i < in.repeat; i++ { + v, err := f.Fake(in.paths[0]) + if err != nil { + return err + } + if i > 0 { + out.WriteString(in.separator) + } + out.WriteString(v) + } + out.WriteString("\n") + return out.Flush() +} func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -// run is main's testable core: it returns the process exit code (0 ok, 1 -// runtime error, 2 misuse) and writes only to the given streams. +// run returns the exit code: 0 ok, 1 runtime error, 2 misuse. func run(args []string, stdout, stderr io.Writer) int { - fs := flag.NewFlagSet("fejkdata", flag.ContinueOnError) - fs.SetOutput(stderr) - fs.Usage = func() { fmt.Fprintln(stderr, usage) } - seed := fs.Uint64("seed", 0, "seed for reproducible output") - repeat := fs.Int("repeat", 1, "render the path this many times") - sep := fs.String("separator", "\n", "string between repeated values") - list := fs.Bool("list", false, "list the paths the data offers, then exit") - showVersion := fs.Bool("version", false, "print the version, then exit") - var dirs stringList - fs.Var(&dirs, "data-path", "a data directory to load (repeatable)") - if err := fs.Parse(args); err != nil { - if errors.Is(err, flag.ErrHelp) { // -h/-help already printed usage - return 0 - } - return 2 + in, err := parseArgs(args) + if err != nil { + return misuse(stderr, err) } - if *showVersion { + if in.help { + fmt.Fprint(stdout, usage) + return 0 + } + if in.version { fmt.Fprintln(stdout, "fejkdata "+buildVersion()) return 0 } - if len(dirs) == 0 { - fs.Usage() - return 2 + if err := in.check(); err != nil { + return misuse(stderr, err) } - - var opts []fejkdata.Option - fs.Visit(func(fl *flag.Flag) { - if fl.Name == "seed" { - opts = append(opts, fejkdata.WithSeed(*seed)) - } - }) - - if *list { - f, err := fejkdata.New(dirs, opts...) - if err != nil { - fmt.Fprintln(stderr, err) - return 1 - } + f, err := fejkdata.New(in.options()...) + if errors.Is(err, fejkdata.ErrNoData) { + return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path")) + } + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if in.list { for _, p := range f.List() { fmt.Fprintln(stdout, p) } return 0 } - if fs.NArg() != 1 { - fs.Usage() - return 2 - } - if *repeat < 1 { - fmt.Fprintln(stderr, "repeat must be a positive integer") - return 2 - } - - path := fs.Arg(0) - f, err := fejkdata.New(dirs, opts...) - if err != nil { + if err := in.write(f, stdout); err != nil { fmt.Fprintln(stderr, err) return 1 } - vals := make([]string, *repeat) - for i := range vals { - if vals[i], err = f.Fake(path); err != nil { - fmt.Fprintln(stderr, err) - return 1 - } - } - fmt.Fprintln(stdout, strings.Join(vals, *sep)) return 0 } -// buildVersion reports the module version stamped into the binary by `go install` -// (or "devel" for a local build), read from the build info — no version constant -// to bump, no extra dependency. +func misuse(stderr io.Writer, err error) int { + fmt.Fprintf(stderr, "fejkdata: %v\ntry 'fejkdata --help'\n", err) + return 2 +} + +// buildVersion is the module version go install stamps into the binary, or "devel". func buildVersion() string { if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" { return info.Main.Version diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 1ced7b8..acc5a23 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "os" + "path/filepath" "strings" "testing" ) @@ -11,7 +13,6 @@ const ( enUS = "../../data/en_US" ) -// runOut runs the CLI and returns exit code, stdout, stderr. func runOut(args ...string) (int, string, string) { var out, errb bytes.Buffer code := run(args, &out, &errb) @@ -19,7 +20,7 @@ func runOut(args ...string) (int, string, string) { } func TestRunOutputsValue(t *testing.T) { - code, out, errb := runOut("-data-path", svSE, "person") + code, out, errb := runOut("--data-path", svSE, "person") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -32,12 +33,11 @@ func TestRunOutputsValue(t *testing.T) { } func TestRunDotPath(t *testing.T) { - // person.last descends to just a surname — never the full "First Last". - code, full, _ := runOut("-seed", "7", "-data-path", svSE, "person") + code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person") if code != 0 { t.Fatalf("person run = %d", code) } - code, last, errb := runOut("-seed", "7", "-data-path", svSE, "person.last") + code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last") if code != 0 { t.Fatalf("person.last run = %d, stderr=%q", code, errb) } @@ -49,17 +49,123 @@ func TestRunDotPath(t *testing.T) { } } -func TestRunSeedDeterministic(t *testing.T) { - _, a, _ := runOut("-seed", "42", "-data-path", svSE, "address") - _, b, _ := runOut("-seed", "42", "-data-path", svSE, "address") - if a != b { - t.Errorf("same seed diverged: %q != %q", a, b) +func TestRunSeedSpellings(t *testing.T) { + _, want, _ := runOut("--seed", "42", "--data-path", svSE, "address") + for _, args := range [][]string{ + {"--seed=42", "--data-path", svSE, "address"}, + {"-s", "42", "-d", svSE, "address"}, + {"--data-path", svSE, "address", "--seed", "42"}, + } { + code, got, errb := runOut(args...) + if code != 0 { + t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb) + } + if got != want { + t.Errorf("run(%v) = %q, want %q", args, got, want) + } + } +} + +func TestRunShortFlagValues(t *testing.T) { + _, want, _ := runOut("--seed", "42", "--data-path", svSE, "address") + for _, args := range [][]string{ + {"-s42", "-d", svSE, "address"}, + {"-s", "42", "-d" + svSE, "address"}, + {"-d", svSE, "address", "-s42"}, + } { + code, got, errb := runOut(args...) + if code != 0 { + t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb) + } + if got != want { + t.Errorf("run(%v) = %q, want %q", args, got, want) + } + } + _, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word") + if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 { + t.Errorf("-n3 gave %d lines: %q", len(lines), three) + } + for _, c := range []struct { + args []string + want string + }{ + {[]string{"-s=42", "-d", svSE, "address"}, "-s42"}, + {[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"}, + {[]string{"-d=" + svSE, "address"}, "--data-path="}, + {[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`}, + {[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`}, + } { + code, out, errb := runOut(c.args...) + if code != 2 || out != "" { + t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } + } + code, out, _ := runOut("-hd", svSE) + if code != 0 || !strings.Contains(out, "Usage") { + t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out) + } +} + +func TestRunDoubleDashEndsFlags(t *testing.T) { + code, out, errb := runOut("--data-path", svSE, "--", "person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb) + } + code, _, errb = runOut("--data-path", svSE, "--", "--list") + if code != 1 || !strings.Contains(errb, "--list") { + t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb) + } +} + +func TestRunSingleDashLongIsRejected(t *testing.T) { + for _, c := range []struct { + args []string + want string + }{ + {[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"}, + {[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"}, + {[]string{"-data-path", svSE, "person"}, "use --data-path"}, + {[]string{"-list", "-d", svSE}, "use --list"}, + {[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"}, + {[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"}, + } { + code, _, errb := runOut(c.args...) + if code != 2 { + t.Errorf("run(%v) = %d, want 2", c.args, code) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } + } +} + +func TestRunFlagValues(t *testing.T) { + for _, c := range []struct { + args []string + want string + }{ + {[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`}, + {[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"}, + {[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"}, + {[]string{"--list=1", "-d", svSE}, "--list takes no value"}, + {[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"}, + {[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"}, + } { + code, _, errb := runOut(c.args...) + if code != 2 { + t.Errorf("run(%v) = %d, want 2", c.args, code) + } + if !strings.Contains(errb, c.want) { + t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want) + } } } func TestRunRepeat(t *testing.T) { - // -repeat N prints N values, one per line by default. - code, out, errb := runOut("-seed", "1", "-repeat", "3", "-data-path", svSE, "word") + code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -67,11 +173,14 @@ func TestRunRepeat(t *testing.T) { if len(lines) != 3 { t.Errorf("repeat=3 gave %d lines: %q", len(lines), out) } + _, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word") + if short != out { + t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out) + } } func TestRunSeparator(t *testing.T) { - // -separator joins the repeated values instead of newlines. - code, out, errb := runOut("-repeat", "3", "-separator", ",", "-data-path", svSE, "word") + code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -84,8 +193,7 @@ func TestRunSeparator(t *testing.T) { } func TestRunRepeatAdvancesRNG(t *testing.T) { - // Each repeat is a fresh draw, not the same value N times. - code, out, errb := runOut("-seed", "1", "-repeat", "5", "-data-path", svSE, "person") + code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -98,35 +206,23 @@ func TestRunRepeatAdvancesRNG(t *testing.T) { } } -func TestRunRepeatInvalid(t *testing.T) { - // A non-positive repeat is misuse. - for _, r := range []string{"0", "-1"} { - code, _, errb := runOut("-repeat", r, "-data-path", svSE, "word") - if code != 2 { - t.Errorf("repeat=%s = %d, want 2", r, code) - } - if errb == "" { - t.Errorf("repeat=%s: want an error message", r) - } - } -} - -func TestRunUsageOnMissingArgs(t *testing.T) { - // Need at least one -data-path and exactly one positional path; else misuse. - for _, args := range [][]string{{}, {"-data-path", svSE}, {"person"}} { - code, _, errb := runOut(args...) +func TestRunMisuse(t *testing.T) { + for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} { + code, out, errb := runOut(args...) if code != 2 { t.Errorf("run(%v) = %d, want 2", args, code) } - if !strings.Contains(errb, "Usage") { - t.Errorf("run(%v) stderr = %q, want usage", args, errb) + if !strings.Contains(errb, "try 'fejkdata --help'") { + t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb) + } + if out != "" { + t.Errorf("run(%v) stdout = %q, want nothing", args, out) } } } func TestRunMultipleDataPaths(t *testing.T) { - // -data-path repeats: dirs merge, last wins; the path stays positional. - code, out, errb := runOut("-data-path", enUS, "-data-path", svSE, "person") + code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person") if code != 0 { t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb) } @@ -136,7 +232,7 @@ func TestRunMultipleDataPaths(t *testing.T) { } func TestRunUnknownCategoryFails(t *testing.T) { - code, _, errb := runOut("-data-path", svSE, "nope") + code, _, errb := runOut("--data-path", svSE, "nope") if code != 1 { t.Fatalf("run = %d, want 1", code) } @@ -146,8 +242,7 @@ func TestRunUnknownCategoryFails(t *testing.T) { } func TestRunList(t *testing.T) { - // -list prints the discoverable paths and exits 0, no positional path needed. - code, out, errb := runOut("-data-path", svSE, "-list") + code, out, errb := runOut("--data-path", svSE, "--list") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } @@ -158,32 +253,31 @@ func TestRunList(t *testing.T) { } } -func TestRunHelpExitsZero(t *testing.T) { - // -h/-help is success (0), not misuse (2), so scripts under `set -e` survive. - for _, h := range []string{"-h", "-help"} { - code, _, errb := runOut(h) +func TestRunHelpOnStdout(t *testing.T) { + for _, h := range []string{"-h", "--help"} { + code, out, errb := runOut(h) if code != 0 { t.Errorf("%s = %d, want 0", h, code) } - if !strings.Contains(errb, "Usage") { - t.Errorf("%s: stderr = %q, want usage", h, errb) + if !strings.Contains(out, "Usage") || errb != "" { + t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb) } } } func TestRunVersion(t *testing.T) { - code, out, errb := runOut("-version") + code, out, errb := runOut("--version") if code != 0 { - t.Fatalf("-version = %d, stderr=%q", code, errb) + t.Fatalf("--version = %d, stderr=%q", code, errb) } version, ok := strings.CutPrefix(out, "fejkdata ") if !ok || strings.TrimSpace(version) == "" { - t.Errorf("-version = %q, want the command name and a version on stdout", out) + t.Errorf("--version = %q, want the command name and a version on stdout", out) } } func TestRunMissingDirFails(t *testing.T) { - code, _, errb := runOut("-data-path", "../../data/nope", "person") + code, _, errb := runOut("--data-path", "../../data/nope", "person") if code != 1 { t.Fatalf("run = %d, want 1", code) } @@ -191,3 +285,88 @@ func TestRunMissingDirFails(t *testing.T) { t.Error("want an error message on stderr") } } + +func TestRunShippedDataByDefault(t *testing.T) { + code, out, errb := runOut("--seed", "1", "sv_SE.person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb) + } + code, list, _ := runOut("--list") + if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") { + t.Errorf("--list without --data-path = %d, %q", code, list) + } + code, _, errb = runOut("person") + if code != 1 || !strings.Contains(errb, "person") { + t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb) + } +} + +func TestRunNoShippedData(t *testing.T) { + code, list, errb := runOut("--no-shipped-data", "-d", svSE, "--list") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + if strings.Contains(list, "en_US") || !strings.Contains(list, "person\n") { + t.Errorf("--no-shipped-data --list = %q, want only the given dir", list) + } + code, out, _ := runOut("--no-shipped-data", "-d", svSE, "-s", "3", "person") + if code != 0 || strings.TrimSpace(out) == "" { + t.Errorf("run = %d, out=%q", code, out) + } + code, _, errb = runOut("--no-shipped-data", "sv_SE.person") + if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") { + t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb) + } +} + +func TestRunRepeatIsBounded(t *testing.T) { + for _, args := range [][]string{ + {"--repeat", "9223372036854775807", "sv_SE.word"}, + {"--repeat", "1048577", "sv_SE.word"}, + {"-n", "99999999999", "sv_SE.word"}, + } { + code, out, errb := runOut(args...) + if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") { + t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb) + } + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil { + t.Fatal(err) + } + code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x") + if code != 0 || len(out) != 1048576+1 { + t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb) + } +} + +func TestRunListTakesNoRepeatOrSeparator(t *testing.T) { + for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} { + code, _, errb := runOut(args...) + if code != 2 || !strings.Contains(errb, "--list takes no") { + t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb) + } + } +} + +func TestRunUnknownFlagIsNamedByRune(t *testing.T) { + code, _, errb := runOut("-ä", "sv_SE.word") + if code != 2 || !strings.Contains(errb, "unknown flag -ä") { + t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb) + } +} + +func TestRunEmptyDataPathIsNamed(t *testing.T) { + code, _, errb := runOut("--data-path=", "sv_SE.word") + if code != 1 || !strings.Contains(errb, "empty") { + t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb) + } +} + +func TestRunNumbersFollowTheShell(t *testing.T) { + _, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word") + code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word") + if code != 0 || got != want { + t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want) + } +} diff --git a/compose.yaml b/compose.yaml index 18f706c..bb47873 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,7 +28,7 @@ x-go: &go services: test: <<: *go - command: go test ./... + command: go test -race ./... cover: <<: *go @@ -42,6 +42,10 @@ services: <<: *go command: go vet ./... + cyclo: + <<: *go + command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . + build: <<: *go command: go build ./... diff --git a/data.go b/data.go index a5b0403..fd2b3f1 100644 --- a/data.go +++ b/data.go @@ -3,29 +3,64 @@ package fejkdata import ( "encoding/json" "fmt" + "io/fs" "os" - "path/filepath" + "path" "strings" ) -// loadData loads every given directory into one namespace tree and returns its -// root children. A directory becomes a group; each *.json file in it compiles to -// a node keyed by its base name (address.json -> "address"); each subdirectory -// becomes a nested group, so folders turn into dot-path segments. Paths are -// merged left to right: matching groups merge by their children, and any other -// clash (a leaf, or a leaf-vs-group) is won by the last directory loaded. Once -// merged, linkRefs binds every {..path} reference against the final tree. -func loadData(paths []string) (map[string]node, error) { +// dataSource is one tree to load: an fs.FS and the directory in it to start from. +// label prefixes file names in errors; onDisk marks path as a directory that must +// exist. +type dataSource struct { + fsys fs.FS + label string + onDisk bool + path string + root string +} + +func (s dataSource) name(p string) string { + if s.label == "" { + return p + } + return path.Join(s.label, p) +} + +// loadData loads every source into one namespace tree and returns its root +// children. A directory becomes a group; each *.json file in it compiles to a node +// keyed by its base name (address.json -> "address"); each subdirectory becomes a +// nested group, so folders turn into dot-path segments. Sources merge left to right: +// matching groups merge by their children, and any other clash is won by the last +// source loaded. Once merged, linkRefs binds every reference against the +// final tree. +func loadData(sources []dataSource) (map[string]node, error) { root := map[string]node{} - for _, p := range paths { - g, err := loadDir(p) + for _, src := range sources { + if src.onDisk { + if src.path == "" { + return nil, fmt.Errorf("a data path is empty") + } + info, err := os.Stat(src.path) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", src.path) + } + } + dir := src.root + if dir == "" { + dir = "." + } + g, err := loadDir(src, dir) if err != nil { return nil, err } mergeChildren(root, g.children) } if len(root) == 0 { - return nil, fmt.Errorf("no .json data found in %v", paths) + return nil, fmt.Errorf("no .json data found") } if err := linkRefs(root); err != nil { return nil, err @@ -33,71 +68,82 @@ func loadData(paths []string) (map[string]node, error) { if err := checkNoCycles(root); err != nil { return nil, err } + if err := checkRepeatReach(root); err != nil { + return nil, err + } if err := checkBoundLevelsHeld(root); err != nil { return nil, err } return root, nil } -// loadDir compiles one directory into a group. os.ReadDir yields entries sorted +// loadDir compiles one directory into a group. fs.ReadDir yields entries sorted // by name, so the tree is built deterministically. Empty subdirectories (no JSON // anywhere under them) are skipped rather than added as empty namespaces. -func loadDir(dir string) (*group, error) { - info, err := os.Stat(dir) +func loadDir(src dataSource, dir string) (*group, error) { + entries, err := fs.ReadDir(src.fsys, dir) if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, fmt.Errorf("%s is not a directory", dir) - } - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", src.name(dir), err) } g := &group{children: map[string]node{}} for _, e := range entries { if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data continue } - full := filepath.Join(dir, e.Name()) + full := path.Join(dir, e.Name()) + load := loadFile if e.IsDir() { - child, err := loadDir(full) - if err != nil { - return nil, err - } - if len(child.children) == 0 { - continue - } - if err := checkName(e.Name()); err != nil { - return nil, fmt.Errorf("%s: folder %w", full, err) - } - g.children[e.Name()] = child - continue + load = loadFolder } - if !strings.HasSuffix(e.Name(), ".json") { - continue - } - name := strings.TrimSuffix(e.Name(), ".json") - if err := checkName(name); err != nil { - return nil, fmt.Errorf("%s: category %w", full, err) - } - b, err := os.ReadFile(full) - if err != nil { + if err := load(src, g, full, e.Name()); err != nil { return nil, err } - var raw any - if err := json.Unmarshal(b, &raw); err != nil { - return nil, fmt.Errorf("%s: %w", full, err) - } - n, err := compile(raw) - if err != nil { - return nil, fmt.Errorf("%s: %w", full, err) - } - g.children[name] = n } return g, nil } +// loadFolder adds a subdirectory as a nested group, unless nothing under it is data. +func loadFolder(src dataSource, g *group, full, name string) error { + child, err := loadDir(src, full) + if err != nil { + return err + } + if len(child.children) == 0 { + return nil + } + if err := checkName(name); err != nil { + return fmt.Errorf("%s: folder %w", src.name(full), err) + } + g.children[name] = child + return nil +} + +// loadFile compiles a *.json file into a category named after it; any other file +// is skipped. +func loadFile(src dataSource, g *group, full, file string) error { + if !strings.HasSuffix(file, ".json") { + return nil + } + name := strings.TrimSuffix(file, ".json") + if err := checkName(name); err != nil { + return fmt.Errorf("%s: category %w", src.name(full), err) + } + b, err := fs.ReadFile(src.fsys, full) + if err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + var raw any + if err := json.Unmarshal(b, &raw); err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + n, err := compile(raw) + if err != nil { + return fmt.Errorf("%s: %w", src.name(full), err) + } + g.children[name] = n + return nil +} + // mergeChildren overlays src onto dst. Two groups under the same key merge // recursively (so locales/categories from several paths combine); every other // key is replaced, making the last-loaded directory win on a conflict. diff --git a/data/en_US/address.json b/data/en_US/address.json index 0fd122e..d48eebe 100644 --- a/data/en_US/address.json +++ b/data/en_US/address.json @@ -1,24 +1,17 @@ -[ - { - "format": "{street-number} {street}\n{locality}, {region} {postal-code}", - "street": [ - { - "format": "{name} {suffix}", - "name": ["Adams", "Ashby", "Aspen", "Bay", "Birch", "Bridge", "Cedar", "Chestnut", "Church", "Clark", "Cypress", "Dogwood", "Elm", "Forest", "Franklin", "Garden", "Grove", "Hawthorn", "Hickory", "Highland", "Jackson", "Jefferson", "Juniper", "Lake", "Laurel", "Liberty", "Lincoln", "Madison", "Magnolia", "Maple", "Market", "Meadow", "Mill", "Oak", "Park", "Pine", "Poplar", "Prospect", "Ridge", "River", "Spruce", "Sunset", "Sycamore", "Union", "Walnut", "Washington", "Willow", "Wilson"], - "suffix": ["Avenue", "Boulevard", "Circle", "Court", "Drive", "Lane", "Place", "Road", "Street", "Terrace", "Trail", "Way"] - } - ], - "street-number": [ - { "format": "10" }, - { "format": "100", "weight": 2 }, - { "format": "1000", "weight": 0.5 }, - { "format": "10100", "weight": 0.3 } - ], - "locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"], - "region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"], - "postal-code": [ - { "format": "10000" }, - { "format": "10000-0000", "weight": 0.3 } - ] - } -] +{ + "format": "{street-number} {street}\n{locality}, {region} {postal-code}", + "street": { + "format": "{name} {suffix}", + "name": ["Adams", "Ashby", "Aspen", "Bay", "Birch", "Bridge", "Cedar", "Chestnut", "Church", "Clark", "Cypress", "Dogwood", "Elm", "Forest", "Franklin", "Garden", "Grove", "Hawthorn", "Hickory", "Highland", "Jackson", "Jefferson", "Juniper", "Lake", "Laurel", "Liberty", "Lincoln", "Madison", "Magnolia", "Maple", "Market", "Meadow", "Mill", "Oak", "Park", "Pine", "Poplar", "Prospect", "Ridge", "River", "Spruce", "Sunset", "Sycamore", "Union", "Walnut", "Washington", "Willow", "Wilson"], + "suffix": ["Avenue", "Boulevard", "Circle", "Court", "Drive", "Lane", "Place", "Road", "Street", "Terrace", "Trail", "Way"] + }, + "street-number": [ + "{int(10,99)}", + { "format": "{int(100,999)}", "weight": 2 }, + { "format": "{int(1000,9999)}", "weight": 0.5 }, + { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 } + ], + "locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"], + "region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"], + "postal-code": ["{int(10000,99999)}", { "format": "{int(10000,99999)}-{digits(4)}", "weight": 0.3 }] +} diff --git a/data/en_US/color.json b/data/en_US/color.json index d472ec4..c6cf983 100644 --- a/data/en_US/color.json +++ b/data/en_US/color.json @@ -1,6 +1,6 @@ [ { - "format": "##{x}{x}{x}{x}{x}{x}", + "format": "#{x}{x}{x}{x}{x}{x}", "x": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], "weight": 2 }, diff --git a/data/en_US/company.json b/data/en_US/company.json index ab045a4..5b27b76 100644 --- a/data/en_US/company.json +++ b/data/en_US/company.json @@ -1,14 +1,12 @@ -[ - { - "format": "{base} {suffix}", - "base": [ - { - "format": "{a}{b}", - "a": ["Acme", "Apex", "Atlas", "Aurora", "Blue", "Bright", "Cedar", "Crest", "Delta", "Eagle", "Ever", "Fair", "Falcon", "Iron", "Maple", "North", "Onyx", "Pioneer", "Polar", "Prime", "Quartz", "River", "Silver", "Solar", "Summit", "Sun", "Swift", "True", "Vertex", "West"], - "b": ["bridge", "core", "field", "flow", "forge", "gate", "grid", "hub", "line", "logic", "mark", "nest", "path", "point", "scape", "soft", "span", "spark", "stone", "tech", "ware", "wave", "works", "yard"] - }, - ["Atlas Industries", "Beacon Labs", "Cedar & Vine", "Compass Group", "Evergreen Trading", "Harbor Logistics", "Hudson Partners", "Liberty Holdings", "Meridian Capital", "Northgate Studios", "Pinnacle Systems", "Riverside", "Sequoia Foods", "Smith & Sons", "Stonebridge Ventures", "Sunrise Bakery", "Wexford Holdings"] - ], - "suffix": ["Co.", "Corp.", "Group", "Inc.", "Inc.", "LLC", "LLC", "Ltd."] - } -] +{ + "format": "{base} {suffix}", + "base": [ + { + "format": "{a}{b}", + "a": ["Acme", "Apex", "Atlas", "Aurora", "Blue", "Bright", "Cedar", "Crest", "Delta", "Eagle", "Ever", "Fair", "Falcon", "Iron", "Maple", "North", "Onyx", "Pioneer", "Polar", "Prime", "Quartz", "River", "Silver", "Solar", "Summit", "Sun", "Swift", "True", "Vertex", "West"], + "b": ["bridge", "core", "field", "flow", "forge", "gate", "grid", "hub", "line", "logic", "mark", "nest", "path", "point", "scape", "soft", "span", "spark", "stone", "tech", "ware", "wave", "works", "yard"] + }, + ["Atlas Industries", "Beacon Labs", "Cedar & Vine", "Compass Group", "Evergreen Trading", "Harbor Logistics", "Hudson Partners", "Liberty Holdings", "Meridian Capital", "Northgate Studios", "Pinnacle Systems", "Riverside", "Sequoia Foods", "Smith & Sons", "Stonebridge Ventures", "Sunrise Bakery", "Wexford Holdings"] + ], + "suffix": ["Co.", "Corp.", "Group", { "format": "Inc.", "weight": 2 }, { "format": "LLC", "weight": 2 }, "Ltd."] +} diff --git a/data/en_US/date.json b/data/en_US/date.json index afee0f8..cf3b880 100644 --- a/data/en_US/date.json +++ b/data/en_US/date.json @@ -1,15 +1,13 @@ -[ - { - "format": "{month}/{day}/{year}", - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], - "year": [ - { "format": "#1970" }, - { "format": "#1980" }, - { "format": "#1990", "weight": 2 }, - { "format": "2#0#00", "weight": 3 }, - { "format": "2#0#10", "weight": 3 }, - { "format": "2#020", "weight": 2 } - ] - } -] +{ + "format": "{month}/{day}/{year}", + "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], + "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], + "year": [ + "197{digits(1)}", + "198{digits(1)}", + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } + ] +} diff --git a/data/en_US/email.json b/data/en_US/email.json index efd9e8f..a24823b 100644 --- a/data/en_US/email.json +++ b/data/en_US/email.json @@ -1,29 +1,26 @@ -[ - { - "format": "{local}@{domain}", - "local": [ - { - "format": "{first}{sep}{last}{n}", - "weight": 3, - "first": ["alex", "amelia", "andrew", "anna", "ava", "benjamin", "carter", "charlotte", "chloe", "chris", "daniel", "david", "dylan", "elijah", "ella", "emily", "emma", "ethan", "evelyn", "gabriel", "grace", "hannah", "harper", "henry", "isaac", "isabella", "jack", "jacob", "james", "john", "joseph", "julia", "leah", "liam", "linda", "lily", "logan", "lucas", "luke", "mary", "mason", "matthew", "maya", "mia", "michael", "natalie", "noah", "nora", "oliver", "olivia", "owen", "patricia", "robert", "ryan", "samuel", "sarah", "sofia", "sophia", "thomas", "victoria", "william", "zoe"], - "last": ["adams", "allen", "anderson", "bailey", "baker", "bennett", "brooks", "brown", "campbell", "carter", "clark", "collins", "cook", "cooper", "davis", "edwards", "evans", "fisher", "flores", "foster", "garcia", "gonzalez", "gray", "green", "hall", "harris", "hayes", "hernandez", "hill", "howard", "hughes", "jackson", "jenkins", "johnson", "jones", "kelly", "king", "lee", "lewis", "long", "lopez", "martin", "martinez", "miller", "mitchell", "moore", "morgan", "morris", "murphy", "nelson", "parker", "perez", "perry", "phillips", "powell", "price", "reed", "reyes", "rivera", "roberts", "robinson", "rodriguez", "rogers", "ross", "russell", "sanchez", "sanders", "scott", "smith", "stewart", "sullivan", "taylor", "thomas", "thompson", "torres", "turner", "walker", "ward", "watson", "white", "williams", "wilson", "wood", "wright", "young"], - "sep": [".", ".", "_", "-", ""], - "n": ["", "", "", "1", "7", "21", "42", "88", "99", "2024"] - }, - { - "format": "{adj}{noun}{n}", - "weight": 2, - "adj": ["blue", "cool", "dark", "epic", "fast", "fluffy", "frosty", "funky", "fuzzy", "golden", "grumpy", "happy", "hyper", "jazzy", "lazy", "lucky", "mega", "mighty", "neon", "ninja", "quiet", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "spicy", "super", "swift", "turbo", "wild", "witty", "zany"], - "noun": ["badger", "banana", "comet", "dragon", "falcon", "ferret", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "ninja", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], - "n": ["", "", "7", "13", "14", "42", "77", "99", "123", "420", "2024"] - }, - { - "format": "{w}{n}", - "weight": 1, - "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] - } - ], - "domain": ["example.com", "example.org", "example.net", "example.io", "example.co", "example.dev", "example.app", "mail.example.com", "mail.example.org", "webmail.example.net", "inbox.example.com", "post.example.org", "demo.example.net", "test.example.org", "dev.example.io"] - } -] +{ + "format": "{local}@{domain}", + "local": [ + { + "format": "{first}{sep}{last}{n}", + "weight": 3, + "first": ["alex", "amelia", "andrew", "anna", "ava", "benjamin", "carter", "charlotte", "chloe", "chris", "daniel", "david", "dylan", "elijah", "ella", "emily", "emma", "ethan", "evelyn", "gabriel", "grace", "hannah", "harper", "henry", "isaac", "isabella", "jack", "jacob", "james", "john", "joseph", "julia", "leah", "liam", "linda", "lily", "logan", "lucas", "luke", "mary", "mason", "matthew", "maya", "mia", "michael", "natalie", "noah", "nora", "oliver", "olivia", "owen", "patricia", "robert", "ryan", "samuel", "sarah", "sofia", "sophia", "thomas", "victoria", "william", "zoe"], + "last": ["adams", "allen", "anderson", "bailey", "baker", "bennett", "brooks", "brown", "campbell", "carter", "clark", "collins", "cook", "cooper", "davis", "edwards", "evans", "fisher", "flores", "foster", "garcia", "gonzalez", "gray", "green", "hall", "harris", "hayes", "hernandez", "hill", "howard", "hughes", "jackson", "jenkins", "johnson", "jones", "kelly", "king", "lee", "lewis", "long", "lopez", "martin", "martinez", "miller", "mitchell", "moore", "morgan", "morris", "murphy", "nelson", "parker", "perez", "perry", "phillips", "powell", "price", "reed", "reyes", "rivera", "roberts", "robinson", "rodriguez", "rogers", "ross", "russell", "sanchez", "sanders", "scott", "smith", "stewart", "sullivan", "taylor", "thomas", "thompson", "torres", "turner", "walker", "ward", "watson", "white", "williams", "wilson", "wood", "wright", "young"], + "sep": [{ "format": ".", "weight": 2 }, "_", "-", ""], + "n": [{ "format": "", "weight": 3 }, "1", "7", "21", "42", "88", "99", "2024"] + }, + { + "format": "{adj}{noun}{n}", + "weight": 2, + "adj": ["blue", "cool", "dark", "epic", "fast", "fluffy", "frosty", "funky", "fuzzy", "golden", "grumpy", "happy", "hyper", "jazzy", "lazy", "lucky", "mega", "mighty", "neon", "ninja", "quiet", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "spicy", "super", "swift", "turbo", "wild", "witty", "zany"], + "noun": ["badger", "banana", "comet", "dragon", "falcon", "ferret", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "ninja", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "14", "42", "77", "99", "123", "420", "2024"] + }, + { + "format": "{w}{n}", + "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + } + ], + "domain": ["example.com", "example.org", "example.net", "example.io", "example.co", "example.dev", "example.app", "mail.example.com", "mail.example.org", "webmail.example.net", "inbox.example.com", "post.example.org", "demo.example.net", "test.example.org", "dev.example.io"] +} diff --git a/data/en_US/ip.json b/data/en_US/ip.json index 2f071a7..c9bf575 100644 --- a/data/en_US/ip.json +++ b/data/en_US/ip.json @@ -2,9 +2,11 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "0", "weight": 3 }, { "format": "10", "weight": 4 }, { "format": "#100", "weight": 2 }] + "o": [ + { "format": "{digits(1)}", "weight": 3 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "1{digits(2)}", "weight": 2 } + ] }, - { - "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" - } + "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" ] diff --git a/data/en_US/person.json b/data/en_US/person.json index c6ff20f..3377a48 100644 --- a/data/en_US/person.json +++ b/data/en_US/person.json @@ -1,16 +1,7 @@ -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"], - "malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"], - "last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"], - "prefix": [ - "", - { - "format": "{title} ", - "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], - "weight": 0.1 - } - ] - } -] +{ + "format": "{prefix}{femalefirst|malefirst} {last}", + "femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"], + "malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"], + "last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"], + "prefix": ["", { "format": "{title} ", "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], "weight": 0.1 }] +} diff --git a/data/en_US/phone.json b/data/en_US/phone.json index 4866991..b47298b 100644 --- a/data/en_US/phone.json +++ b/data/en_US/phone.json @@ -3,13 +3,13 @@ "format": "({area}) {exch}-{line}", "weight": 2, "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "100" }], - "line": [{ "format": "0000" }] + "exch": "{int(100,999)}", + "line": "{digits(4)}" }, { "format": "{area}-{exch}-{line}", "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], - "exch": [{ "format": "100" }], - "line": [{ "format": "0000" }] + "exch": "{int(100,999)}", + "line": "{digits(4)}" } ] diff --git a/data/en_US/price.json b/data/en_US/price.json index 25197d2..2719fa0 100644 --- a/data/en_US/price.json +++ b/data/en_US/price.json @@ -1,19 +1,12 @@ -[ - { - "format": "${amt}.{cents}", - "amt": [ - { "format": "1", "weight": 2 }, - { "format": "10", "weight": 4 }, - { "format": "100", "weight": 3 }, - { "format": "1#,000", "weight": 1 }, - { "format": "10#,000", "weight": 0.4 }, - { "format": "100#,000", "weight": 0.1 } - ], - "cents": [ - { "format": "00", "weight": 3 }, - "49", - "95", - "99" - ] - } -] +{ + "format": "${amt}.{cents}", + "amt": [ + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + "{int(1,9)},{digits(3)}", + { "format": "{int(10,99)},{digits(3)}", "weight": 0.4 }, + { "format": "{int(100,999)},{digits(3)}", "weight": 0.1 } + ], + "cents": [{ "format": "{digits(2)}", "weight": 3 }, "49", "95", "99"] +} diff --git a/data/en_US/sentence.json b/data/en_US/sentence.json index d844cf6..2f02dfe 100644 --- a/data/en_US/sentence.json +++ b/data/en_US/sentence.json @@ -7,7 +7,7 @@ "noun": ["anchor", "archer", "beacon", "bridge", "captain", "cavern", "cottage", "ember", "engine", "falcon", "ferry", "forest", "fox", "garden", "glacier", "harbor", "hermit", "hollow", "island", "lantern", "lighthouse", "mariner", "meadow", "mountain", "orchard", "otter", "pilgrim", "prairie", "quarry", "raven", "ridge", "river", "sailor", "scholar", "signal", "sparrow", "stranger", "summit", "thicket", "tower", "traveler", "valley", "voyager", "wanderer", "willow", "woodland"], "verb": ["abandons", "anchors", "awakens", "befriends", "builds", "calms", "carries", "chases", "conceals", "crosses", "embraces", "follows", "gathers", "greets", "guards", "guides", "haunts", "heralds", "honors", "kindles", "leads", "mends", "mirrors", "outlasts", "ponders", "reveals", "salutes", "shapes", "shelters", "shields", "summons", "tends", "traces", "weathers"], "prep": ["above", "across", "against", "alongside", "amid", "around", "beneath", "beside", "between", "beyond", "near", "over", "past", "through", "toward", "under", "within"], - "end": [".", ".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 4 }, "!"] }, { "format": "{lead} {noun} {adv} {verb} {prep} {adj} {noun}{end}", @@ -18,7 +18,7 @@ "verb": ["builds", "carries", "chases", "crosses", "follows", "gathers", "guards", "guides", "leads", "mirrors", "outlasts", "reveals", "shapes", "shelters", "summons", "tends", "traces", "weathers"], "adj": ["ancient", "bright", "distant", "fierce", "fragile", "gentle", "golden", "hidden", "lonely", "quiet", "restless", "rugged", "silent", "stormy", "weary", "wild"], "prep": ["across", "against", "alongside", "amid", "beneath", "between", "beyond", "near", "over", "past", "through", "toward", "within"], - "end": [".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 3 }, "!"] }, { "format": "{q} {adj} {noun} {verb} {prep} {adj} {noun}?", diff --git a/data/en_US/ssn.json b/data/en_US/ssn.json index d7ff95d..501335d 100644 --- a/data/en_US/ssn.json +++ b/data/en_US/ssn.json @@ -1,3 +1 @@ -[ - { "format": "100-00-0000" } -] +"{int(100,999)}-{digits(2)}-{digits(4)}" diff --git a/data/en_US/time.json b/data/en_US/time.json index 0a04213..1c69ac9 100644 --- a/data/en_US/time.json +++ b/data/en_US/time.json @@ -1,8 +1,6 @@ -[ - { - "format": "{hour}:{minute} {ampm}", - "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], - "minute": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], - "ampm": ["AM", "PM"] - } -] +{ + "format": "{hour}:{minute} {ampm}", + "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + "minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "ampm": ["AM", "PM"] +} diff --git a/data/en_US/url.json b/data/en_US/url.json index d033d72..29a784b 100644 --- a/data/en_US/url.json +++ b/data/en_US/url.json @@ -1,12 +1,17 @@ -[ - { - "format": "https://{host}{path}", - "host": ["example.com", "www.example.com", "example.org", "www.example.org", "example.net", "example.io", "example.co", "blog.example.com", "blog.example.net", "shop.example.com", "store.example.org", "app.example.io", "api.example.com", "docs.example.org", "news.example.net", "mail.example.com"], - "path": [ - "", - "", - { "format": "/{seg}", "seg": ["about", "account", "blog", "careers", "cart", "categories", "checkout", "contact", "dashboard", "downloads", "events", "faq", "features", "help", "home", "items", "login", "news", "order", "pricing", "privacy", "products", "profile", "register", "search", "settings", "signup", "support", "terms", "welcome"] }, - { "format": "/{seg}/{sub}", "weight": 0.5, "seg": ["blog", "docs", "help", "products", "category", "user"], "sub": ["getting-started", "overview", "guide", "2024", "latest", "popular", "archive", "details"] } - ] - } -] +{ + "format": "https://{host}{path}", + "host": ["example.com", "www.example.com", "example.org", "www.example.org", "example.net", "example.io", "example.co", "blog.example.com", "blog.example.net", "shop.example.com", "store.example.org", "app.example.io", "api.example.com", "docs.example.org", "news.example.net", "mail.example.com"], + "path": [ + { "format": "", "weight": 2 }, + { + "format": "/{seg}", + "seg": ["about", "account", "blog", "careers", "cart", "categories", "checkout", "contact", "dashboard", "downloads", "events", "faq", "features", "help", "home", "items", "login", "news", "order", "pricing", "privacy", "products", "profile", "register", "search", "settings", "signup", "support", "terms", "welcome"] + }, + { + "format": "/{seg}/{sub}", + "weight": 0.5, + "seg": ["blog", "docs", "help", "products", "category", "user"], + "sub": ["getting-started", "overview", "guide", "2024", "latest", "popular", "archive", "details"] + } + ] +} diff --git a/data/en_US/username.json b/data/en_US/username.json index 9aa8743..5624d8a 100644 --- a/data/en_US/username.json +++ b/data/en_US/username.json @@ -4,21 +4,20 @@ "weight": 2, "first": ["alex", "ash", "avery", "bailey", "blake", "cameron", "casey", "charlie", "chris", "dakota", "drew", "elliot", "emerson", "finley", "frankie", "harley", "hayden", "jamie", "jordan", "kai", "kendall", "lane", "logan", "morgan", "parker", "quinn", "reese", "riley", "rowan", "sam", "sawyer", "skyler", "spencer", "taylor"], "last": ["adams", "brooks", "carter", "cole", "evans", "fisher", "fox", "grant", "gray", "hayes", "hunt", "lane", "lowe", "marsh", "mason", "morris", "north", "page", "quinn", "reed", "rhodes", "shaw", "stone", "vale", "wells", "west", "wolfe"], - "sep": ["", "", "", ".", "_"], - "n": ["", "", "", "", "1", "7", "12", "23", "42", "77", "99", "2024"] + "sep": [{ "format": "", "weight": 3 }, ".", "_"], + "n": [{ "format": "", "weight": 4 }, "1", "7", "12", "23", "42", "77", "99", "2024"] }, { "format": "{adj}{sep}{noun}{n}", "weight": 2, "adj": ["bluish", "brave", "chill", "cosmic", "cool", "dizzy", "electric", "epic", "fluffy", "frosty", "funky", "fuzzy", "golden", "groovy", "grumpy", "happy", "hyper", "jazzy", "lucky", "mega", "mighty", "neon", "nimble", "quirky", "rad", "rusty", "salty", "shiny", "silent", "sleepy", "snappy", "sneaky", "sparkly", "spicy", "sunny", "super", "swift", "turbo", "wild", "witty", "zany"], "noun": ["badger", "biscuit", "cactus", "comet", "donut", "dragon", "falcon", "ferret", "gizmo", "goblin", "hamster", "koala", "llama", "mango", "muffin", "narwhal", "noodle", "otter", "panda", "penguin", "pickle", "pirate", "pixel", "potato", "pretzel", "raccoon", "raptor", "robot", "taco", "toast", "trunk", "turtle", "unicorn", "viking", "waffle", "walrus", "wizard", "yeti"], - "sep": ["", "", "", ".", "_"], - "n": ["", "", "7", "13", "42", "77", "99", "123", "420", "2024"] + "sep": [{ "format": "", "weight": 3 }, ".", "_"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "42", "77", "99", "123", "420", "2024"] }, { "format": "{w}{n}", - "weight": 1, "w": ["blooper", "blupp", "boop", "byteme", "coolcat", "derpina", "doomguy", "epicgamer", "fizzbuzz", "glitchlord", "glorptron", "gronkzilla", "kapowski", "lazerwolf", "megashark", "mooncat", "narfle", "nightowl", "noodlemaster", "oofster", "pixelfox", "pixelpusher", "plonkers", "retrowave", "snork", "splatmaster", "stardust", "superdog", "voidwalker", "vroom", "wahoo", "waooooh", "wootwoot", "yeetus", "zapper", "zoomzoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] } ] diff --git a/data/en_US/version.json b/data/en_US/version.json index 65bc1cc..43be399 100644 --- a/data/en_US/version.json +++ b/data/en_US/version.json @@ -1,8 +1,6 @@ -[ - { - "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "0", "weight": 3 }, { "format": "10" }], - "pre": ["", { "format": "v", "weight": 0.4 }], - "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] - } -] +{ + "format": "{pre}{n}.{n}.{n}{suffix}", + "n": [{ "format": "{digits(1)}", "weight": 3 }, "{int(10,99)}"], + "pre": ["", { "format": "v", "weight": 0.4 }], + "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] +} diff --git a/data/misc/car.json b/data/misc/car.json index 208e638..91886c6 100644 --- a/data/misc/car.json +++ b/data/misc/car.json @@ -1,10 +1,10 @@ [ - { "format": "{maker} {model}", "maker": ["BMW"], "model": ["3 Series", "5 Series", "X3", "X5"] }, - { "format": "{maker} {model}", "maker": ["Ford"], "model": ["Fiesta", "Focus", "Mustang", "Explorer"] }, - { "format": "{maker} {model}", "maker": ["Honda"], "model": ["Civic", "Accord", "CR-V", "Jazz"] }, - { "format": "{maker} {model}", "maker": ["Mercedes-Benz"], "model": ["A-Class", "C-Class", "E-Class", "GLC"] }, - { "format": "{maker} {model}", "maker": ["Tesla"], "model": ["Model 3", "Model S", "Model X", "Model Y"] }, - { "format": "{maker} {model}", "maker": ["Toyota"], "model": ["Corolla", "Camry", "RAV4", "Yaris"] }, - { "format": "{maker} {model}", "maker": ["Volkswagen"], "model": ["Golf", "Passat", "Polo", "Tiguan"] }, - { "format": "{maker} {model}", "maker": ["Volvo"], "model": ["XC40", "XC60", "XC90", "V60"] } + { "format": "{maker} {model}", "maker": "BMW", "model": ["3 Series", "5 Series", "X3", "X5"] }, + { "format": "{maker} {model}", "maker": "Ford", "model": ["Fiesta", "Focus", "Mustang", "Explorer"] }, + { "format": "{maker} {model}", "maker": "Honda", "model": ["Civic", "Accord", "CR-V", "Jazz"] }, + { "format": "{maker} {model}", "maker": "Mercedes-Benz", "model": ["A-Class", "C-Class", "E-Class", "GLC"] }, + { "format": "{maker} {model}", "maker": "Tesla", "model": ["Model 3", "Model S", "Model X", "Model Y"] }, + { "format": "{maker} {model}", "maker": "Toyota", "model": ["Corolla", "Camry", "RAV4", "Yaris"] }, + { "format": "{maker} {model}", "maker": "Volkswagen", "model": ["Golf", "Passat", "Polo", "Tiguan"] }, + { "format": "{maker} {model}", "maker": "Volvo", "model": ["XC40", "XC60", "XC90", "V60"] } ] diff --git a/data/misc/coordinate.json b/data/misc/coordinate.json index b00f222..d2a35a7 100644 --- a/data/misc/coordinate.json +++ b/data/misc/coordinate.json @@ -1,7 +1 @@ -[ - { - "format": "{lat}, {lon}", - "lat": { "format": "{float(-90,90,6)}" }, - "lon": { "format": "{float(-180,180,6)}" } - } -] +{ "format": "{lat}, {lon}", "lat": "{float(-90,90,6)}", "lon": "{float(-180,180,6)}" } diff --git a/data/misc/country.json b/data/misc/country.json index cf6ae99..dc58f22 100644 --- a/data/misc/country.json +++ b/data/misc/country.json @@ -1,22 +1,22 @@ [ - { "format": "{name}", "name": ["Australia"], "alpha2": ["AU"], "alpha3": ["AUS"] }, - { "format": "{name}", "name": ["Brazil"], "alpha2": ["BR"], "alpha3": ["BRA"] }, - { "format": "{name}", "name": ["Canada"], "alpha2": ["CA"], "alpha3": ["CAN"] }, - { "format": "{name}", "name": ["China"], "alpha2": ["CN"], "alpha3": ["CHN"] }, - { "format": "{name}", "name": ["Denmark"], "alpha2": ["DK"], "alpha3": ["DNK"] }, - { "format": "{name}", "name": ["Finland"], "alpha2": ["FI"], "alpha3": ["FIN"] }, - { "format": "{name}", "name": ["France"], "alpha2": ["FR"], "alpha3": ["FRA"] }, - { "format": "{name}", "name": ["Germany"], "alpha2": ["DE"], "alpha3": ["DEU"] }, - { "format": "{name}", "name": ["India"], "alpha2": ["IN"], "alpha3": ["IND"] }, - { "format": "{name}", "name": ["Italy"], "alpha2": ["IT"], "alpha3": ["ITA"] }, - { "format": "{name}", "name": ["Japan"], "alpha2": ["JP"], "alpha3": ["JPN"] }, - { "format": "{name}", "name": ["Mexico"], "alpha2": ["MX"], "alpha3": ["MEX"] }, - { "format": "{name}", "name": ["Netherlands"], "alpha2": ["NL"], "alpha3": ["NLD"] }, - { "format": "{name}", "name": ["Norway"], "alpha2": ["NO"], "alpha3": ["NOR"] }, - { "format": "{name}", "name": ["Poland"], "alpha2": ["PL"], "alpha3": ["POL"] }, - { "format": "{name}", "name": ["Spain"], "alpha2": ["ES"], "alpha3": ["ESP"] }, - { "format": "{name}", "name": ["Sweden"], "alpha2": ["SE"], "alpha3": ["SWE"] }, - { "format": "{name}", "name": ["Switzerland"], "alpha2": ["CH"], "alpha3": ["CHE"] }, - { "format": "{name}", "name": ["United Kingdom"], "alpha2": ["GB"], "alpha3": ["GBR"] }, - { "format": "{name}", "name": ["United States"], "alpha2": ["US"], "alpha3": ["USA"] } + { "format": "{name}", "name": "Australia", "alpha2": "AU", "alpha3": "AUS" }, + { "format": "{name}", "name": "Brazil", "alpha2": "BR", "alpha3": "BRA" }, + { "format": "{name}", "name": "Canada", "alpha2": "CA", "alpha3": "CAN" }, + { "format": "{name}", "name": "China", "alpha2": "CN", "alpha3": "CHN" }, + { "format": "{name}", "name": "Denmark", "alpha2": "DK", "alpha3": "DNK" }, + { "format": "{name}", "name": "Finland", "alpha2": "FI", "alpha3": "FIN" }, + { "format": "{name}", "name": "France", "alpha2": "FR", "alpha3": "FRA" }, + { "format": "{name}", "name": "Germany", "alpha2": "DE", "alpha3": "DEU" }, + { "format": "{name}", "name": "India", "alpha2": "IN", "alpha3": "IND" }, + { "format": "{name}", "name": "Italy", "alpha2": "IT", "alpha3": "ITA" }, + { "format": "{name}", "name": "Japan", "alpha2": "JP", "alpha3": "JPN" }, + { "format": "{name}", "name": "Mexico", "alpha2": "MX", "alpha3": "MEX" }, + { "format": "{name}", "name": "Netherlands", "alpha2": "NL", "alpha3": "NLD" }, + { "format": "{name}", "name": "Norway", "alpha2": "NO", "alpha3": "NOR" }, + { "format": "{name}", "name": "Poland", "alpha2": "PL", "alpha3": "POL" }, + { "format": "{name}", "name": "Spain", "alpha2": "ES", "alpha3": "ESP" }, + { "format": "{name}", "name": "Sweden", "alpha2": "SE", "alpha3": "SWE" }, + { "format": "{name}", "name": "Switzerland", "alpha2": "CH", "alpha3": "CHE" }, + { "format": "{name}", "name": "United Kingdom", "alpha2": "GB", "alpha3": "GBR" }, + { "format": "{name}", "name": "United States", "alpha2": "US", "alpha3": "USA" } ] diff --git a/data/misc/creditcard.json b/data/misc/creditcard.json index a9d7447..0bdce70 100644 --- a/data/misc/creditcard.json +++ b/data/misc/creditcard.json @@ -1,5 +1,18 @@ [ - { "format": "4{d}{luhn()}", "d": { "format": "0", "repeat": 14 }, "weight": 4 }, - { "format": "5{m}{d}{luhn()}", "m": ["1", "2", "3", "4", "5"], "d": { "format": "0", "repeat": 13 }, "weight": 3 }, - { "format": "3{m}{d}{luhn()}", "m": ["4", "7"], "d": { "format": "0", "repeat": 12 }, "weight": 1 } + { + "format": "4{d}{luhn()}", + "d": { "format": "{digits(1)}", "repeat": 14 }, + "weight": 4 + }, + { + "format": "5{m}{d}{luhn()}", + "m": ["1", "2", "3", "4", "5"], + "d": { "format": "{digits(1)}", "repeat": 13 }, + "weight": 3 + }, + { + "format": "3{m}{d}{luhn()}", + "m": ["4", "7"], + "d": { "format": "{digits(1)}", "repeat": 12 } + } ] diff --git a/data/misc/currency.json b/data/misc/currency.json index 7e79b8a..6427331 100644 --- a/data/misc/currency.json +++ b/data/misc/currency.json @@ -1,18 +1,18 @@ [ - { "format": "{code}", "code": ["AUD"], "name": ["Australian Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["BRL"], "name": ["Brazilian Real"], "symbol": ["R$"] }, - { "format": "{code}", "code": ["CAD"], "name": ["Canadian Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["CHF"], "name": ["Swiss Franc"], "symbol": ["CHF"] }, - { "format": "{code}", "code": ["CNY"], "name": ["Chinese Yuan"], "symbol": ["¥"] }, - { "format": "{code}", "code": ["DKK"], "name": ["Danish Krone"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["EUR"], "name": ["Euro"], "symbol": ["€"] }, - { "format": "{code}", "code": ["GBP"], "name": ["Pound Sterling"], "symbol": ["£"] }, - { "format": "{code}", "code": ["INR"], "name": ["Indian Rupee"], "symbol": ["₹"] }, - { "format": "{code}", "code": ["JPY"], "name": ["Japanese Yen"], "symbol": ["¥"] }, - { "format": "{code}", "code": ["MXN"], "name": ["Mexican Peso"], "symbol": ["$"] }, - { "format": "{code}", "code": ["NOK"], "name": ["Norwegian Krone"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["PLN"], "name": ["Polish Zloty"], "symbol": ["zł"] }, - { "format": "{code}", "code": ["SEK"], "name": ["Swedish Krona"], "symbol": ["kr"] }, - { "format": "{code}", "code": ["USD"], "name": ["US Dollar"], "symbol": ["$"] }, - { "format": "{code}", "code": ["ZAR"], "name": ["South African Rand"], "symbol": ["R"] } + { "format": "{code}", "code": "AUD", "name": "Australian Dollar", "symbol": "$" }, + { "format": "{code}", "code": "BRL", "name": "Brazilian Real", "symbol": "R$" }, + { "format": "{code}", "code": "CAD", "name": "Canadian Dollar", "symbol": "$" }, + { "format": "{code}", "code": "CHF", "name": "Swiss Franc", "symbol": "CHF" }, + { "format": "{code}", "code": "CNY", "name": "Chinese Yuan", "symbol": "¥" }, + { "format": "{code}", "code": "DKK", "name": "Danish Krone", "symbol": "kr" }, + { "format": "{code}", "code": "EUR", "name": "Euro", "symbol": "€" }, + { "format": "{code}", "code": "GBP", "name": "Pound Sterling", "symbol": "£" }, + { "format": "{code}", "code": "INR", "name": "Indian Rupee", "symbol": "₹" }, + { "format": "{code}", "code": "JPY", "name": "Japanese Yen", "symbol": "¥" }, + { "format": "{code}", "code": "MXN", "name": "Mexican Peso", "symbol": "$" }, + { "format": "{code}", "code": "NOK", "name": "Norwegian Krone", "symbol": "kr" }, + { "format": "{code}", "code": "PLN", "name": "Polish Zloty", "symbol": "zł" }, + { "format": "{code}", "code": "SEK", "name": "Swedish Krona", "symbol": "kr" }, + { "format": "{code}", "code": "USD", "name": "US Dollar", "symbol": "$" }, + { "format": "{code}", "code": "ZAR", "name": "South African Rand", "symbol": "R" } ] diff --git a/data/misc/emoji.json b/data/misc/emoji.json index 193f29a..6b3d096 100644 --- a/data/misc/emoji.json +++ b/data/misc/emoji.json @@ -1,6 +1 @@ -[ - "😀", "😂", "😍", "🤔", "😎", "😭", "😡", "👍", "👎", "🙏", - "👏", "🙌", "💪", "🔥", "✨", "🎉", "❤️", "💔", "💯", "👀", - "🚀", "⭐", "🌈", "☀️", "🌙", "⚡", "❄️", "🍕", "🍔", "🍺", - "☕", "🎂", "🐶", "🐱", "🦊", "🐢", "🦄", "🌸", "🌍", "💡" -] +["😀", "😂", "😍", "🤔", "😎", "😭", "😡", "👍", "👎", "🙏", "👏", "🙌", "💪", "🔥", "✨", "🎉", "❤️", "💔", "💯", "👀", "🚀", "⭐", "🌈", "☀️", "🌙", "⚡", "❄️", "🍕", "🍔", "🍺", "☕", "🎂", "🐶", "🐱", "🦊", "🐢", "🦄", "🌸", "🌍", "💡"] diff --git a/data/misc/httpstatus.json b/data/misc/httpstatus.json index b27e5ad..0a9af07 100644 --- a/data/misc/httpstatus.json +++ b/data/misc/httpstatus.json @@ -1,18 +1,18 @@ [ - { "format": "{code} {reason}", "code": ["200"], "reason": ["OK"] }, - { "format": "{code} {reason}", "code": ["201"], "reason": ["Created"] }, - { "format": "{code} {reason}", "code": ["204"], "reason": ["No Content"] }, - { "format": "{code} {reason}", "code": ["301"], "reason": ["Moved Permanently"] }, - { "format": "{code} {reason}", "code": ["302"], "reason": ["Found"] }, - { "format": "{code} {reason}", "code": ["304"], "reason": ["Not Modified"] }, - { "format": "{code} {reason}", "code": ["400"], "reason": ["Bad Request"] }, - { "format": "{code} {reason}", "code": ["401"], "reason": ["Unauthorized"] }, - { "format": "{code} {reason}", "code": ["403"], "reason": ["Forbidden"] }, - { "format": "{code} {reason}", "code": ["404"], "reason": ["Not Found"] }, - { "format": "{code} {reason}", "code": ["409"], "reason": ["Conflict"] }, - { "format": "{code} {reason}", "code": ["422"], "reason": ["Unprocessable Entity"] }, - { "format": "{code} {reason}", "code": ["429"], "reason": ["Too Many Requests"] }, - { "format": "{code} {reason}", "code": ["500"], "reason": ["Internal Server Error"] }, - { "format": "{code} {reason}", "code": ["502"], "reason": ["Bad Gateway"] }, - { "format": "{code} {reason}", "code": ["503"], "reason": ["Service Unavailable"] } + { "format": "{code} {reason}", "code": "200", "reason": "OK" }, + { "format": "{code} {reason}", "code": "201", "reason": "Created" }, + { "format": "{code} {reason}", "code": "204", "reason": "No Content" }, + { "format": "{code} {reason}", "code": "301", "reason": "Moved Permanently" }, + { "format": "{code} {reason}", "code": "302", "reason": "Found" }, + { "format": "{code} {reason}", "code": "304", "reason": "Not Modified" }, + { "format": "{code} {reason}", "code": "400", "reason": "Bad Request" }, + { "format": "{code} {reason}", "code": "401", "reason": "Unauthorized" }, + { "format": "{code} {reason}", "code": "403", "reason": "Forbidden" }, + { "format": "{code} {reason}", "code": "404", "reason": "Not Found" }, + { "format": "{code} {reason}", "code": "409", "reason": "Conflict" }, + { "format": "{code} {reason}", "code": "422", "reason": "Unprocessable Entity" }, + { "format": "{code} {reason}", "code": "429", "reason": "Too Many Requests" }, + { "format": "{code} {reason}", "code": "500", "reason": "Internal Server Error" }, + { "format": "{code} {reason}", "code": "502", "reason": "Bad Gateway" }, + { "format": "{code} {reason}", "code": "503", "reason": "Service Unavailable" } ] diff --git a/data/misc/language.json b/data/misc/language.json index 566825b..3e5689c 100644 --- a/data/misc/language.json +++ b/data/misc/language.json @@ -1,18 +1,18 @@ [ - { "format": "{name}", "name": ["Arabic"], "code": ["ar"] }, - { "format": "{name}", "name": ["Chinese"], "code": ["zh"] }, - { "format": "{name}", "name": ["Danish"], "code": ["da"] }, - { "format": "{name}", "name": ["Dutch"], "code": ["nl"] }, - { "format": "{name}", "name": ["English"], "code": ["en"] }, - { "format": "{name}", "name": ["Finnish"], "code": ["fi"] }, - { "format": "{name}", "name": ["French"], "code": ["fr"] }, - { "format": "{name}", "name": ["German"], "code": ["de"] }, - { "format": "{name}", "name": ["Italian"], "code": ["it"] }, - { "format": "{name}", "name": ["Japanese"], "code": ["ja"] }, - { "format": "{name}", "name": ["Norwegian"], "code": ["no"] }, - { "format": "{name}", "name": ["Polish"], "code": ["pl"] }, - { "format": "{name}", "name": ["Portuguese"], "code": ["pt"] }, - { "format": "{name}", "name": ["Russian"], "code": ["ru"] }, - { "format": "{name}", "name": ["Spanish"], "code": ["es"] }, - { "format": "{name}", "name": ["Swedish"], "code": ["sv"] } + { "format": "{name}", "name": "Arabic", "code": "ar" }, + { "format": "{name}", "name": "Chinese", "code": "zh" }, + { "format": "{name}", "name": "Danish", "code": "da" }, + { "format": "{name}", "name": "Dutch", "code": "nl" }, + { "format": "{name}", "name": "English", "code": "en" }, + { "format": "{name}", "name": "Finnish", "code": "fi" }, + { "format": "{name}", "name": "French", "code": "fr" }, + { "format": "{name}", "name": "German", "code": "de" }, + { "format": "{name}", "name": "Italian", "code": "it" }, + { "format": "{name}", "name": "Japanese", "code": "ja" }, + { "format": "{name}", "name": "Norwegian", "code": "no" }, + { "format": "{name}", "name": "Polish", "code": "pl" }, + { "format": "{name}", "name": "Portuguese", "code": "pt" }, + { "format": "{name}", "name": "Russian", "code": "ru" }, + { "format": "{name}", "name": "Spanish", "code": "es" }, + { "format": "{name}", "name": "Swedish", "code": "sv" } ] diff --git a/data/misc/mac.json b/data/misc/mac.json index 4b3d5b1..fe5ba69 100644 --- a/data/misc/mac.json +++ b/data/misc/mac.json @@ -1,3 +1 @@ -[ - { "format": "{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" } -] +"{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" diff --git a/data/misc/mimetype.json b/data/misc/mimetype.json index e7ffda2..dcbe676 100644 --- a/data/misc/mimetype.json +++ b/data/misc/mimetype.json @@ -1,17 +1,17 @@ [ - { "format": "{type}", "type": ["application/gzip"], "ext": [".gz"] }, - { "format": "{type}", "type": ["application/json"], "ext": [".json"] }, - { "format": "{type}", "type": ["application/pdf"], "ext": [".pdf"] }, - { "format": "{type}", "type": ["application/xml"], "ext": [".xml"] }, - { "format": "{type}", "type": ["application/zip"], "ext": [".zip"] }, - { "format": "{type}", "type": ["audio/mpeg"], "ext": [".mp3"] }, - { "format": "{type}", "type": ["image/gif"], "ext": [".gif"] }, - { "format": "{type}", "type": ["image/jpeg"], "ext": [".jpg"] }, - { "format": "{type}", "type": ["image/png"], "ext": [".png"] }, - { "format": "{type}", "type": ["image/svg+xml"], "ext": [".svg"] }, - { "format": "{type}", "type": ["image/webp"], "ext": [".webp"] }, - { "format": "{type}", "type": ["text/csv"], "ext": [".csv"] }, - { "format": "{type}", "type": ["text/html"], "ext": [".html"] }, - { "format": "{type}", "type": ["text/plain"], "ext": [".txt"] }, - { "format": "{type}", "type": ["video/mp4"], "ext": [".mp4"] } + { "format": "{type}", "type": "application/gzip", "ext": ".gz" }, + { "format": "{type}", "type": "application/json", "ext": ".json" }, + { "format": "{type}", "type": "application/pdf", "ext": ".pdf" }, + { "format": "{type}", "type": "application/xml", "ext": ".xml" }, + { "format": "{type}", "type": "application/zip", "ext": ".zip" }, + { "format": "{type}", "type": "audio/mpeg", "ext": ".mp3" }, + { "format": "{type}", "type": "image/gif", "ext": ".gif" }, + { "format": "{type}", "type": "image/jpeg", "ext": ".jpg" }, + { "format": "{type}", "type": "image/png", "ext": ".png" }, + { "format": "{type}", "type": "image/svg+xml", "ext": ".svg" }, + { "format": "{type}", "type": "image/webp", "ext": ".webp" }, + { "format": "{type}", "type": "text/csv", "ext": ".csv" }, + { "format": "{type}", "type": "text/html", "ext": ".html" }, + { "format": "{type}", "type": "text/plain", "ext": ".txt" }, + { "format": "{type}", "type": "video/mp4", "ext": ".mp4" } ] diff --git a/data/misc/objectid.json b/data/misc/objectid.json index 9681734..8174210 100644 --- a/data/misc/objectid.json +++ b/data/misc/objectid.json @@ -1,3 +1 @@ -[ - { "format": "{hex(24)}" } -] +"{hex(24)}" diff --git a/data/misc/timezone.json b/data/misc/timezone.json index 62d6cb5..5c14f62 100644 --- a/data/misc/timezone.json +++ b/data/misc/timezone.json @@ -1,29 +1 @@ -[ - "UTC", - "Africa/Cairo", - "Africa/Johannesburg", - "America/Chicago", - "America/Denver", - "America/Los_Angeles", - "America/Mexico_City", - "America/New_York", - "America/Sao_Paulo", - "America/Toronto", - "Asia/Dubai", - "Asia/Hong_Kong", - "Asia/Kolkata", - "Asia/Shanghai", - "Asia/Singapore", - "Asia/Tokyo", - "Australia/Sydney", - "Europe/Amsterdam", - "Europe/Berlin", - "Europe/London", - "Europe/Madrid", - "Europe/Moscow", - "Europe/Oslo", - "Europe/Paris", - "Europe/Stockholm", - "Europe/Warsaw", - "Pacific/Auckland" -] +["UTC", "Africa/Cairo", "Africa/Johannesburg", "America/Chicago", "America/Denver", "America/Los_Angeles", "America/Mexico_City", "America/New_York", "America/Sao_Paulo", "America/Toronto", "Asia/Dubai", "Asia/Hong_Kong", "Asia/Kolkata", "Asia/Shanghai", "Asia/Singapore", "Asia/Tokyo", "Australia/Sydney", "Europe/Amsterdam", "Europe/Berlin", "Europe/London", "Europe/Madrid", "Europe/Moscow", "Europe/Oslo", "Europe/Paris", "Europe/Stockholm", "Europe/Warsaw", "Pacific/Auckland"] diff --git a/data/misc/useragent.json b/data/misc/useragent.json index f42bf9f..e659a1b 100644 --- a/data/misc/useragent.json +++ b/data/misc/useragent.json @@ -1,10 +1 @@ -[ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0" -] +["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"] diff --git a/data/misc/uuid.json b/data/misc/uuid.json index 18cbf98..4fa8075 100644 --- a/data/misc/uuid.json +++ b/data/misc/uuid.json @@ -1,6 +1 @@ -[ - { - "format": "{hex(8)}-{hex(4)}-4{hex(3)}-{variant}{hex(3)}-{hex(12)}", - "variant": ["8", "9", "a", "b"] - } -] +{ "format": "{hex(8)}-{hex(4)}-4{hex(3)}-{variant}{hex(3)}-{hex(12)}", "variant": ["8", "9", "a", "b"] } diff --git a/data/sv_SE/address.json b/data/sv_SE/address.json index 94de7d1..25e20ab 100644 --- a/data/sv_SE/address.json +++ b/data/sv_SE/address.json @@ -1,25 +1,21 @@ -[ - { - "format": "{street} {street-number}\n{postal-code} {locality}", - "street": [ - { - "format": "{first}{last}", - "first": ["Ängs", "Bergs", "Björk", "Drottning", "Eke", "Furu", "Hamn", "Köpmans", "Kungs", "Linné", "Norra", "Nybro", "Oden", "Öster", "Park", "Skogs", "Skol", "Slotts", "Söder", "Stations", "Stor", "Strand", "Trädgårds", "Vasa", "Väster"], - "last": ["gatan", "vägen", "stigen", "gränd", "backen", "torget", "allén"] - }, - ["Avenyn", "Birger Jarlsgatan", "Promenaden", "Staby", "Sveavägen", "Vintjärn"] - ], - "street-number": [ - { "format": "1" }, - { "format": "10" }, - { "format": "100", "weight": 0.2 }, - { "format": "1A", "weight": 0.2 }, - { "format": "10A", "weight": 0.1 }, - { "format": "100A", "weight": 0.05 } - ], - "postal-code": [ - { "format": "100 10" } - ], - "locality": ["Alingsås", "Alvesta", "Ängelholm", "Arboga", "Arvika", "Avesta", "Boden", "Bollnäs", "Borås", "Borlänge", "Enköping", "Eskilstuna", "Eslöv", "Fagersta", "Falkenberg", "Falköping", "Falun", "Finspång", "Gällivare", "Gävle", "Göteborg", "Halmstad", "Haparanda", "Härnösand", "Hässleholm", "Helsingborg", "Huddinge", "Hudiksvall", "Jönköping", "Kalmar", "Karlshamn", "Karlskoga", "Karlskrona", "Karlstad", "Katrineholm", "Kiruna", "Köping", "Kramfors", "Kristianstad", "Kristinehamn", "Landskrona", "Lidingö", "Lidköping", "Lindesberg", "Linköping", "Ljungby", "Ludvika", "Luleå", "Lund", "Lycksele", "Malmö", "Mariestad", "Mjölby", "Mölndal", "Mora", "Motala", "Nacka", "Nässjö", "Norrköping", "Norrtälje", "Nyköping", "Nynäshamn", "Örebro", "Örnsköldsvik", "Oskarshamn", "Östersund", "Piteå", "Rabbalshede", "Ronneby", "Säffle", "Sandviken", "Sävsjö", "Sigtuna", "Skara", "Skellefteå", "Skövde", "Söderhamn", "Södertälje", "Sollentuna", "Solna", "Sölvesborg", "Stockholm", "Strängnäs", "Sundbyberg", "Sundsvall", "Täby", "Tierp", "Tranås", "Trelleborg", "Trollhättan", "Uddevalla", "Ulricehamn", "Umeå", "Upplands Väsby", "Uppsala", "Vänersborg", "Varberg", "Värnamo", "Västerås", "Västervik", "Växjö", "Vetlanda", "Vimmerby", "Visby", "Ystad"] - } -] +{ + "format": "{street} {street-number}\n{postal-code} {locality}", + "street": [ + { + "format": "{first}{last}", + "first": ["Ängs", "Bergs", "Björk", "Drottning", "Eke", "Furu", "Hamn", "Köpmans", "Kungs", "Linné", "Norra", "Nybro", "Oden", "Öster", "Park", "Skogs", "Skol", "Slotts", "Söder", "Stations", "Stor", "Strand", "Trädgårds", "Vasa", "Väster"], + "last": ["gatan", "vägen", "stigen", "gränd", "backen", "torget", "allén"] + }, + ["Avenyn", "Birger Jarlsgatan", "Promenaden", "Staby", "Sveavägen", "Vintjärn"] + ], + "street-number": [ + "{int(1,9)}", + "{int(10,99)}", + { "format": "{int(100,999)}", "weight": 0.2 }, + { "format": "{int(1,9)}{upper(1)}", "weight": 0.2 }, + { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 }, + { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 } + ], + "postal-code": "{int(100,999)} {int(10,99)}", + "locality": ["Alingsås", "Alvesta", "Ängelholm", "Arboga", "Arvika", "Avesta", "Boden", "Bollnäs", "Borås", "Borlänge", "Enköping", "Eskilstuna", "Eslöv", "Fagersta", "Falkenberg", "Falköping", "Falun", "Finspång", "Gällivare", "Gävle", "Göteborg", "Halmstad", "Haparanda", "Härnösand", "Hässleholm", "Helsingborg", "Huddinge", "Hudiksvall", "Jönköping", "Kalmar", "Karlshamn", "Karlskoga", "Karlskrona", "Karlstad", "Katrineholm", "Kiruna", "Köping", "Kramfors", "Kristianstad", "Kristinehamn", "Landskrona", "Lidingö", "Lidköping", "Lindesberg", "Linköping", "Ljungby", "Ludvika", "Luleå", "Lund", "Lycksele", "Malmö", "Mariestad", "Mjölby", "Mölndal", "Mora", "Motala", "Nacka", "Nässjö", "Norrköping", "Norrtälje", "Nyköping", "Nynäshamn", "Örebro", "Örnsköldsvik", "Oskarshamn", "Östersund", "Piteå", "Rabbalshede", "Ronneby", "Säffle", "Sandviken", "Sävsjö", "Sigtuna", "Skara", "Skellefteå", "Skövde", "Söderhamn", "Södertälje", "Sollentuna", "Solna", "Sölvesborg", "Stockholm", "Strängnäs", "Sundbyberg", "Sundsvall", "Täby", "Tierp", "Tranås", "Trelleborg", "Trollhättan", "Uddevalla", "Ulricehamn", "Umeå", "Upplands Väsby", "Uppsala", "Vänersborg", "Varberg", "Värnamo", "Västerås", "Västervik", "Växjö", "Vetlanda", "Vimmerby", "Visby", "Ystad"] +} diff --git a/data/sv_SE/color.json b/data/sv_SE/color.json index 58cbd03..52f4747 100644 --- a/data/sv_SE/color.json +++ b/data/sv_SE/color.json @@ -1,6 +1,6 @@ [ { - "format": "##{x}{x}{x}{x}{x}{x}", + "format": "#{x}{x}{x}{x}{x}{x}", "x": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], "weight": 2 }, diff --git a/data/sv_SE/company.json b/data/sv_SE/company.json index a79bb01..b40d001 100644 --- a/data/sv_SE/company.json +++ b/data/sv_SE/company.json @@ -1,14 +1,12 @@ -[ - { - "format": "{base} {suffix}", - "base": [ - { - "format": "{a}{b}", - "a": ["Berg", "Blå", "Eko", "Fjäll", "Grön", "Järn", "Nord", "Ny", "Sjö", "Skog", "Sol", "Sten", "Stor", "Syd", "Söder", "Trä", "Vind", "Väst", "Ås", "Öst"], - "b": ["bygg", "data", "design", "energi", "frakt", "gruppen", "handel", "konsult", "kraft", "logistik", "media", "produkter", "system", "teknik", "verkstad", "vård"] - }, - ["Aktiebolaget Vega", "Bröderna Lund", "Ekonomihuset", "Hantverkargruppen", "Lindberg", "Mälardalens Bygg", "Nordström", "Skandinaviska Handelshuset", "Sundbergs Verkstad", "Svensson & Co", "Vasa Logistik"] - ], - "suffix": ["AB", "AB", "AB", "AB", "HB", "KB"] - } -] +{ + "format": "{base} {suffix}", + "base": [ + { + "format": "{a}{b}", + "a": ["Berg", "Blå", "Eko", "Fjäll", "Grön", "Järn", "Nord", "Ny", "Sjö", "Skog", "Sol", "Sten", "Stor", "Syd", "Söder", "Trä", "Vind", "Väst", "Ås", "Öst"], + "b": ["bygg", "data", "design", "energi", "frakt", "gruppen", "handel", "konsult", "kraft", "logistik", "media", "produkter", "system", "teknik", "verkstad", "vård"] + }, + ["Aktiebolaget Vega", "Bröderna Lund", "Ekonomihuset", "Hantverkargruppen", "Lindberg", "Mälardalens Bygg", "Nordström", "Skandinaviska Handelshuset", "Sundbergs Verkstad", "Svensson & Co", "Vasa Logistik"] + ], + "suffix": [{ "format": "AB", "weight": 4 }, "HB", "KB"] +} diff --git a/data/sv_SE/date.json b/data/sv_SE/date.json index f19ce7b..a591bf7 100644 --- a/data/sv_SE/date.json +++ b/data/sv_SE/date.json @@ -1,15 +1,13 @@ -[ - { - "format": "{year}-{month}-{day}", - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], - "year": [ - { "format": "#1970" }, - { "format": "#1980" }, - { "format": "#1990", "weight": 2 }, - { "format": "2#0#00", "weight": 3 }, - { "format": "2#0#10", "weight": 3 }, - { "format": "2#020", "weight": 2 } - ] - } -] +{ + "format": "{year}-{month}-{day}", + "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], + "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"], + "year": [ + "197{digits(1)}", + "198{digits(1)}", + { "format": "199{digits(1)}", "weight": 2 }, + { "format": "200{digits(1)}", "weight": 3 }, + { "format": "201{digits(1)}", "weight": 3 }, + { "format": "202{digits(1)}", "weight": 2 } + ] +} diff --git a/data/sv_SE/email.json b/data/sv_SE/email.json index c1d3fbf..cadf729 100644 --- a/data/sv_SE/email.json +++ b/data/sv_SE/email.json @@ -1,29 +1,26 @@ -[ - { - "format": "{local}@{domain}", - "local": [ - { - "format": "{first}{sep}{last}{n}", - "weight": 3, - "first": ["agnes", "alice", "anders", "anna", "anton", "asa", "axel", "bjorn", "david", "ebba", "elin", "elsa", "emil", "emma", "erik", "ester", "frida", "fredrik", "gustav", "hampus", "hanna", "henrik", "hugo", "ida", "ingrid", "isak", "johan", "jonas", "julia", "karin", "kjell", "klara", "lars", "lena", "linus", "magnus", "maja", "mats", "mattias", "mikael", "moa", "nils", "olle", "oskar", "par", "patrik", "per", "pontus", "rasmus", "samuel", "sara", "sofia", "soren", "stina", "sven", "tobias", "tuva", "viktor", "wilma", "ylva"], - "last": ["ahlberg", "andersson", "berg", "berglund", "bergstrom", "blom", "dahlberg", "eklund", "ekstrom", "engstrom", "eriksson", "falk", "forsberg", "fredriksson", "gustafsson", "hellstrom", "holm", "holmberg", "jakobsson", "johansson", "jonsson", "karlsson", "larsson", "lind", "lindberg", "lindgren", "lindqvist", "lundberg", "lundgren", "lundqvist", "magnusson", "nilsson", "norberg", "nystrom", "olsson", "persson", "petersson", "samuelsson", "sandberg", "sjoberg", "sjogren", "strand", "strom", "sundberg", "svensson", "soderberg", "wallin", "aberg", "oberg"], - "sep": [".", ".", "_", "-", ""], - "n": ["", "", "", "1", "7", "42", "88", "99"] - }, - { - "format": "{adj}{noun}{n}", - "weight": 2, - "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], - "noun": ["abborren", "bamsen", "bjorn", "drake", "ekorre", "falken", "grodan", "hund", "igelkott", "kaninen", "katt", "korpen", "krabban", "musen", "nallen", "ninja", "orn", "panda", "pirat", "raven", "robot", "sillen", "sparven", "uggla", "varg", "vargen"], - "n": ["", "", "7", "13", "14", "42", "77", "99", "123", "420", "2024"] - }, - { - "format": "{w}{n}", - "weight": 1, - "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "doot", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pew", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] - } - ], - "domain": ["example.com", "example.org", "example.net", "example.se", "example.nu", "example.io", "mail.example.se", "webmail.example.se", "inkorg.example.se", "post.example.se", "demo.example.se", "test.example.org", "dev.example.se"] - } -] +{ + "format": "{local}@{domain}", + "local": [ + { + "format": "{first}{sep}{last}{n}", + "weight": 3, + "first": ["agnes", "alice", "anders", "anna", "anton", "asa", "axel", "bjorn", "david", "ebba", "elin", "elsa", "emil", "emma", "erik", "ester", "frida", "fredrik", "gustav", "hampus", "hanna", "henrik", "hugo", "ida", "ingrid", "isak", "johan", "jonas", "julia", "karin", "kjell", "klara", "lars", "lena", "linus", "magnus", "maja", "mats", "mattias", "mikael", "moa", "nils", "olle", "oskar", "par", "patrik", "per", "pontus", "rasmus", "samuel", "sara", "sofia", "soren", "stina", "sven", "tobias", "tuva", "viktor", "wilma", "ylva"], + "last": ["ahlberg", "andersson", "berg", "berglund", "bergstrom", "blom", "dahlberg", "eklund", "ekstrom", "engstrom", "eriksson", "falk", "forsberg", "fredriksson", "gustafsson", "hellstrom", "holm", "holmberg", "jakobsson", "johansson", "jonsson", "karlsson", "larsson", "lind", "lindberg", "lindgren", "lindqvist", "lundberg", "lundgren", "lundqvist", "magnusson", "nilsson", "norberg", "nystrom", "olsson", "persson", "petersson", "samuelsson", "sandberg", "sjoberg", "sjogren", "strand", "strom", "sundberg", "svensson", "soderberg", "wallin", "aberg", "oberg"], + "sep": [{ "format": ".", "weight": 2 }, "_", "-", ""], + "n": [{ "format": "", "weight": 3 }, "1", "7", "42", "88", "99"] + }, + { + "format": "{adj}{noun}{n}", + "weight": 2, + "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], + "noun": ["abborren", "bamsen", "bjorn", "drake", "ekorre", "falken", "grodan", "hund", "igelkott", "kaninen", "katt", "korpen", "krabban", "musen", "nallen", "ninja", "orn", "panda", "pirat", "raven", "robot", "sillen", "sparven", "uggla", "varg", "vargen"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "14", "42", "77", "99", "123", "420", "2024"] + }, + { + "format": "{w}{n}", + "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "doot", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pew", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + } + ], + "domain": ["example.com", "example.org", "example.net", "example.se", "example.nu", "example.io", "mail.example.se", "webmail.example.se", "inkorg.example.se", "post.example.se", "demo.example.se", "test.example.org", "dev.example.se"] +} diff --git a/data/sv_SE/ip.json b/data/sv_SE/ip.json index 2f071a7..c9bf575 100644 --- a/data/sv_SE/ip.json +++ b/data/sv_SE/ip.json @@ -2,9 +2,11 @@ { "format": "{o}.{o}.{o}.{o}", "weight": 8, - "o": [{ "format": "0", "weight": 3 }, { "format": "10", "weight": 4 }, { "format": "#100", "weight": 2 }] + "o": [ + { "format": "{digits(1)}", "weight": 3 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "1{digits(2)}", "weight": 2 } + ] }, - { - "format": "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" - } + "{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}:{hex(4)}" ] diff --git a/data/sv_SE/person.json b/data/sv_SE/person.json index ff0a9ba..2806e36 100644 --- a/data/sv_SE/person.json +++ b/data/sv_SE/person.json @@ -1,41 +1,32 @@ -[ - { - "format": "{prefix}{femalefirst|malefirst} {last}", - "femalefirst": ["Agnes", "Alice", "Alicia", "Alma", "Amanda", "Anna", "Astrid", "Cornelia", "Ebba", "Elin", "Ella", "Ellen", "Elsa", "Emilia", "Emma", "Ester", "Eva", "Frida", "Hanna", "Hedda", "Ida", "Ingrid", "Isabelle", "Julia", "Karin", "Klara", "Kristina", "Lena", "Linnéa", "Lova", "Maja", "Maria", "Moa", "Molly", "Märta", "Nellie", "Nora", "Olivia", "Saga", "Sara", "Selma", "Signe", "Siri", "Sofia", "Stina", "Tilde", "Tuva", "Wilma", "Ylva", "Åsa"], - "malefirst": ["Adam", "Albin", "Alexander", "Alfred", "Anders", "Anton", "Arvid", "Axel", "Bengt", "Bo", "Carl", "David", "Edvin", "Elias", "Emil", "Erik", "Filip", "Folke", "Fredrik", "Gustav", "Göran", "Hampus", "Hans", "Henrik", "Hugo", "Isak", "Johan", "Jonas", "Karl", "Kjell", "Lars", "Leo", "Linus", "Love", "Magnus", "Mats", "Mattias", "Mikael", "Nils", "Olle", "Oskar", "Otto", "Patrik", "Per", "Pontus", "Rasmus", "Samuel", "Sixten", "Stefan", "Sten", "Sven", "Theodor", "Tobias", "Viktor", "William", "Åke"], - "last": [ - { - "format": "{first}sson", - "weight": 5, - "first": ["Ander", "Arvid", "Bengt", "Daniel", "David", "Erik", "Fredrik", "Gustaf", "Henrik", "Håkan", "Isak", "Jakob", "Johan", "Jon", "Jön", "Karl", "Lar", "Magnu", "Matt", "Mikael", "Mårten", "Nil", "Ol", "Per", "Petter", "Samuel", "Sven"] - }, - { - "format": "{first}{last}", - "weight": 3.6, - "first": ["Ahl", "Berg", "Blom", "Ceder", "Dahl", "Ek", "Eng", "Falk", "Fors", "Gran", "Hag", "Hed", "Holm", "Lind", "Lund", "Mal", "Ny", "Rosen", "Sand", "Sjö", "Skog", "Söder", "Sten", "Strand", "Sund", "Wik", "Öst"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] - }, - { - "format": "{first}{last}", - "weight": 0.267, - "first": ["Hell", "Wall"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] - }, - { - "format": "{first}{last}", - "weight": 0.133, - "first": ["Norr"], - "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "stedt", "sten", "strand", "ström", "vall"] - }, - ["Berg", "Blom", "Eismar", "Falk", "Holm", "Lind", "Norberg", "Strand", "Ström", "von Flemming", "Åberg", "Öberg"] - ], - "prefix": [ - "", - { - "format": "{string} ", - "string": ["dr", "prof"], - "weight": 0.05 - } - ] - } -] +{ + "format": "{prefix}{femalefirst|malefirst} {last}", + "femalefirst": ["Agnes", "Alice", "Alicia", "Alma", "Amanda", "Anna", "Astrid", "Cornelia", "Ebba", "Elin", "Ella", "Ellen", "Elsa", "Emilia", "Emma", "Ester", "Eva", "Frida", "Hanna", "Hedda", "Ida", "Ingrid", "Isabelle", "Julia", "Karin", "Klara", "Kristina", "Lena", "Linnéa", "Lova", "Maja", "Maria", "Moa", "Molly", "Märta", "Nellie", "Nora", "Olivia", "Saga", "Sara", "Selma", "Signe", "Siri", "Sofia", "Stina", "Tilde", "Tuva", "Wilma", "Ylva", "Åsa"], + "malefirst": ["Adam", "Albin", "Alexander", "Alfred", "Anders", "Anton", "Arvid", "Axel", "Bengt", "Bo", "Carl", "David", "Edvin", "Elias", "Emil", "Erik", "Filip", "Folke", "Fredrik", "Gustav", "Göran", "Hampus", "Hans", "Henrik", "Hugo", "Isak", "Johan", "Jonas", "Karl", "Kjell", "Lars", "Leo", "Linus", "Love", "Magnus", "Mats", "Mattias", "Mikael", "Nils", "Olle", "Oskar", "Otto", "Patrik", "Per", "Pontus", "Rasmus", "Samuel", "Sixten", "Stefan", "Sten", "Sven", "Theodor", "Tobias", "Viktor", "William", "Åke"], + "last": [ + { + "format": "{first}sson", + "weight": 5, + "first": ["Ander", "Arvid", "Bengt", "Daniel", "David", "Erik", "Fredrik", "Gustaf", "Henrik", "Håkan", "Isak", "Jakob", "Johan", "Jon", "Jön", "Karl", "Lar", "Magnu", "Matt", "Mikael", "Mårten", "Nil", "Ol", "Per", "Petter", "Samuel", "Sven"] + }, + { + "format": "{first}{last}", + "weight": 3.6, + "first": ["Ahl", "Berg", "Blom", "Ceder", "Dahl", "Ek", "Eng", "Falk", "Fors", "Gran", "Hag", "Hed", "Holm", "Lind", "Lund", "Mal", "Ny", "Rosen", "Sand", "Sjö", "Skog", "Söder", "Sten", "Strand", "Sund", "Wik", "Öst"], + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] + }, + { + "format": "{first}{last}", + "weight": 0.267, + "first": ["Hell", "Wall"], + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "man", "mark", "qvist", "roth", "stedt", "sten", "strand", "ström", "vall"] + }, + { + "format": "{first}{last}", + "weight": 0.133, + "first": "Norr", + "last": ["berg", "blad", "crona", "dahl", "ed", "fors", "gren", "holm", "in", "kvist", "löf", "lund", "man", "mark", "qvist", "stedt", "sten", "strand", "ström", "vall"] + }, + ["Berg", "Blom", "Eismar", "Falk", "Holm", "Lind", "Norberg", "Strand", "Ström", "von Flemming", "Åberg", "Öberg"] + ], + "prefix": ["", { "format": "{string} ", "string": ["dr", "prof"], "weight": 0.05 }] +} diff --git a/data/sv_SE/phone.json b/data/sv_SE/phone.json index 69a4c4c..841d442 100644 --- a/data/sv_SE/phone.json +++ b/data/sv_SE/phone.json @@ -3,15 +3,15 @@ "format": "{prefix}-{a} {b} {c}", "weight": 10, "prefix": ["070", "072", "073", "076", "079"], - "a": [{ "format": "000" }], - "b": [{ "format": "00" }], - "c": [{ "format": "00" }] + "a": "{digits(3)}", + "b": "{digits(2)}", + "c": "{digits(2)}" }, { "format": "{prefix}-{a} {b} {c}", "prefix": ["08", "011", "013", "018", "019", "021", "023", "026", "031", "033", "035", "036", "040", "042", "044", "046", "054", "060", "063", "090"], - "a": [{ "format": "000" }], - "b": [{ "format": "00" }], - "c": [{ "format": "00" }] + "a": "{digits(3)}", + "b": "{digits(2)}", + "c": "{digits(2)}" } ] diff --git a/data/sv_SE/price.json b/data/sv_SE/price.json index d82c69e..12e369b 100644 --- a/data/sv_SE/price.json +++ b/data/sv_SE/price.json @@ -1,13 +1,11 @@ -[ - { - "format": "{amt}{ore} kr", - "amt": [ - { "format": "1", "weight": 2 }, - { "format": "10", "weight": 4 }, - { "format": "100", "weight": 3 }, - { "format": "1 000", "weight": 1 }, - { "format": "10 000", "weight": 0.3 } - ], - "ore": ["", { "format": ",{c}", "c": [{ "format": "00" }], "weight": 0.4 }] - } -] +{ + "format": "{amt}{ore} kr", + "amt": [ + { "format": "{int(1,9)}", "weight": 2 }, + { "format": "{int(10,99)}", "weight": 4 }, + { "format": "{int(100,999)}", "weight": 3 }, + "{int(1,9)} {digits(3)}", + { "format": "{int(10,99)} {digits(3)}", "weight": 0.3 } + ], + "ore": ["", { "format": ",{c}", "c": "{digits(2)}", "weight": 0.4 }] +} diff --git a/data/sv_SE/sentence.json b/data/sv_SE/sentence.json index 0cc0414..1b1ede9 100644 --- a/data/sv_SE/sentence.json +++ b/data/sv_SE/sentence.json @@ -7,7 +7,7 @@ "noun": ["arkivet", "berget", "bäcken", "dalen", "eken", "fjället", "floden", "fyren", "fågeln", "gården", "glaciären", "hamnen", "holmen", "klippan", "kusten", "lampan", "lyktan", "motorn", "måsen", "ravinen", "räven", "seglaren", "signalen", "sjön", "skogen", "slätten", "stigen", "stjärnan", "stranden", "tornet", "trädgården", "vandraren", "vinden", "vågen", "ängen"], "verb": ["anar", "bevakar", "bygger", "bär", "döljer", "famnar", "följer", "formar", "gömmer", "hälsar", "korsar", "leder", "lockar", "lyser", "minns", "möter", "når", "skyddar", "speglar", "söker", "tänder", "vaktar", "väcker", "värnar", "återvänder"], "prep": ["bakom", "bland", "bortom", "framför", "genom", "intill", "kring", "längs", "mot", "nära", "ovanför", "runt", "under", "vid", "över"], - "end": [".", ".", ".", "!"] + "end": [{ "format": ".", "weight": 3 }, "!"] }, { "format": "{q} {adj} {noun} {verb} {prep} {noun}?", diff --git a/data/sv_SE/ssn.json b/data/sv_SE/ssn.json index 3dbc854..90ab875 100644 --- a/data/sv_SE/ssn.json +++ b/data/sv_SE/ssn.json @@ -1,25 +1,22 @@ -[ - { - "format": "00{mmdd}-000{luhn()}", - "mmdd": [ - { - "format": "{m}{d}", - "weight": 7, - "m": ["01", "03", "05", "07", "08", "10", "12"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"] - }, - { - "format": "{m}{d}", - "weight": 4, - "m": ["04", "06", "09", "11"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30"] - }, - { - "format": "{m}{d}", - "weight": 1, - "m": ["02"], - "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"] - } - ] - } -] +{ + "format": "{digits(2)}{mmdd}-{digits(3)}{luhn()}", + "mmdd": [ + { + "format": "{m}{d}", + "weight": 7, + "m": ["01", "03", "05", "07", "08", "10", "12"], + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"] + }, + { + "format": "{m}{d}", + "weight": 4, + "m": ["04", "06", "09", "11"], + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30"] + }, + { + "format": "{m}{d}", + "m": "02", + "d": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"] + } + ] +} diff --git a/data/sv_SE/time.json b/data/sv_SE/time.json index 856e733..af61b7d 100644 --- a/data/sv_SE/time.json +++ b/data/sv_SE/time.json @@ -1,8 +1,17 @@ -[ - { - "format": "{hour}:{minute}{sec}", - "hour": [{ "format": "#00", "weight": 4 }, { "format": "#10", "weight": 4 }, { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 }], - "minute": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], - "sec": ["", { "format": ":{s}", "s": [{ "format": "{t}0", "t": ["0", "1", "2", "3", "4", "5"] }], "weight": 0.4 }] - } -] +{ + "format": "{hour}:{minute}{sec}", + "hour": [ + { "format": "0{digits(1)}", "weight": 4 }, + { "format": "1{digits(1)}", "weight": 4 }, + { "format": "2{t}", "t": ["0", "1", "2", "3"], "weight": 2 } + ], + "minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "sec": [ + "", + { + "format": ":{s}", + "s": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] }, + "weight": 0.4 + } + ] +} diff --git a/data/sv_SE/url.json b/data/sv_SE/url.json index 6faca12..131e0b5 100644 --- a/data/sv_SE/url.json +++ b/data/sv_SE/url.json @@ -1,12 +1,17 @@ -[ - { - "format": "https://{host}{path}", - "host": ["example.com", "www.example.com", "example.org", "example.se", "www.example.se", "blogg.example.net", "butik.example.com", "shop.example.se", "api.example.se", "dokument.example.org", "nyheter.example.net", "mail.example.se"], - "path": [ - "", - "", - { "format": "/{seg}", "seg": ["om", "om-oss", "produkter", "blogg", "kontakt", "hjalp", "artiklar", "logga-in", "nyheter", "start", "sok", "kundvagn", "kassa", "villkor", "integritet", "priser", "tjanster", "foretag", "karriar", "support", "konto", "installningar", "registrera"] }, - { "format": "/{seg}/{sub}", "weight": 0.5, "seg": ["blogg", "produkter", "hjalp", "dokument", "kategori"], "sub": ["oversikt", "kom-igang", "guide", "2024", "senaste", "arkiv", "detaljer"] } - ] - } -] +{ + "format": "https://{host}{path}", + "host": ["example.com", "www.example.com", "example.org", "example.se", "www.example.se", "blogg.example.net", "butik.example.com", "shop.example.se", "api.example.se", "dokument.example.org", "nyheter.example.net", "mail.example.se"], + "path": [ + { "format": "", "weight": 2 }, + { + "format": "/{seg}", + "seg": ["om", "om-oss", "produkter", "blogg", "kontakt", "hjalp", "artiklar", "logga-in", "nyheter", "start", "sok", "kundvagn", "kassa", "villkor", "integritet", "priser", "tjanster", "foretag", "karriar", "support", "konto", "installningar", "registrera"] + }, + { + "format": "/{seg}/{sub}", + "weight": 0.5, + "seg": ["blogg", "produkter", "hjalp", "dokument", "kategori"], + "sub": ["oversikt", "kom-igang", "guide", "2024", "senaste", "arkiv", "detaljer"] + } + ] +} diff --git a/data/sv_SE/username.json b/data/sv_SE/username.json index d97df0e..cb8e4ce 100644 --- a/data/sv_SE/username.json +++ b/data/sv_SE/username.json @@ -4,21 +4,20 @@ "weight": 2, "first": ["agnes", "alice", "anders", "anna", "asa", "axel", "bjorn", "ebba", "elin", "emil", "emma", "erik", "frida", "gustav", "hanna", "hugo", "ida", "johan", "jonas", "julia", "karin", "klara", "lars", "lena", "magnus", "maja", "moa", "nils", "oskar", "per", "sara", "sofia", "viktor", "ylva"], "last": ["ahl", "alm", "berg", "bjork", "dahl", "ek", "falk", "gran", "hag", "hed", "holm", "lind", "lund", "mark", "nor", "ohman", "ros", "sand", "sjo", "skog", "sten", "strom", "vik"], - "sep": ["", "", ".", "_"], - "n": ["", "", "", "1", "7", "23", "99", "2024"] + "sep": [{ "format": "", "weight": 2 }, ".", "_"], + "n": [{ "format": "", "weight": 3 }, "1", "7", "23", "99", "2024"] }, { "format": "{adj}{sep}{noun}{n}", "weight": 2, "adj": ["blixt", "busig", "cool", "frisk", "galen", "glad", "kaxig", "kvick", "listig", "lugn", "mega", "modig", "mysig", "pigg", "rolig", "smart", "snabb", "stark", "super", "trygg", "turbo", "varm", "vild"], "noun": ["abborre", "bamse", "bjorn", "drake", "ekorre", "falk", "groda", "hund", "igelkott", "kanin", "katt", "korp", "krabba", "mus", "nalle", "ninja", "orn", "panda", "pirat", "rav", "robot", "sill", "sparv", "uggla", "varg"], - "sep": ["", "", ".", "_"], - "n": ["", "", "7", "13", "42", "77", "99", "123", "420", "2024"] + "sep": [{ "format": "", "weight": 2 }, ".", "_"], + "n": [{ "format": "", "weight": 2 }, "7", "13", "42", "77", "99", "123", "420", "2024"] }, { "format": "{w}{n}", - "weight": 1, "w": ["blip", "blupp", "boop", "busigekorre", "coolkatten", "fizz", "floof", "galnaalgen", "gladgrodan", "glorp", "kapow", "kazoo", "kvickharen", "listigaraven", "megaraven", "mysmoln", "nattugglan", "pigghund", "pixelfox", "plonk", "rolighund", "snabbkatten", "snork", "splat", "superhunden", "turbohund", "vildvargen", "vroom", "wahoo", "waooooh", "woot", "yeet", "zap", "zoom"], - "n": ["", "", "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] + "n": [{ "format": "", "weight": 2 }, "7", "14", "42", "69", "99", "123", "420", "1337", "2000", "9000"] } ] diff --git a/data/sv_SE/version.json b/data/sv_SE/version.json index 65bc1cc..43be399 100644 --- a/data/sv_SE/version.json +++ b/data/sv_SE/version.json @@ -1,8 +1,6 @@ -[ - { - "format": "{pre}{n}.{n}.{n}{suffix}", - "n": [{ "format": "0", "weight": 3 }, { "format": "10" }], - "pre": ["", { "format": "v", "weight": 0.4 }], - "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] - } -] +{ + "format": "{pre}{n}.{n}.{n}{suffix}", + "n": [{ "format": "{digits(1)}", "weight": 3 }, "{int(10,99)}"], + "pre": ["", { "format": "v", "weight": 0.4 }], + "suffix": ["", { "format": "-{tag}.{m}", "tag": ["alpha", "beta", "rc"], "m": ["1", "2", "3"], "weight": 0.3 }] +} diff --git a/data_test.go b/data_test.go index 7de80ec..7e0b6b6 100644 --- a/data_test.go +++ b/data_test.go @@ -264,3 +264,40 @@ func luhnValid(s string) bool { } return sum%10 == 0 } + +// swedishName matches one or more letter-words, optionally space/hyphen joined +// ("Storgatan", "Norra Promenaden", "von Flemming"). Used by the composition +// tests so shipped name lists can grow without re-enumerating them here. +var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`) + +func TestShippedStreetComposition(t *testing.T) { + // street is a choice of composed {first}{last} templates and literal names. + f := newGenerator(t, "data/sv_SE", WithSeed(5)) + for i := 0; i < 300; i++ { + if s := fake(t, f, "address.street"); !swedishName.MatchString(s) { + t.Fatalf("street %q is not a Swedish street name", s) + } + } +} + +func TestShippedLastNameComposition(t *testing.T) { + // last is a choice of patronymic {first}sson templates, compound + // {first}{last} templates and literal surnames. + f := newGenerator(t, "data/sv_SE", WithSeed(6)) + for i := 0; i < 300; i++ { + if s := fake(t, f, "person.last"); !swedishName.MatchString(s) { + t.Fatalf("last name %q is not a Swedish surname", s) + } + } +} + +func TestShippedStreetNumberFormats(t *testing.T) { + // Reachable via a hyphenated path; covers all five weighted number variants. + f := newGenerator(t, "data/sv_SE", WithSeed(8)) + re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`) + for i := 0; i < 300; i++ { + if n := fake(t, f, "address.street-number"); !re.MatchString(n) { + t.Fatalf("street-number %q does not match %s", n, re) + } + } +} diff --git a/edge_test.go b/edge_test.go deleted file mode 100644 index 61f4bb6..0000000 --- a/edge_test.go +++ /dev/null @@ -1,386 +0,0 @@ -package fejkdata - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - "testing" -) - -// --- format string edge cases --- - -func TestEscapeEdgeCases(t *testing.T) { - f := engine(1) - cases := map[string]string{ - `{"format":""}`: "", // empty format - `{"format":"#"}`: "#", // trailing escape is a literal # - `{"format":"##"}`: "#", // escaped hash - `{"format":"#0#1#A#a"}`: "01Aa", // escaped class chars stay literal - `{"format":"#{x#}"}`: "{x}", // escaping braces disables tokens - `{"format":"x}y"}`: "x}y", // an unmatched } is literal (x, y aren't classes) - } - for tmpl, want := range cases { - if got := mustRender(t, f, tmpl); got != want { - t.Errorf("render(%s) = %q, want %q", tmpl, got, want) - } - } -} - -func TestMultibyteFormat(t *testing.T) { - // Scanning is rune-aware: multibyte literals coexist with class chars and - // tokens without corrupting indices. - got := mustRender(t, engine(2), `{"format":"Öster{x}-0å","x":["väg"]}`) - if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { - t.Fatalf("multibyte format = %q", got) - } -} - -func TestAlternationThreeWay(t *testing.T) { - f := engine(4) - seen := map[string]bool{} - for i := 0; i < 200; i++ { - seen[mustRender(t, f, `{"format":"{a|b|c}","a":["A"],"b":["B"],"c":["C"]}`)] = true - } - if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 { - t.Fatalf("3-way alternation produced %v, want A, B and C", seen) - } -} - -func TestNewErrors(t *testing.T) { - // Pointing New at a file (not a directory) fails. - file := filepath.Join(t.TempDir(), "xx_XX") - if err := os.WriteFile(file, []byte("{}"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := New([]string{file}); err == nil { - t.Error("New(file) = nil error, want not-a-directory error") - } - // Invalid JSON in a category file fails. - if _, err := New([]string{writeData(t, map[string]string{"broken": `{ not json`})}); err == nil { - t.Error("New(invalid JSON) = nil error") - } - // An option that cannot take effect, and a category or folder no dot path can - // reach, are mistakes New must name rather than accept and ignore. - rejected := map[string]struct { - files map[string]string - want string - }{ - "separator without repeat": { - map[string]string{"a": `{"format":"{x}","x":["1"],"separator":","}`}, - "has no effect without a repeat above 1", - }, - "separator with an explicit repeat of 1": { - map[string]string{"a": `{"format":"{x}","x":["1"],"repeat":1,"separator":","}`}, - "has no effect without a repeat above 1", - }, - "weight outside a choice": { - map[string]string{"a": `{"format":"x","weight":5}`}, - "weight only skews a choice's items", - }, - "non-numeric weight outside a choice": { - map[string]string{"a": `{"format":"x","weight":"bad"}`}, - "weight only skews a choice's items", - }, - "weight used as a field": { - map[string]string{"a": `{"format":"{name} {weight}kg","name":["Anvil"],"weight":["7"]}`}, - "can never be a field", - }, - "an option name used as a token": { - map[string]string{"a": `{"format":"{weight}"}`}, - `"weight" is an option and can never be a field`, - }, - "category name with a dot": { - map[string]string{"a.b": `["1"]`}, - `category "a.b" contains "."`, - }, - "folder name with a dot": { - map[string]string{"a.b/cat": `["1"]`}, - `/a.b: folder "a.b" contains "."`, - }, - // The token grammar reserves three more characters. A name carrying one - // still resolves by dot path, but no format can name it, so it is rejected - // where it is authored rather than at the token that cannot reach it. - "field name with a pipe": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b|c":["2"]}`}, - `field "b|c" contains "|"`, - }, - "field name with a paren": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b(c":["2"]}`}, - `field "b(c" contains "("`, - }, - "field name with a closing brace": { - map[string]string{"a": `{"format":"{x}","x":["1"],"b}c":["2"]}`}, - `field "b}c" contains "}"`, - }, - "category name with a pipe": { - map[string]string{"a|b": `["1"]`}, - `category "a|b" contains "|"`, - }, - // An empty name is not a path segment, so List never offered it — while a - // bare {}, a trailing dot in Fake("a.") and a {..a.} reference all reached - // it. The engine accepted spellings it would never advertise. - "empty field name": { - map[string]string{"a": `{"format":"[{}]","":"VALUE"}`}, - `field "" is empty`, - }, - "folder name with a paren": { - map[string]string{"a(b/cat": `["1"]`}, - `folder "a(b" contains "("`, - }, - // A repeated arm skews an alternation, which weight is the spelling for. - "repeated alternation arm": { - map[string]string{"a": `{"format":"{x|x}","x":["1"]}`}, - `arm "x" is repeated`, - }, - "repeated arm among others": { - map[string]string{"a": `{"format":"{x|y|x}","x":["1"],"y":["2"]}`}, - `arm "x" is repeated`, - }, - "repeated path arm": { - map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":["1"]}}`}, - `arm "p.v" is repeated`, - }, - // A reference arm is the only kind that reaches the repeat check by passing - // the per-arm checks rather than falling through them. - "repeated reference arm": { - map[string]string{"a": `["x"]`, "b": `{"format":"{..a|..a}"}`}, - `arm "..a" is repeated`, - }, - // An arm that is broken on its own terms is reported as that, not as a - // repeat: the repeat is a consequence of the real mistake. - "repeated arm with no path": { - map[string]string{"a": `{"format":"{..|..}"}`}, - "reference has no path", - }, - // No field can be named "", so the token is told that rather than sent to - // name one — the fix "no field" points at is itself a load error. - "repeated empty arm": { - map[string]string{"a": `{"format":"{|}"}`}, - "a name is never empty", - }, - "bare empty token": { - map[string]string{"a": `{"format":"[{}]","x":["1"]}`}, - "a name is never empty", - }, - } - for name, c := range rejected { - _, err := New([]string{writeData(t, c.files)}) - if err == nil { - t.Errorf("%s: New = nil error, want it rejected at load", name) - continue - } - if !strings.Contains(err.Error(), c.want) { - t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) - } - } - // Data that works today must keep working, and stay reachable. - accepted := map[string]struct { - files map[string]string - path string - want string - }{ - "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":["Anvil"],"Weight":["7"]}`}, "a", "Anvil 7kg"}, - "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":["PDF"]}`}, "a", "PDF"}, - "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":["1"]}`}, "a.x-y", "1"}, - "category named Format": {map[string]string{"Format": `["1"]`}, "Format", "1"}, - "folder named Repeat": {map[string]string{"Repeat/cat": `["1"]`}, "Repeat.cat", "1"}, - "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":["2"]}`}, "a", "2"}, - "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":["1"]}`}, "a", "111"}, - // One name in two separate tokens is two independent draws, not a repeated - // arm; only a repeat within one alternation is rejected. - "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":["1"]}`}, "a", "11"}, - } - for name, c := range accepted { - f, err := New([]string{writeData(t, c.files)}) - if err != nil { - t.Errorf("%s: New = %v, want it accepted", name, err) - continue - } - if got, err := f.Fake(c.path); err != nil || got != c.want { - t.Errorf("%s: Fake(%q) = %q, %v, want %q", name, c.path, got, err, c.want) - } - } -} - -// --- deep path navigation --- - -func TestDeepDottedPath(t *testing.T) { - // A 5-segment path descends through alternating object/array nodes; choices - // on the path are single-variant, so it resolves deterministically. - f := engine(1) - f.categories = map[string]node{ - "deep": compiled(t, `[{"format":"{a}","a":[{"format":"{b}","b":[{"format":"{c}","c":[{"format":"{d}","d":["leaf"]}]}]}]}]`), - } - if got, err := f.Fake("deep.a.b.c.d"); err != nil || got != "leaf" { - t.Fatalf("Fake(deep.a.b.c.d) = %q, %v, want leaf", got, err) - } - // Rendering the whole tree resolves the same chain. - if got, err := f.Fake("deep"); err != nil || got != "leaf" { - t.Fatalf("Fake(deep) = %q, %v, want leaf", got, err) - } -} - -func TestDescendIntoLiteralErrors(t *testing.T) { - f := engine(1) - f.categories = map[string]node{"greeting": compiled(t, `["hej"]`)} - if _, err := f.Fake("greeting.extra"); err == nil { - t.Fatal("Fake(greeting.extra) = nil error, want descend-into-literal error") - } -} - -// TestPathThroughChoice pins the rule that keeps a dotted path from rendering on -// one call and failing on the next: every variant must carry the rest of the path. -func TestPathThroughChoice(t *testing.T) { - dir := writeData(t, map[string]string{ - "every": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"]}]`, - "notall": `[{"format":"{f}","f":["1"]},["plain"]]`, - "some": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"],"extra":["x"]}]`, - }) - f := newGenerator(t, dir, WithSeed(1)) - for i := 0; i < 200; i++ { - if got := fake(t, f, "every.f"); got != "1" && got != "2" { - t.Fatalf("every.f = %q, want 1 or 2", got) - } - } - // A path only some variants carry is reported against what all of them carry. - if _, err := f.Fake("some.extra"); err == nil || !strings.Contains(err.Error(), "all carry [f]") { - t.Errorf("Fake(some.extra) = %v, want it to name what every variant carries", err) - } - var first string - for i := 0; i < 200; i++ { - _, err := f.Fake("notall.f") - if err == nil { - t.Fatal("notall.f = nil error, want the same failure every call") - } - if i == 0 { - first = err.Error() - } else if err.Error() != first { - t.Fatalf("notall.f error varies between calls:\n %s\n %s", first, err.Error()) - } - } -} - -// TestPathKeyIsUnambiguous pins that the one shape which could collide cannot be -// written: a field literally named "a.b" and a field "a" holding "b" would both -// spell "a.b", so a dotted field name is rejected at New and a dot means a path -// wherever it appears. -func TestPathKeyIsUnambiguous(t *testing.T) { - dir := writeData(t, map[string]string{ - "cat": `[{"format":"{a.b}","a.b":["1"]},{"format":"{a}","a":{"format":"{b}","b":["2"]}}]`, - }) - _, err := New([]string{dir}) - if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { - t.Fatalf("New = %v, want the dotted field name rejected", err) - } - // The same data without the dotted key is fine, and the path resolves. - f := newGenerator(t, writeData(t, map[string]string{ - "cat": `{"format":"{a.b}","a":{"format":"{b}","b":["2"]}}`, - }), WithSeed(1)) - if !slices.Contains(f.List(), "cat.a.b") { - t.Error("List() omits cat.a.b, which the data carries") - } - if got := fake(t, f, "cat"); got != "2" { - t.Fatalf("cat = %q, want 2", got) - } -} - -// TestMissingFieldNamesItself keeps the precise diagnosis for the ordinary typo: a -// single-variant choice always picks the same item, so it needs no every-variant -// guard and the error can name the field that is missing. -func TestMissingFieldNamesItself(t *testing.T) { - f := newGenerator(t, "data/sv_SE", WithSeed(1)) - _, err := f.Fake("person.typo") - if err == nil || !strings.Contains(err.Error(), `no field "typo"`) { - t.Errorf("Fake(person.typo) = %v, want it to name the missing field", err) - } -} - -// --- category root shapes --- - -func TestCategoryRootShapes(t *testing.T) { - dir := writeData(t, map[string]string{ - "obj": `{"format":"00"}`, // object root - "lit": `"hello"`, // bare-string root - }) - f := newGenerator(t, dir, WithSeed(1)) - if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { - t.Errorf("object-root category = %q, want two digits", got) - } - if got := fake(t, f, "lit"); got != "hello" { - t.Errorf("string-root category = %q, want hello", got) - } -} - -// --- very long lists --- - -func TestLongStringList(t *testing.T) { - const n = 2000 - names := make([]string, n) - for i := range names { - names[i] = fmt.Sprintf("name-%04d", i) - } - list, err := json.Marshal(names) - if err != nil { - t.Fatal(err) - } - f := newGenerator(t, writeData(t, map[string]string{"name": string(list)}), WithSeed(1)) - - valid := map[string]bool{} - for _, v := range names { - valid[v] = true - } - seen := map[string]bool{} - for i := 0; i < 20000; i++ { - v := fake(t, f, "name") - if !valid[v] { - t.Fatalf("got %q, not in the list", v) - } - seen[v] = true - } - if len(seen) < n*8/10 { - t.Fatalf("only %d/%d distinct values seen; selection looks skewed", len(seen), n) - } -} - -// --- composition against the shipped sv_SE data --- - -// swedishName matches one or more letter-words, optionally space/hyphen joined -// ("Storgatan", "Norra Promenaden", "von Flemming"). Used by the composition -// tests so shipped name lists can grow without re-enumerating them here. -var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`) - -func TestShippedStreetComposition(t *testing.T) { - // street is a choice of composed {first}{last} templates and literal names. - f := newGenerator(t, "data/sv_SE", WithSeed(5)) - for i := 0; i < 300; i++ { - if s := fake(t, f, "address.street"); !swedishName.MatchString(s) { - t.Fatalf("street %q is not a Swedish street name", s) - } - } -} - -func TestShippedLastNameComposition(t *testing.T) { - // last is a choice of patronymic {first}sson templates, compound - // {first}{last} templates and literal surnames. - f := newGenerator(t, "data/sv_SE", WithSeed(6)) - for i := 0; i < 300; i++ { - if s := fake(t, f, "person.last"); !swedishName.MatchString(s) { - t.Fatalf("last name %q is not a Swedish surname", s) - } - } -} - -func TestShippedStreetNumberFormats(t *testing.T) { - // Reachable via a hyphenated path; covers all five weighted number variants. - f := newGenerator(t, "data/sv_SE", WithSeed(8)) - re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`) - for i := 0; i < 300; i++ { - if n := fake(t, f, "address.street-number"); !re.MatchString(n) { - t.Fatalf("street-number %q does not match %s", n, re) - } - } -} diff --git a/fejkdata.go b/fejkdata.go index e9f8319..6cfdc01 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -1,86 +1,131 @@ // Package fejkdata generates fake data from recursive JSON templates. // -// Data lives in JSON on disk, not in Go. Point [New] at one or more data -// directories; folders and files become a dot-path namespace, then generate -// values by path: +// Data lives in JSON, not in Go. A generator starts from the shipped data set and +// layers any directories you add; folders and files become a dot-path namespace, +// then generate values by path: // -// f, _ := fejkdata.New([]string{"./data/sv_SE"}, fejkdata.WithSeed(42)) -// f.Fake("address") // "Storgatan 12\n234 56 Göteborg" -// f.Fake("address.locality") // "Göteborg" +// f, _ := fejkdata.New(fejkdata.WithSeed(42)) +// f.Fake("sv_SE.address") // "Storgatan 12\n234 56 Göteborg" +// f.Fake("sv_SE.address.locality") // "Göteborg" // -// A subdirectory is a namespace segment, so pointing at "./data" instead reaches -// a category as "sv_SE.address". Several directories are merged left to right; -// the last one wins on a name clash, so you can layer custom data over the -// built-ins. The JSON template format is documented in the README. -// -// A [Generator] is not safe for concurrent use; create one per goroutine. +// Several sources merge in order, the last winning a name clash, so custom data +// layers over the built-ins. The JSON template format is documented in the README. package fejkdata import ( crand "crypto/rand" + "embed" "encoding/binary" + "errors" "fmt" + "io/fs" "math/rand/v2" + "os" "sort" + "sync" ) +//go:embed data +var shippedFS embed.FS + +// MaxRepeat caps a repeat, and the renders nested repeats multiply to along any +// path; the CLI's --repeat shares it. +const MaxRepeat = 1 << 20 + +var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions + +// ErrNoData is returned by New when no source is loaded at all. +var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") + // Generator generates fake data from a loaded namespace tree. Create one with [New]. +// It is safe for concurrent use; a seeded sequence is reproducible only when drawn +// from one goroutine. type Generator struct { + mu sync.Mutex rand *session - categories map[string]node // root namespace: name -> compiled node tree + categories map[string]node } // session is one generator's mutable render state: the seeded rng plus the {seq()} -// counters. Scoping it to the Generator means sequences (and randomness) belong to -// that generator and reset when you create a new one. Embedding *rand.Rand makes a -// *session satisfy the rng interface the renderer draws from. +// counters. type session struct { *rand.Rand counters map[string]uint64 } -// next returns the next value (counting from 1) of the named {seq()} counter. func (s *session) next(key string) uint64 { s.counters[key]++ return s.counters[key] } type config struct { - seed uint64 - seeded bool + seed uint64 + seeded bool + shipped bool + sources []dataSource } // Option configures a [Generator]. type Option func(*config) -// WithSeed makes output reproducible: two generators with the same seed and locale +// WithSeed makes output reproducible: two generators with the same seed and data // emit identical sequences. func WithSeed(seed uint64) Option { return func(c *config) { c.seed, c.seeded = seed, true } } -// New builds a generator from one or more data directories (e.g. "./data/sv_SE"). -// Each JSON file becomes a category named after the file (address.json -> -// "address") and each subdirectory a namespace segment; directories are merged -// in order, the last winning a name clash. It errors on a missing directory, -// invalid JSON, or no data found. -func New(paths []string, opts ...Option) (*Generator, error) { - cats, err := loadData(paths) - if err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) +// WithDataPath layers a data directory over what is loaded before it. Repeat it to +// layer several; the last wins a name clash. +func WithDataPath(dir string) Option { + return func(c *config) { + c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir}) } - var c config +} + +// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own. +func WithDataFS(fsys fs.FS) Option { + return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) } +} + +// WithoutShippedData leaves the shipped data set out, so only the sources given +// with [WithDataPath] and [WithDataFS] load. +func WithoutShippedData() Option { + return func(c *config) { c.shipped = false } +} + +// New builds a generator from the shipped data set and the options' sources, merged +// in order with the last winning a name clash. Each JSON file becomes a category +// named after the file (address.json -> "address") and each subdirectory a +// namespace segment. It errors on a missing directory, invalid JSON, invalid data, +// or no data at all. +func New(opts ...Option) (*Generator, error) { + c := config{shipped: true} for _, opt := range opts { opt(&c) } - return &Generator{rand: newRand(c.seed, c.seeded), categories: cats}, nil + var sources []dataSource + if c.shipped { + sources = append(sources, dataSource{fsys: shippedFS, root: "data"}) + } + sources = append(sources, c.sources...) + if len(sources) == 0 { + return nil, fmt.Errorf("fejkdata: %w", ErrNoData) + } + cats, err := loadData(sources) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + rng, err := newRand(c.seed, c.seeded) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + return &Generator{rand: rng, categories: cats}, nil } // List returns the sorted dotted paths Fake can render: every category, the dotted -// fields within a template, and folder segments — descending transparently through -// single-variant choices the way a reference does. A choice consumes no segment, so -// a path continues through a multi-variant one only where every variant carries it, -// which is the rule Fake applies too: List is the set of paths Fake accepts. +// fields within a template, and folder segments. A choice consumes no segment, so a +// path continues through one only where every variant carries it, which is the +// rule Fake applies too: List is the set of paths Fake accepts. func (f *Generator) List() []string { var out []string for _, name := range sortedNames(f.categories) { @@ -116,22 +161,17 @@ func paths(n node) []string { } return out case *choice: - if len(n.items) == 1 { - return paths(n.items[0]) - } out := []string{""} for p := range n.shared { out = append(out, p) } return out - case literal: - return []string{""} } return nil } // sharedPaths is the sub-paths every item carries — the only ones a path may step -// through a multi-variant choice to reach. It intersects, bailing as soon as the set +// through a choice to reach. It intersects, bailing as soon as the set // is empty, which is immediate for a choice of plain strings. func sharedPaths(items []node) map[string]bool { shared := subPaths(items[0]) @@ -170,12 +210,19 @@ func join(prefix, name string) string { return prefix + "." + name } -func newRand(seed uint64, seeded bool) *session { - r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) - if !seeded { +// randomBytes seeds an unseeded generator. +var randomBytes = crand.Read + +func newRand(seed uint64, seeded bool) (*session, error) { + var r *rand.Rand + if seeded { + r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) + } else { var b [16]byte - _, _ = crand.Read(b[:]) + if _, err := randomBytes(b[:]); err != nil { + return nil, fmt.Errorf("seeding from crypto/rand: %w", err) + } r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:]))) } - return &session{Rand: r, counters: map[string]uint64{}} + return &session{Rand: r, counters: map[string]uint64{}}, nil } diff --git a/fejkdata_test.go b/fejkdata_test.go index ff5f564..51e8e24 100644 --- a/fejkdata_test.go +++ b/fejkdata_test.go @@ -1,8 +1,11 @@ package fejkdata import ( + "errors" + "io/fs" "strings" "testing" + "testing/fstest" ) // newGenerator creates a generator over a single data directory, failing on @@ -15,7 +18,11 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator { func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator { t.Helper() - f, err := New(dirs, opts...) + all := []Option{WithoutShippedData()} + for _, dir := range dirs { + all = append(all, WithDataPath(dir)) + } + f, err := New(append(all, opts...)...) if err != nil { t.Fatalf("New(%q): %v", dirs, err) } @@ -33,9 +40,26 @@ func fake(t *testing.T, f *Generator, path string) string { } func TestNewMissingDirectory(t *testing.T) { - _, err := New([]string{"data/de_DE"}) - if err == nil || !strings.Contains(err.Error(), "de_DE") { - t.Fatalf("New(missing) error = %v, want it to name the path", err) + _, err := New(WithoutShippedData(), WithDataPath("data/de_DE")) + if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("New(missing) error = %v, want the real error, naming the path", err) + } +} + +func TestNewRejectsAnEmptyDataPath(t *testing.T) { + _, err := New(WithoutShippedData(), WithDataPath("")) + if err == nil || !strings.Contains(err.Error(), "empty") { + t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err) + } +} + +func TestNewReportsAnEntropyFailure(t *testing.T) { + saved := randomBytes + randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") } + defer func() { randomBytes = saved }() + _, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}})) + if err == nil || !strings.Contains(err.Error(), "no entropy") { + t.Fatalf("New() without entropy = %v, want the failure reported", err) } } diff --git a/graph.go b/graph.go new file mode 100644 index 0000000..887c0ec --- /dev/null +++ b/graph.go @@ -0,0 +1,222 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// walkNodes calls fn once per contained node, passing the dot path that reaches it, +// visiting keys in sorted order so which of several broken nodes gets reported does +// not depend on map iteration. +func walkNodes(root map[string]node, fn func(path string, n node) error) error { + seen := map[node]bool{} + var visit func(string, node) error + visit = func(path string, n node) error { + if n == nil || seen[n] { + return nil + } + seen[n] = true + if err := fn(path, n); err != nil { + return err + } + for _, c := range contained(n) { + if err := visit(join(path, c.name), c.node); err != nil { + return err + } + } + return nil + } + for _, name := range sortedNames(root) { + if err := visit(name, root[name]); err != nil { + return err + } + } + return nil +} + +// namedNode is a contained child and the segment reaching it; a choice's items carry +// no segment, matching how a dot path steps over a choice. +type namedNode struct { + name string + node node +} + +func contained(n node) []namedNode { + switch n := n.(type) { + case *group: + return named(n.children) + case *choice: + out := make([]namedNode, len(n.items)) + for i, it := range n.items { + out[i] = namedNode{node: it} + } + return out + case *template: + return named(n.fields) + default: + return nil + } +} + +// named skips a bound {/path} key: it is a render edge, not containment, so using +// it as a path segment would report a node under a path that does not reach it. Only +// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a +// group's children never carry the prefix — so this one skip serves both. +func named(m map[string]node) []namedNode { + out := make([]namedNode, 0, len(m)) + for _, name := range sortedNames(m) { + if isRef(name) { + continue + } + out = append(out, namedNode{name: name, node: m[name]}) + } + return out +} + +func sortedNames(m map[string]node) []string { + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// renderEdge is a child a node renders into, labelled by what reaches it (a field +// name, reference, or choice index) for a readable cycle report. operand names +// the builtin when the label is its operand rather than a token, so an error can +// name it the way the author wrote it. +type renderEdge struct { + to node + label string + operand string +} + +// reached names an edge as the author spelled it, the vocabulary boundReaders uses +// for the sibling fence. +func (e renderEdge) reached() string { + if e.operand != "" { + return fmt.Sprintf("%s operand %q", e.operand, e.label) + } + return "{" + e.label + "}" +} + +// renderEdges lists the children rendering n recurses into, mirroring expand: a +// choice's items, and a template's field/reference tokens plus its operands. A +// group renders nothing, so it has no edges. +func renderEdges(n node) []renderEdge { + switch n := n.(type) { + case *choice: + es := make([]renderEdge, len(n.items)) + for i, it := range n.items { + es[i] = renderEdge{to: it, label: fmt.Sprintf("[%d]", i)} + } + return es + case *template: + var es []renderEdge + add := func(name, operand string) { + a := splitArm(name, n.refs) + c, ok := n.fields[a.key] + if !ok { + return + } + for _, leaf := range pathLeaves(c, a.tail) { + es = append(es, renderEdge{leaf, name, operand}) + } + } + _ = eachToken(n.format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + add(operand, fn) + } + return nil + } + for _, name := range strings.Split(t.body, "|") { + add(name, "") + } + return nil + }) + return es + default: + return nil + } +} + +// pathLeaves lists what a token's dotted tail renders: a choice on the way +// contributes every variant, since any of them may be the one drawn. checkPath has +// already proved the tail resolves in every variant. +func pathLeaves(n node, tail []string) []node { + var out []node + _ = walkPath(n, tail, pathWalk{ + choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil }, + leaf: func(n node) error { out = append(out, n); return nil }, + }) + return out +} + +// checkRepeatReach bounds the renders a repeat multiplies to along any root-to-leaf +// path, so nested repeats cannot build what one repeat may not. It runs after +// checkNoCycles, whose guarantee is what lets the walk terminate. +func checkRepeatReach(root map[string]node) error { + reach := map[node]int{} + var of func(n node) int + of = func(n node) int { + if r, done := reach[n]; done { + return r + } + r := 1 + for _, e := range renderEdges(n) { + if c := of(e.to); c > r { + r = c + } + } + if t, ok := n.(*template); ok { + r *= t.repeat + } + reach[n] = r + return r + } + return walkNodes(root, func(path string, n node) error { + if t, ok := n.(*template); ok && t.repeat > 1 && of(n) > MaxRepeat { + return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, of(n), MaxRepeat) + } + return nil + }) +} + +// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself +// — directly, mutually, or through a chain — never terminates, so it must fail at +// New rather than stack-overflow at render. It is a depth-first walk of the render +// graph (renderEdges); grey marks nodes on the current path so a back-edge to one +// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped. +// Every node is a root: a field its parent's format never renders is still reachable +// by dot path, so a cycle in one would otherwise reach render and be fatal there. +func checkNoCycles(root map[string]node) error { + const ( + grey = 1 + black = 2 + ) + color := map[node]int{} + var visit func(n node, path string) error + visit = func(n node, path string) error { + switch color[n] { + case grey: + return fmt.Errorf("reference cycle: %s", path) + case black: + return nil + } + color[n] = grey + for _, e := range renderEdges(n) { + if err := visit(e.to, path+" -> "+e.label); err != nil { + return err + } + } + color[n] = black + return nil + } + return walkNodes(root, func(path string, n node) error { return visit(n, path) }) +} diff --git a/hold.go b/hold.go new file mode 100644 index 0000000..bb5c3fe --- /dev/null +++ b/hold.go @@ -0,0 +1,314 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// checkBoundLevelsHeld rejects every route to a held name except the ones that read +// its draw. An expansion holds one draw of that name; anything else that renders it +// draws again, and the two disagree. checkNoOverlap settles the spellings within one +// format (a token, an operand); this settles the rest — a reference, whether it +// sits in that format or in anything the format renders, however deep. +// +// It runs after checkNoCycles, whose guarantee is what lets the walk terminate. +func checkBoundLevelsHeld(root map[string]node) error { + return walkNodes(root, func(path string, n node) error { + t, ok := n.(*template) + if !ok || len(t.held) == 0 { + return nil + } + readers := boundReaders(t.format, t.bound, t.refs) + for _, head := range heldHeads(t) { + if err := checkHeadHeld(t, head, readers); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + return nil + }) +} + +// heldHeads lists a template's held names, operand heads first, then paths, each +// in name order: a level read both ways is reported by the operand's fence, and +// which overlap is reported does not vary. +func heldHeads(t *template) []string { + heads := make([]string, 0, len(t.held)) + for head := range t.held { + heads = append(heads, head) + } + sort.Slice(heads, func(i, j int) bool { + _, pi := t.bound[heads[i]] + _, pj := t.bound[heads[j]] + if pi != pj { + return !pi + } + return heads[i] < heads[j] + }) + return heads +} + +// pinned collects what the hold of head answers for: a path pins the levels it +// passes through and the leaf it lands on, an operand exactly the value its render +// produces. +func pinned(t *template, head string, readers []reader) map[node]bool { + held := map[node]bool{} + if _, isPath := t.bound[head]; !isPath { + operandDraw(t.fields[head], held) + return held + } + for _, r := range readers { + if a := splitArm(r.name, t.refs); a.key == head { + coverPath(t.fields[head], a.tail, held) + } + } + return held +} + +// checkHeadHeld rejects every route to what head's hold pins except the readers +// holding it. One seen set across the edges: a node that cannot reach the level +// cannot reach it by another route either, so it is walked once. +func checkHeadHeld(t *template, head string, readers []reader) error { + held := pinned(t, head, readers) + if len(held) == 0 { + return nil // a fixed head holds nothing to reach + } + reader, isPath := t.bound[head] + seen := map[node]bool{} + for _, e := range renderEdges(t) { + if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) { + continue + } + if isPath { + return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader) + } + return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head)) + } + return nil +} + +// operandReader names the builtin whose operand holds head. +func operandReader(t *template, head string) string { + fn := "" + _ = eachToken(t.format, func(tok ftoken) error { + if tok.kind != 'b' || fn != "" { + return nil + } + if name, _, isFunc := funcCall(tok.body); isFunc { + for _, operand := range tokenOperands(tok.body) { + if splitArm(operand, t.refs).key == head { + fn = name + } + } + } + return nil + }) + return fn +} + +// coverPath collects what holding one path pins: every choice level the path +// passes through, whole, and the leaf it renders. +func coverPath(n node, tail []string, into map[node]bool) { + _ = walkPath(n, tail, pathWalk{ + choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil }, + leaf: func(n node) error { cover(n, into, false); return nil }, + }) +} + +// cover collects a level and everything contained in it. A fixed string outside a +// choice is left out — it cannot disagree with itself — but inside one each +// variant carries its own, so there it counts. +func cover(n node, into map[node]bool, inChoice bool) { + if isFixed(n) && !inChoice { + return + } + into[n] = true + _, isChoice := n.(*choice) + for _, c := range contained(n) { + cover(c.node, into, inChoice || isChoice) + } +} + +// operandDraw collects what one held draw of an operand answers for: the operand +// and what rendering it settles inside itself. The builtin renders its operand +// whole, so that draw fixes every value the render produced, and a second route to +// any of them disagrees with it. +// +// The walk stops at a reference edge, which is where the operand's own value ends +// and a shared source begins: two names referencing one category are two draws, the +// same rule {word} {word} follows. +func operandDraw(n node, into map[node]bool) { + if isFixed(n) { + return + } + if into[n] { + return + } + into[n] = true + for _, e := range renderEdges(n) { + if isRef(e.label) { + continue + } + operandDraw(e.to, into) + } +} + +// isFixed is a string that varies nothing: fixed text with no fields to read into. +func isFixed(n node) bool { + t, ok := n.(*template) + return ok && t.fixed && len(t.fields) == 0 +} + +// renders reports whether rendering n can reach anything in want, following the +// same edges expand does. seen keeps a node shared by several routes from being +// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk +// ends. +func renders(n node, want, seen map[node]bool) bool { + if want[n] { + return true + } + if seen[n] { + return false + } + seen[n] = true + for _, e := range renderEdges(n) { + if renders(e.to, want, seen) { + return true + } + } + return false +} + +// checkNoOverlap rejects a format that both renders a level and reads a path into +// it — {p} beside {p.first}, {p.addr} beside {p.addr.city}, {.p} beside +// {/sv_SE.p.first}. The path reads the level's held draw while rendering the level +// expands it afresh, so their values would disagree. Reads are compared by their +// one spelling, in sorted order, so which pair is reported depends neither on how +// a reference was written nor on where the tokens sit. +func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error { + names := boundReaders(format, bound, refs) + // Stable over one format-order scan, so two readers of one name (a token and a + // calc operand both naming "p") are reported as the format writes them. + sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path }) + for i, level := range names { + for _, path := range names[i+1:] { + if strings.HasPrefix(path.path, level.path+".") { + return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name) + } + } + } + return nil +} + +// reader is one way a format reaches a bound field: as written, by its one +// spelling, and how to name it. +type reader struct{ name, path, label string } + +// boundReaders lists every way a format reaches a bound field, in the order the +// format writes them. An operand renders its field, so it names a level exactly +// as a token does; one scan finds both, which is what puts them in one order. +func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader { + var names []reader + _ = eachToken(format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if fn, _, isFunc := funcCall(t.body); isFunc { + for _, operand := range tokenOperands(t.body) { + a := splitArm(operand, refs) + if _, isBound := bound[a.key]; isBound { + names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)}) + } + } + return nil + } + for _, a := range splitArms(t.body, refs) { + if _, isBound := bound[a.key]; isBound { + names = append(names, reader{a.name, a.path, "token {" + a.name + "}"}) + } + } + return nil + }) + return names +} + +// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside +// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The +// error names the single-token spelling. +func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error { + count := map[string]int{} + return eachToken(format, func(t ftoken) error { + if t.kind != 'b' { + return nil + } + if _, _, isFunc := funcCall(t.body); isFunc { + return nil + } + for _, a := range splitArms(t.body, refs) { + if len(a.tail) > 0 || !c.held[a.key] { + continue + } + if count[a.key]++; count[a.key] > 1 { + return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name) + } + } + return nil + }) +} + +// draws is what an expansion has already drawn for its held names: the variant each +// was drawn as, so every path under it reads one row, and the value each read, by +// its one spelling, so the same read written twice reads one value. +type draws struct { + variant map[string]node + value map[string]string +} + +// readField renders one arm of a token. An arm's key is a sibling field or a +// reference linkRefs bound into fields. A name the expansion holds — a level some +// token addresses by dotted path, or a field an operand reads — is drawn once and +// kept, so {place.postal-code} and {place.locality} read one row, either read twice +// gives one value, and a shown operand is the operand computed. Every other name is +// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and +// linkRefs prove every step, so the walk cannot fail. +func readField(s *session, t *template, held *draws, a arm) string { + if !t.held[a.key] { + if len(a.tail) > 0 { + panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key)) + } + return render(s, t.fields[a.key]) + } + if v, read := held.value[a.path]; read { + return v + } + var v string + _ = walkPath(t.fields[a.key], a.tail, pathWalk{ + // Hold the draw at every level passed through, so two paths sharing a + // prefix share it. + choice: func(c *choice, rest []string) ([]node, error) { + key := a.key + if consumed := len(a.tail) - len(rest); consumed > 0 { + key = a.steps[consumed-1] + } + n, drew := held.variant[key] + if !drew { + n = drawn(s, c) + held.variant[key] = n + } + return []node{n}, nil + }, + leaf: func(n node) error { v = render(s, n); return nil }, + }) + held.value[a.path] = v + return v +} + +// drawn resolves a choice to one variant, so a bound head is a concrete node the +// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not +// another set to pick from. +func drawn(s *session, n node) node { + for c, ok := n.(*choice); ok; c, ok = n.(*choice) { + n = pick(s, c) + } + return n +} diff --git a/bound_test.go b/hold_test.go similarity index 74% rename from bound_test.go rename to hold_test.go index b6eeac0..3827d55 100644 --- a/bound_test.go +++ b/hold_test.go @@ -15,10 +15,7 @@ import ( // swedishPlaces is a two-variant sibling whose variants pair a locality with the // postal-code prefix that really belongs to it. -const swedishPlaces = `{"format":"%s","place":[ - {"format":"{locality}","locality":"Stockholm","postal-code":{"format":"#100 00"}}, - {"format":"{locality}","locality":"Tranås","postal-code":{"format":"#5#7#3 00"}} -]}` +const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}` // agree reports whether a rendered "postcode locality" pair is a real pairing. var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`) @@ -48,13 +45,13 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { // other rendered — so the pair is a load error rather than a shape whose two // halves can disagree. rejected := map[string]string{ - "bare head beside a path": `{"format":"{p} + {p.first}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, - "prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":[{"format":"{addr}","addr":[` + - `{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}]}`, - "either order": `{"format":"{p.first} + {p}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + "bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + "prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` + + `{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`, + "either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil { t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name) continue @@ -64,9 +61,9 @@ func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) { } } // The same path twice is one spelling, so it stays legal. - if _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{p.first} + {p.first}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, - })}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + }))); err != nil { t.Errorf("New = %v, want one path read twice accepted", err) } } @@ -75,17 +72,17 @@ func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) { // A calc operand renders its field, so naming a bound level in one is the same // overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a // path into, and the two disagree. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` + `{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Fatalf("New = %v, want the calc operand rejected as an overlap", err) } // Arithmetic over fields no path names is untouched. - if _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`, - })}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`, + }))); err != nil { t.Errorf("New = %v, want plain arithmetic accepted", err) } } @@ -95,25 +92,25 @@ func TestReferenceNamingABoundLevelIsRejected(t *testing.T) { // afresh beside the path that reads its held draw — the same overlap by // another spelling. rejected := map[string]string{ - "reference names the head": `{"format":"{p.first}|{..cat.p}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, - "reference names the leaf": `{"format":"{p.addr}|{..cat.p.addr}","p":{"format":"x","addr":["A","B","C","D"]}}`, + "reference names the head": `{"format":"{p.first}|{/cat.p}","p":{"format":"{first}","first":["Anna","Bo"]}}`, + "reference names the leaf": `{"format":"{p.addr}|{/cat.p.addr}","p":{"format":"x","addr":["A","B","C","D"]}}`, // The reference need not sit in the format that binds: any field it renders // reaches the level just the same, however deep. "reference from a sibling field": `{"format":"{p.first}|{inner}","p":[` + `{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` + - `"inner":{"format":"{..cat.p}"}}`, + `"inner":"{/cat.p}"}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err) } } // A reference to anything this format does not bind is untouched. - if _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{p.first} {..surname}","p":[{"format":"{first}","first":["Anna","Bo"]}]}`, + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`, "surname": `["Eriksson","Lindqvist"]`, - })}); err != nil { + }))); err != nil { t.Errorf("New = %v, want a reference outside the bound level accepted", err) } } @@ -122,9 +119,9 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { // A path token renders what it lands on, not the level it started from, so the // cycle walk has to follow it there. A head whose own format names nothing // would otherwise hide the cycle until render, where it is fatal. - _, err := New([]string{writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"static","x":{"format":"{..a}"}}}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "a": `{"format":"{p.x}","p":{"format":"static","x":"{/a}"}}`, + }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) } @@ -133,10 +130,10 @@ func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) { func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) { // {p.a} renders q, so it is a route to the level {q.x} holds — even though p's // own format names nothing. - _, err := New([]string{writeData(t, map[string]string{ - "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":{"format":"{..thing.q}"}},` + + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/thing.q}"},` + `"q":{"format":"{x}","x":["1","2"]}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), "reads a path into") { t.Fatalf("New = %v, want the second route to q rejected", err) } @@ -154,14 +151,14 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { }{ "a reference beside the operand": { map[string]string{ - "cat": `{"format":"{..cat.net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`, + "cat": `{"format":"{/cat.net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`, }, - `{..cat.net} renders "net"`, + `{/cat.net} renders "net"`, }, "a reference one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` + - `"q":{"format":"{..cat.net}"}}`, + `"q":"{/cat.net}"}`, }, `{q} renders "net"`, }, @@ -169,8 +166,8 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { // so wrapping the operand in one changes nothing. "a reference to an operand wrapped in a choice": { map[string]string{ - "cat": `{"format":"{calc(n * 2, 2)} {q}","n":[{"format":"{v}","v":["1","2"]}],` + - `"q":{"format":"{..cat.n}"}}`, + "cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` + + `"q":"{/cat.n}"}`, }, `{q} renders "n"`, }, @@ -179,7 +176,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "an operand reaching another operand": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` + - `"b":{"format":"{..cat.a}"}}`, + `"b":"{/cat.a}"}`, }, `calc operand "b" renders "a"`, }, @@ -188,15 +185,15 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { // rejected, so the reference spelling has to be. "a reference into the operand": { map[string]string{ - "cat": `{"format":"{calc(net * 2, 2)}|{..cat.net.v}",` + + "cat": `{"format":"{calc(net * 2, 2)}|{/cat.net.v}",` + `"net":{"format":"{v}","v":["10.00","20.00"]}}`, }, - `{..cat.net.v} renders "net"`, + `{/cat.net.v} renders "net"`, }, "a reference into the operand one level down": { map[string]string{ "cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` + - `"q":{"format":"{..cat.net.v}"}}`, + `"q":"{/cat.net.v}"}`, }, `{q} renders "net"`, }, @@ -206,13 +203,13 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a violation on the second of two held heads": { map[string]string{ "cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` + - `"b":{"format":"{y}","y":["3","4"]},"w":{"format":"{..cat.b}"}}`, + `"b":{"format":"{y}","y":["3","4"]},"w":"{/cat.b}"}`, }, `{w} renders "b"`, }, } for name, c := range rejected { - _, err := New([]string{writeData(t, c.files)}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") { t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err) continue @@ -233,24 +230,24 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { "a reference to a sibling the operand never renders": { "cat": `{"format":"{calc(net * 2, 2)} {unit}",` + `"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` + - `"unit":{"format":"{..cat.net.spare}"}}`, + `"unit":"{/cat.net.spare}"}`, }, // Two operands drawing from one source are two names, so two draws: each is // held under its own name and shown once, and neither can disagree. "two operands sharing one source": { "die": `["1","2","3","4","5","6"]`, - "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":{"format":"{..die}"},` + - `"d2":{"format":"{..die}"}}`, + "cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` + + `"d2":"{/die}"}`, }, - // Likewise a reference drawing from what the operand draws from: {..common} + // Likewise a reference drawing from what the operand draws from: {/common} // and net are two names, not two spellings of one field. "a reference to what an operand renders through": { "common": `["1","2"]`, - "cat": `{"format":"{calc(net * 2, 2)} {..common}","net":{"format":"{..common}"}}`, + "cat": `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`, }, // A fixed string cannot disagree with itself, so it needs no fence. "a literal operand named twice": { - "cat": `{"format":"{calc(n * 2, 0)} {..cat.n}","n":"5"}`, + "cat": `{"format":"{calc(n * 2, 0)} {/cat.n}","n":"5"}`, }, // One node reached twice while walking the operand: the walk must not // revisit it, and the repeat is not a second route to anything. @@ -259,7 +256,7 @@ func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) { }, } for name, files := range accepted { - if _, err := New([]string{writeData(t, files)}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { t.Errorf("%s: New = %v, want it accepted", name, err) } } @@ -272,17 +269,17 @@ func TestAPathReachesEveryVariantItMightDraw(t *testing.T) { // to a held level disagrees silently. rejected := map[string]struct{ file, want string }{ "a cycle in a later variant": { - `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":{"format":"{..cat}"}}]}`, + `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/cat}"}]}`, "reference cycle", }, "a second route in a later variant": { - `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":{"format":"{..cat.q}"}}],` + + `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/cat.q}"}],` + `"q":{"format":"{y}","y":["1","2"]}}`, "reads a path into", }, } for name, c := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": c.file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file}))) if err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) } @@ -295,12 +292,12 @@ func TestALevelAPathNeverRendersIsAccepted(t *testing.T) { // path at all. Both orders must load. accepted := map[string]string{ "reference in the head's own format": `{"format":"{p.first} {q.a}",` + - `"p":{"format":"{first} {..thing.q}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}`, + `"p":{"format":"{first} {/thing.q}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}`, "the mirror shape": `{"format":"{p.first} {q.a}",` + - `"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {..thing.p}","a":["1","2"]}}`, + `"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/thing.p}","a":["1","2"]}}`, } for name, file := range accepted { - if _, err := New([]string{writeData(t, map[string]string{"thing": file})}); err != nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"thing": file}))); err != nil { t.Errorf("%s: New = %v, want it accepted", name, err) } } @@ -309,13 +306,13 @@ func TestALevelAPathNeverRendersIsAccepted(t *testing.T) { func TestADeepDiamondChainLoads(t *testing.T) { // A diamond chain is 2^n routes through n nodes. The search past a bound level // must walk each node once, not once per route, or a deep chain never loads. - files := map[string]string{"l0": `["x"]`} + files := map[string]string{"l0": `"x"`} for i := 1; i <= 30; i++ { files[fmt.Sprintf("l%d", i)] = fmt.Sprintf( - `{"format":"{a}{b}","a":{"format":"{..l%d}"},"b":{"format":"{..l%d}"}}`, i-1, i-1) + `{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1) } - files["thing"] = `{"format":"{p.first} {..l30}","p":{"format":"x","first":["A","B"]}}` - if _, err := New([]string{writeData(t, files)}); err != nil { + files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}` + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil { t.Fatalf("New = %v, want a deep diamond chain to load", err) } } @@ -324,11 +321,11 @@ func TestASharedNodeIsWalkedOnce(t *testing.T) { // Two fields reaching one node make the render graph a diamond, not a tree. // The search past a bound level must take that in its stride rather than walk // the shared node once per route. - f, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{p.first}|{q}","p":[{"format":"{first}","first":["Anna","Bo"]}],` + - `"q":{"format":"{a}{b}","a":{"format":"{..shared}"},"b":{"format":"{..shared}"}}}`, - "shared": `["x"]`, - })}, WithSeed(1)) + f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` + + `"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`, + "shared": `"x"`, + })), WithSeed(1)) if err != nil { t.Fatalf("New = %v, want a shared node accepted", err) } @@ -341,9 +338,9 @@ func TestTheEarlierReaderIsNamed(t *testing.T) { // A token and a calc operand can name one level. The one the format writes // first is the one reported, so the error points at the same place a reader // looking at the format would start. - _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":[{"format":"{first}","first":["1","2"]}]}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`, + }))) if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) { t.Fatalf("New = %v, want the calc operand named, being written first", err) } @@ -353,10 +350,10 @@ func TestReferenceToAMatchingStringIsAccepted(t *testing.T) { // A literal renders one fixed string, so no draw of it can disagree with a // held one. Two unrelated literals that merely spell the same text must not // read as the same node — the trap when comparing a value type. - if _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"{p.city} {..other.tag}","p":{"format":"{city}","city":"Stockholm"}}`, + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`, "other": `{"format":"x","tag":"Stockholm"}`, - })}); err != nil { + }))); err != nil { t.Fatalf("New = %v, want a reference to a matching string accepted", err) } } @@ -368,7 +365,7 @@ func TestNestedChoiceDrawsOneVariant(t *testing.T) { f := engine(21) seen := map[string]bool{} for i := 0; i < 200; i++ { - seen[mustRender(t, f, `{"format":"{p.x}","p":[[{"format":"{x}","x":"1"}],[{"format":"{x}","x":"2"}]]}`)] = true + seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true } if !seen["1"] || !seen["2"] || len(seen) != 2 { t.Fatalf("200 draws produced %v, want 1 and 2", seen) @@ -378,9 +375,9 @@ func TestNestedChoiceDrawsOneVariant(t *testing.T) { func TestRepeatingLevelIsNamedInTheError(t *testing.T) { // The level carrying the repeat is the one to fix, so the error names it // rather than the head the path started from. - _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":["z"]}}}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`, + }))) if err == nil || !strings.Contains(err.Error(), `"p.a"`) { t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err) } @@ -390,9 +387,9 @@ func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) { // A choice of rows is the shape this feature is for, so the repeat rule has to // reach inside one — otherwise the direct spelling is a load error and the same // mistake behind a choice silently drops the repeat. - _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":["x"]},{"format":"{a}","a":["y"]}]}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`, + }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want the repeat behind a choice rejected", err) } @@ -401,9 +398,9 @@ func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) { func TestPathIntoARepeatingLevelIsRejected(t *testing.T) { // A path reads one level's draw, so it can never apply that level's repeat. // The engine rejects options that cannot take effect, and this is one. - _, err := New([]string{writeData(t, map[string]string{ - "cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":["z"]}}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`, + }))) if err == nil || !strings.Contains(err.Error(), "repeat") { t.Fatalf("New = %v, want a path into a repeating level rejected", err) } @@ -412,9 +409,9 @@ func TestPathIntoARepeatingLevelIsRejected(t *testing.T) { func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) { // A head that is one template, not a choice, reports the missing segment // directly — there are no variants to compare. - _, err := New([]string{writeData(t, map[string]string{ + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ "cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`, - })}) + }))) if err == nil || !strings.Contains(err.Error(), `no field "nope"`) { t.Fatalf("New = %v, want it to name the missing field", err) } @@ -511,7 +508,7 @@ func TestDottedTokenErrors(t *testing.T) { want string }{ "unknown head": { - `{"format":"{nope.x}","place":[{"format":"{v}","v":"A"}]}`, + `{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`, `no field "nope"`, }, "unknown tail": { @@ -519,7 +516,7 @@ func TestDottedTokenErrors(t *testing.T) { `"nope"`, }, "tail only some variants carry": { - `{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},{"format":"x"}]}`, + `{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`, `locality`, }, "path into a literal": { @@ -527,12 +524,12 @@ func TestDottedTokenErrors(t *testing.T) { `"y"`, }, "dotted field key": { - `{"format":"[{a.b}]","a.b":["V"]}`, + `{"format":"[{a.b}]","a.b":"V"}`, `contains "."`, }, } for name, c := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": c.file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file}))) if err == nil { t.Errorf("%s: New = nil error, want it rejected at load", name) continue @@ -548,8 +545,7 @@ func TestSamePathReadTwiceReadsOneValue(t *testing.T) { // what a name shown in a display form and again in an address needs. f := engine(7) for i := 0; i < 300; i++ { - got := mustRender(t, f, `{"format":"{p.first}|{p.first}", - "p":[{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}]}`) + got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`) parts := strings.Split(got, "|") if parts[0] != parts[1] { t.Fatalf("draw %d = %q, want one value read twice", i, got) @@ -571,9 +567,7 @@ func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) { // deepPlaces nests the correlated facts a level down: the row holds an addr, and // the addr holds the city and the zip that belongs to it. -const deepPlaces = `{"format":"%s","p":[{"format":"{addr}","addr":[ - {"format":"{city}","city":"Stockholm","zip":"11111"}, - {"format":"{city}","city":"Kiruna","zip":"98100"}]}]}` +const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}` func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) { // A choice below the head is drawn once too, so two paths sharing a prefix @@ -595,13 +589,7 @@ func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) { func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) { // The rule does not run out at two: every level a path passes through is held. f := engine(12) - tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":[{"format":"{geo}","geo":[ - {"format":"{town}","region":"Norrbotten","town":[ - {"format":"{name}","name":"Kiruna","zip":"98100"}, - {"format":"{name}","name":"Luleå","zip":"97200"}]}, - {"format":"{town}","region":"Skåne","town":[ - {"format":"{name}","name":"Malmö","zip":"21100"}, - {"format":"{name}","name":"Lund","zip":"22100"}]}]}]}` + tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}` want := map[string]bool{ "Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true, "Malmö/21100 Skåne": true, "Lund/22100 Skåne": true, @@ -641,9 +629,7 @@ func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) { // Two paths share only what they actually share: a common prefix is one draw, // and each keeps its own draw past the point they part. f := engine(14) - tmpl := `{"format":"{p.a.v}{p.b.v}","p":[{"format":"x", - "a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}], - "b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}]}` + tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}` seen := map[string]bool{} for i := 0; i < 400; i++ { seen[mustRender(t, f, tmpl)] = true @@ -656,16 +642,15 @@ func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) { } func TestEmptyPathSegmentIsRejected(t *testing.T) { - // "{a.}", "{.b}" and "{a..b}" are unfinished paths, and the segment naming - // nothing is reported as that rather than as a missing field. An empty name is - // rejected where it is authored, so no data can make these resolve. + // "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is + // reported as that rather than as a missing field. An empty name is rejected + // where it is authored, so no data can make these resolve. rejected := map[string]string{ - "trailing dot": `{"format":"[{a.}]","a":{"format":"x"}}`, - "leading dot": `{"format":"[{.b}]","a":{"format":"{b}","b":["V"]}}`, - "double dot": `{"format":"[{a..b}]","a":{"format":"x"}}`, + "trailing dot": `{"format":"[{a.}]","a":"x"}`, + "double dot": `{"format":"[{a..b}]","a":"x"}`, } for name, file := range rejected { - _, err := New([]string{writeData(t, map[string]string{"cat": file})}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) if err == nil { t.Errorf("%s: New = nil error, want the unfinished path rejected", name) continue @@ -680,9 +665,9 @@ func TestCycleThroughAPathTokenIsRejected(t *testing.T) { // A path token is a render edge like any other, so a cycle routed through one // must be caught at New. Reaching render would be fatal: the recursion never // terminates, and a stack overflow cannot be recovered. - _, err := New([]string{writeData(t, map[string]string{ - "a": `{"format":"{p.x}","p":{"format":"{x}","x":{"format":"{..a}"}}}`, - })}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/a}"}}`, + }))) if err == nil || !strings.Contains(err.Error(), "reference cycle") { t.Fatalf("New = %v, want the cycle through {p.x} rejected", err) } @@ -692,7 +677,7 @@ func TestBoundPathIsReachableByFake(t *testing.T) { // Binding changes how a format reads a sibling, not what List and Fake offer: // the sub-fields stay addressable on their own. dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")}) - f, err := New([]string{dir}, WithSeed(9)) + f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9)) if err != nil { t.Fatalf("New = %v", err) } @@ -702,3 +687,37 @@ func TestBoundPathIsReachableByFake(t *testing.T) { } } } + +func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`: "write {w} once", + `{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`: "write {w} once", + `{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`: "write {n} once", + `{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once", + } { + _, err := compile(parse(t, src)) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") { + t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want) + } + } + for _, ok := range []string{ + `{"format":"{w} {w}","w":["a","b"]}`, + `{"format":"{w} {uppercase(w)}","w":["a","b"]}`, + `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`, + `{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`, + } { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v, want it accepted", ok, err) + } + } +} + +func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) { + tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template) + if !ok { + t.Fatal("not a template") + } + mustPanic(t, "unheld arm with a path", func() { + readField(engine(1).rand, tm, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"}) + }) +} diff --git a/loading_test.go b/loading_test.go index 5727bde..be407ed 100644 --- a/loading_test.go +++ b/loading_test.go @@ -1,9 +1,12 @@ package fejkdata import ( + "encoding/json" + "fmt" "os" "path/filepath" "reflect" + "regexp" "slices" "strings" "testing" @@ -13,13 +16,13 @@ import ( // folder segments — descending transparently through single-variant choices. func TestList(t *testing.T) { dir := writeData(t, map[string]string{ - "person": `{"format":"{first} {last}","first":["A"],"last":["B"]}`, + "person": `{"format":"{first} {last}","first":"A","last":"B"}`, "word": `["x", "y"]`, - "geo/city": `["Z"]`, - // A bound {..path} reference is a render edge, not an addressable field. - "greeting": `{"format":"hej {..person.first} and {own}","own":["x"]}`, + "geo/city": `"Z"`, + // A bound {/path} reference is a render edge, not an addressable field. + "greeting": `{"format":"hej {/person.first} and {own}","own":"x"}`, // Only the fields every variant carries are addressable, so "extra" is not. - "coin": `[{"format":"{code}","code":["A"],"name":["Aa"]},{"format":"{code}","code":["B"],"name":["Bb"],"extra":["x"]}]`, + "coin": `[{"format":"{code}","code":"A","name":"Aa"},{"format":"{code}","code":"B","name":"Bb","extra":"x"}]`, }) got := newGenerator(t, dir, WithSeed(1)).List() want := []string{"coin", "coin.code", "coin.name", "geo.city", "greeting", "greeting.own", "person", "person.first", "person.last", "word"} @@ -57,7 +60,7 @@ func TestNewLoadsAnyDirName(t *testing.T) { } func TestNewEmptyDirErrors(t *testing.T) { - if _, err := New([]string{writeData(t, nil)}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, nil))); err == nil { t.Fatal("New(empty dir) = nil error") } } @@ -66,8 +69,8 @@ func TestNewEmptyDirErrors(t *testing.T) { // namespace, so data//person.json is reachable as ".person". func TestFoldersBecomeDotPaths(t *testing.T) { dir := writeData(t, map[string]string{ - "sv_SE/greeting": `["hej"]`, - "en_US/greeting": `["hi"]`, + "sv_SE/greeting": `"hej"`, + "en_US/greeting": `"hi"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "sv_SE.greeting"); got != "hej" { @@ -82,7 +85,7 @@ func TestFoldersBecomeDotPaths(t *testing.T) { // a/b/c, then deep JSON fields inside the file at the end of that path. func TestNestedFoldersAndJSON(t *testing.T) { dir := writeData(t, map[string]string{ - "a/b/c/thing": `{"format":"{x}","x":{"format":"{y}","y":{"format":"{z}","z":["leaf"]}}}`, + "a/b/c/thing": `{"format":"{x}","x":{"format":"{y}","y":{"format":"{z}","z":"leaf"}}}`, }) f := newGenerator(t, dir, WithSeed(1)) // Folders a.b.c, file thing, then JSON fields x.y.z — one continuous path. @@ -96,7 +99,7 @@ func TestNestedFoldersAndJSON(t *testing.T) { } func TestRenderingAFolderErrors(t *testing.T) { - dir := writeData(t, map[string]string{"sv_SE/greeting": `["hej"]`}) + dir := writeData(t, map[string]string{"sv_SE/greeting": `"hej"`}) f := newGenerator(t, dir, WithSeed(1)) if _, err := f.Fake("sv_SE"); err == nil { t.Fatal("Fake(folder) = nil error, want a not-a-value error") @@ -106,8 +109,8 @@ func TestRenderingAFolderErrors(t *testing.T) { // TestMultiPathLastWins loads two dirs; on a name clash the later dir wins, and // non-clashing entries from both are reachable (data combines). func TestMultiPathLastWins(t *testing.T) { - a := writeData(t, map[string]string{"greeting": `["from-a"]`, "only-a": `["a"]`}) - b := writeData(t, map[string]string{"greeting": `["from-b"]`, "only-b": `["b"]`}) + a := writeData(t, map[string]string{"greeting": `"from-a"`, "only-a": `"a"`}) + b := writeData(t, map[string]string{"greeting": `"from-b"`, "only-b": `"b"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "greeting"); got != "from-b" { t.Fatalf("greeting = %q, want from-b (last loaded wins)", got) @@ -124,8 +127,8 @@ func TestMultiPathLastWins(t *testing.T) { // children rather than replacing wholesale: each dir adds a file to sv_SE, and // a per-file clash inside still resolves last-wins. func TestMultiPathMergesFolders(t *testing.T) { - a := writeData(t, map[string]string{"sv_SE/person": `["from-a"]`, "sv_SE/shared": `["a"]`}) - b := writeData(t, map[string]string{"sv_SE/company": `["from-b"]`, "sv_SE/shared": `["b"]`}) + a := writeData(t, map[string]string{"sv_SE/person": `"from-a"`, "sv_SE/shared": `"a"`}) + b := writeData(t, map[string]string{"sv_SE/company": `"from-b"`, "sv_SE/shared": `"b"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "sv_SE.person"); got != "from-a" { t.Fatalf("sv_SE.person = %q, want from-a (folder merged, not replaced)", got) @@ -163,10 +166,10 @@ func TestListedPathsAllRender(t *testing.T) { // carrying no JSON never contributed a namespace, so neither may fail the load. func TestHiddenEntriesAreSkipped(t *testing.T) { dir := writeData(t, map[string]string{ - "cat": `["V"]`, - ".git/HEAD": `["ignored"]`, - ".hidden": `["ignored"]`, - "empty.folder/doc": `["ignored"]`, + "cat": `"V"`, + ".git/HEAD": `"ignored"`, + ".hidden": `"ignored"`, + "empty.folder/doc": `"ignored"`, }) if err := os.Rename(filepath.Join(dir, "empty.folder", "doc.json"), filepath.Join(dir, "empty.folder", "doc.txt")); err != nil { t.Fatal(err) @@ -181,3 +184,222 @@ func TestHiddenEntriesAreSkipped(t *testing.T) { } } } + +func TestRepeatProductAlongAPathIsCapped(t *testing.T) { + for name, files := range map[string]map[string]string{ + "nested": {"cat": `{"format":"{a}","repeat":2048,"a":{"format":"{b}","repeat":2048,"b":"x"}}`}, + "through a reference": { + "a": `{"format":"{/b}","repeat":2048}`, + "b": `{"format":"x","repeat":2048}`, + }, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))) + if err == nil || !strings.Contains(err.Error(), "repeat") || !strings.Contains(err.Error(), "1048576") { + t.Errorf("%s: New = %v, want the repeat product rejected naming the maximum", name, err) + } + } + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "cat": `{"format":"{a}{c}","repeat":1024,"a":{"format":"{b}","repeat":1024,"b":"x"},"c":{"format":"y","repeat":1024}}`, + }))); err != nil { + t.Errorf("New = %v, want 1024 x 1024 along one path accepted", err) + } +} + +func TestNewErrors(t *testing.T) { + // Pointing New at a file (not a directory) fails. + file := filepath.Join(t.TempDir(), "xx_XX") + if err := os.WriteFile(file, []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := New(WithoutShippedData(), WithDataPath(file)); err == nil { + t.Error("New(file) = nil error, want not-a-directory error") + } + // Invalid JSON in a category file fails. + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"broken": `{ not json`}))); err == nil { + t.Error("New(invalid JSON) = nil error") + } + // An option that cannot take effect, and a category or folder no dot path can + // reach, are mistakes New must name rather than accept and ignore. + rejected := map[string]struct { + files map[string]string + want string + }{ + "separator without repeat": { + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, + "has no effect without a repeat above 1", + }, + "separator with an explicit repeat of 1": { + map[string]string{"a": `{"format":"{x}","x":"1","separator":","}`}, + "has no effect without a repeat above 1", + }, + "weight outside a choice": { + map[string]string{"a": `{"format":"x","weight":5}`}, + "weight only skews a choice's items", + }, + "non-numeric weight outside a choice": { + map[string]string{"a": `{"format":"x","weight":"bad"}`}, + "weight only skews a choice's items", + }, + "weight used as a field": { + map[string]string{"a": `{"format":"{name} {weight}kg","name":"Anvil","weight":["7"]}`}, + "can never be a field", + }, + "an option name used as a token": { + map[string]string{"a": `"{weight}"`}, + `"weight" is an option and can never be a field`, + }, + "category name with a dot": { + map[string]string{"a.b": `"1"`}, + `category "a.b" contains "."`, + }, + "folder name with a dot": { + map[string]string{"a.b/cat": `"1"`}, + `/a.b: folder "a.b" contains "."`, + }, + // The token grammar reserves three more characters. A name carrying one + // still resolves by dot path, but no format can name it, so it is rejected + // where it is authored rather than at the token that cannot reach it. + "field name with a pipe": { + map[string]string{"a": `{"format":"{x}","x":"1","b|c":"2"}`}, + `field "b|c" contains "|"`, + }, + "field name with a paren": { + map[string]string{"a": `{"format":"{x}","x":"1","b(c":"2"}`}, + `field "b(c" contains "("`, + }, + "field name with a closing brace": { + map[string]string{"a": `{"format":"{x}","x":"1","b}c":"2"}`}, + `field "b}c" contains "}"`, + }, + "category name with a pipe": { + map[string]string{"a|b": `"1"`}, + `category "a|b" contains "|"`, + }, + // An empty name is not a path segment, so List never offered it — while a + // bare {}, a trailing dot in Fake("a.") and a {/a.} reference all reached + // it. The engine accepted spellings it would never advertise. + "empty field name": { + map[string]string{"a": `{"format":"[{}]","":"VALUE"}`}, + `field "" is empty`, + }, + "folder name with a paren": { + map[string]string{"a(b/cat": `"1"`}, + `folder "a(b" contains "("`, + }, + // A repeated arm skews an alternation, which weight is the spelling for. + "repeated alternation arm": { + map[string]string{"a": `{"format":"{x|x}","x":"1"}`}, + `arm "x" is repeated`, + }, + "repeated arm among others": { + map[string]string{"a": `{"format":"{x|y|x}","x":"1","y":"2"}`}, + `arm "x" is repeated`, + }, + "repeated path arm": { + map[string]string{"a": `{"format":"{p.v|p.v}","p":{"format":"{v}","v":"1"}}`}, + `arm "p.v" is repeated`, + }, + // A reference arm is the only kind that reaches the repeat check by passing + // the per-arm checks rather than falling through them. + "repeated reference arm": { + map[string]string{"a": `"x"`, "b": `"{/a|/a}"`}, + `arm "/a" is repeated`, + }, + // An arm that is broken on its own terms is reported as that, not as a + // repeat: the repeat is a consequence of the real mistake. + "repeated arm with no path": { + map[string]string{"a": `"{/|/}"`}, + "reference has no path", + }, + // No field can be named "", so the token is told that rather than sent to + // name one — the fix "no field" points at is itself a load error. + "repeated empty arm": { + map[string]string{"a": `"{|}"`}, + "a name is never empty", + }, + "bare empty token": { + map[string]string{"a": `{"format":"[{}]","x":"1"}`}, + "a name is never empty", + }, + } + for name, c := range rejected { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) + if err == nil { + t.Errorf("%s: New = nil error, want it rejected at load", name) + continue + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want) + } + } + // Data that works today must keep working, and stay reachable. + accepted := map[string]struct { + files map[string]string + path string + want string + }{ + "option name as a field": {map[string]string{"a": `{"format":"{name} {Weight}kg","name":"Anvil","Weight":"7"}`}, "a", "Anvil 7kg"}, + "format spelling as a field": {map[string]string{"a": `{"format":"{Format}","Format":"PDF"}`}, "a", "PDF"}, + "hyphenated field": {map[string]string{"a": `{"format":"{x-y}","x-y":"1"}`}, "a.x-y", "1"}, + "category named Format": {map[string]string{"Format": `"1"`}, "Format", "1"}, + "folder named Repeat": {map[string]string{"Repeat/cat": `"1"`}, "Repeat.cat", "1"}, + "field with a closing paren": {map[string]string{"a": `{"format":"{b)c}","b)c":"2"}`}, "a", "2"}, + "repeat without a separator": {map[string]string{"a": `{"format":"{x}","repeat":3,"x":"1"}`}, "a", "111"}, + // One name in two separate tokens is two independent draws, not a repeated + // arm; only a repeat within one alternation is rejected. + "one name in two tokens": {map[string]string{"a": `{"format":"{x}{x}","x":"1"}`}, "a", "11"}, + } + for name, c := range accepted { + f, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) + if err != nil { + t.Errorf("%s: New = %v, want it accepted", name, err) + continue + } + if got, err := f.Fake(c.path); err != nil || got != c.want { + t.Errorf("%s: Fake(%q) = %q, %v, want %q", name, c.path, got, err, c.want) + } + } +} + +func TestCategoryRootShapes(t *testing.T) { + dir := writeData(t, map[string]string{ + "obj": `"{digits(2)}"`, // object root + "lit": `"hello"`, // bare-string root + }) + f := newGenerator(t, dir, WithSeed(1)) + if got := fake(t, f, "obj"); !regexp.MustCompile(`^\d\d$`).MatchString(got) { + t.Errorf("object-root category = %q, want two digits", got) + } + if got := fake(t, f, "lit"); got != "hello" { + t.Errorf("string-root category = %q, want hello", got) + } +} + +func TestLongStringList(t *testing.T) { + const n = 2000 + names := make([]string, n) + for i := range names { + names[i] = fmt.Sprintf("name-%04d", i) + } + list, err := json.Marshal(names) + if err != nil { + t.Fatal(err) + } + f := newGenerator(t, writeData(t, map[string]string{"name": string(list)}), WithSeed(1)) + + valid := map[string]bool{} + for _, v := range names { + valid[v] = true + } + seen := map[string]bool{} + for i := 0; i < 20000; i++ { + v := fake(t, f, "name") + if !valid[v] { + t.Fatalf("got %q, not in the list", v) + } + seen[v] = true + } + if len(seen) < n*8/10 { + t.Fatalf("only %d/%d distinct values seen; selection looks skewed", len(seen), n) + } +} diff --git a/node.go b/node.go index 4e81294..fb6aee9 100644 --- a/node.go +++ b/node.go @@ -1,22 +1,18 @@ package fejkdata import ( + "encoding/json" "fmt" "math" "sort" "strings" ) -// node is a compiled template element: literal, choice, or template. Compiling -// JSON into these once (see compile) means rendering never re-inspects the raw -// JSON or re-sums weights. +// node is a compiled template element: a choice or a template. Compiling JSON +// into these once (see compile) means rendering never re-inspects the raw JSON or +// re-sums weights. type node interface{ isNode() } -// literal is emitted verbatim, never formatted. -type literal string - -func (literal) isNode() {} - // group is a namespace of named children, built from a directory of JSON files // and subdirectories. It has no value of its own: descend into a named child by // dot path; rendering one is an error (see Fake). @@ -36,16 +32,20 @@ type choice struct { func (*choice) isNode() {} -// template renders a format string, substituting {tokens} from fields. repeat -// (default 1) renders that format that many times and joins the results with -// separator (default ""), each render an independent pick. +// template renders a format string, substituting {tokens} from fields. A bare +// JSON string is a template with no fields. repeat (default 1) renders that format +// that many times and joins the results with separator (default ""), each render +// an independent pick. type template struct { format string fields map[string]node repeat int separator string - ops []op // format compiled once (see compileOps); what expand walks - grow int // minimum output size, to size the render buffer + ops []op // format compiled once (see compileOps); what expand walks + grow int // minimum output size, to size the render buffer + fixed bool // no op varies, so every render is lit + lit string // the whole output when fixed + refs map[string]refBinding // each reference the format reads -> what it is bound to // bound maps each field the format addresses by dotted path to one path token // reading it, which is the half of an overlap the fences name. nil when the // format takes no path. @@ -57,6 +57,15 @@ type template struct { func (*template) isNode() {} +// field is the node a path segment names; a binding is a render edge, not a field. +func (t *template) field(seg string) (node, bool) { + if isRef(seg) { + return nil, false + } + n, ok := t.fields[seg] + return n, ok +} + // compile converts parsed JSON into a node tree, validating structure up front. // Only a choice's items carry a weight, so one here would be inert whatever its type. func compile(v any) (node, error) { @@ -72,7 +81,7 @@ func compile(v any) (node, error) { func compileItem(v any) (node, error) { switch v := v.(type) { case string: - return literal(v), nil + return compileString(v) case []any: return compileChoice(v) case map[string]any: @@ -82,10 +91,47 @@ func compileItem(v any) (node, error) { } } +func compileString(s string) (node, error) { + if err := checkTokens(s, nil); err != nil { + return nil, err + } + t := &template{format: s, repeat: 1} + if err := t.compileFormat(); err != nil { + return nil, err + } + return t, nil +} + +// compileFormat compiles the format into ops once every field is in place, and +// applies the fences that need the compiled reads. +func (t *template) compileFormat() error { + c := compileOps(t.format, t.refs) + t.ops, t.grow, t.bound, t.held = c.ops, c.grow, c.bound, c.held + t.fixed = true + for _, o := range t.ops { + if o.kind != 'l' { + t.fixed = false + } + } + if t.fixed && len(t.ops) == 1 { + t.lit = t.ops[0].lit + } + if err := checkNoOverlap(t.format, t.bound, t.refs); err != nil { + return err + } + return checkNoRepeatedRead(t.format, c, t.refs) +} + func compileChoice(items []any) (node, error) { if len(items) == 0 { return nil, fmt.Errorf("empty choice") } + if len(items) == 1 { + return nil, fmt.Errorf("a one-item choice is its item; write the item") + } + if err := checkNoRepeatedItem(items); err != nil { + return nil, err + } c := &choice{items: make([]node, len(items))} cum := make([]float64, len(items)) var total float64 @@ -107,50 +153,112 @@ func compileChoice(items []any) (node, error) { c.items[i] = n } if weighted { // uniform choices skip the weight table and pick in O(1) - if total <= 0 || math.IsInf(total, 1) { - return nil, fmt.Errorf("choice weights must sum to a finite positive number, got %v", total) + if math.IsInf(total, 1) { + return nil, fmt.Errorf("choice weights must sum to a finite number, got %v", total) } c.cum = cum } - if len(c.items) > 1 { - // Safe to precompute: a choice's items come from one file, so no group can - // appear inside one, and neither mergeChildren nor linkRefs can reach in. - c.shared = sharedPaths(c.items) - } + // Computed before linkRefs binds references into the items: a binding is keyed + // by a reference sigil, which paths skips, so the set is the same after. + c.shared = sharedPaths(c.items) return c, nil } -func compileTemplate(m map[string]any) (node, error) { - format, ok := m["format"].(string) - if !ok { - return nil, fmt.Errorf("template object missing string \"format\"") +// checkNoRepeatedItem rejects a choice that lists one item twice: a pick is even +// over the items, so a repeat is a second spelling of weight. The error names the +// spelling that does skew a pick. +func checkNoRepeatedItem(items []any) error { + seen := make(map[string]int, len(items)) + for i, raw := range items { + key, isString := raw.(string) + if !isString { + b, err := json.Marshal(raw) + if err != nil { + return err + } + key = "\x00" + string(b) + } + if j, dup := seen[key]; dup { + if s, isString := raw.(string); isString { + return fmt.Errorf("choice item %q is repeated; skew the odds with a weight instead: { \"format\": %q, \"weight\": 2 }", s, s) + } + return fmt.Errorf("choice item %d repeats item %d; skew the odds with a weight on one of them instead", i, j) + } + seen[string(key)] = i } - repeat, err := repeatOf(m) + return nil +} + +func compileTemplate(m map[string]any) (node, error) { + o, err := readOptions(m) if err != nil { return nil, err } - sep := "" + fields, err := compileFields(m) + if err != nil { + return nil, err + } + if len(fields) == 0 && o.repeat == 1 && !o.weighted { + return nil, fmt.Errorf("an object holding only a format is a string; write %q", o.format) + } + if err := checkTokens(o.format, fields); err != nil { + return nil, err + } + t := &template{format: o.format, fields: fields, repeat: o.repeat, separator: o.separator} + if err := t.compileFormat(); err != nil { + return nil, err + } + return t, nil +} + +// templateOptions is what a template object's option keys say. +type templateOptions struct { + format string + repeat int + separator string + weighted bool +} + +func readOptions(m map[string]any) (templateOptions, error) { + var o templateOptions + format, ok := m["format"].(string) + if !ok { + return o, fmt.Errorf("template object missing string \"format\"") + } + o.format = format + repeat, err := repeatOf(m) + if err != nil { + return o, err + } + o.repeat = repeat if sv, ok := m["separator"]; ok { - if sep, ok = sv.(string); !ok { - return nil, fmt.Errorf("separator must be a string, got %T", sv) + if o.separator, ok = sv.(string); !ok { + return o, fmt.Errorf("separator must be a string, got %T", sv) } if repeat == 1 { - return nil, fmt.Errorf("separator joins repeated renders, so it has no effect without a repeat above 1") + return o, fmt.Errorf("separator joins repeated renders, so it has no effect without a repeat above 1") + } + if o.separator == "" { + return o, fmt.Errorf("separator \"\" is the default, so it has no effect; drop it") } } - t := &template{format: format, fields: make(map[string]node, len(m)), repeat: repeat, separator: sep} + _, o.weighted = m["weight"] + return o, nil +} + +// compileFields compiles every non-option key of a template object, in name order +// so which of several bad fields is reported does not vary. +func compileFields(m map[string]any) (map[string]node, error) { + fields := make(map[string]node, len(m)) keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } - sort.Strings(keys) // so which of several bad fields is reported does not vary + sort.Strings(keys) for _, k := range keys { if isOption(k) { continue } - if isRef(k) { - return nil, fmt.Errorf("field %q starts with %q, which is reserved for {..path} bindings", k, refPrefix) - } if err := checkName(k); err != nil { return nil, fmt.Errorf("field %w", err) } @@ -158,62 +266,13 @@ func compileTemplate(m map[string]any) (node, error) { if err != nil { return nil, fmt.Errorf("field %q: %w", k, err) } - t.fields[k] = n - } - if err := checkTokens(format, t.fields); err != nil { - return nil, err - } - t.ops, t.grow, t.bound, t.held = compileOps(format) - if err := checkNoOverlap(format, t.bound); err != nil { - return nil, err - } - return t, nil -} - -// checkPath reports whether a token's dotted tail can address a node whichever way -// the draw goes, by the reachability rule descend applies — a multi-variant choice -// must carry the whole remaining path in the set every variant shares — plus the -// rules a held draw adds, which descend has no need of: a level a path reads may -// not carry a repeat, and each variant answers for that itself. So a path that -// validates here resolves on every render, and a typo is a New-time error. -func checkPath(n node, tail []string, level string) error { - if len(tail) == 0 { - return nil - } - switch n := n.(type) { - case *template: - if n.repeat > 1 { - return fmt.Errorf("the level %q carries a repeat, which a path reading one draw of it cannot apply", level) - } - child, ok := n.fields[tail[0]] - if !ok { - return fmt.Errorf("no field %q", tail[0]) - } - return checkPath(child, tail[1:], level+"."+tail[0]) - case *choice: - if len(n.items) > 1 { - if want := strings.Join(tail, "."); !n.shared[want] { - return unreachableInChoice(n, want) - } - // Reachability is settled; each variant still answers for itself, so a - // rule about the level (its repeat) holds behind a choice as in front. - for _, item := range n.items { - if err := checkPath(item, tail, level); err != nil { - return err - } - } - return nil - } - return checkPath(n.items[0], tail, level) - case literal: - return fmt.Errorf("a plain string has no field %q", tail[0]) - default: - return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) + fields[k] = n } + return fields, nil } // repeatOf reads a template's "repeat" (default 1): how many times its format -// is rendered and concatenated. A present one must be a positive integer. +// is rendered and concatenated. A present one must be an integer above 1. func repeatOf(m map[string]any) (int, error) { rv, ok := m["repeat"] if !ok { @@ -226,14 +285,17 @@ func repeatOf(m map[string]any) (int, error) { if math.IsNaN(r) || math.IsInf(r, 0) || r < 1 || r != math.Trunc(r) { return 0, fmt.Errorf("repeat must be a positive integer, got %v", rv) } - if r > maxLen { // cap so a fat-fingered repeat can't build a multi-GB string - return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, maxLen) + if r == 1 { + return 0, fmt.Errorf("repeat 1 is the default, so it has no effect; drop it") + } + if r > MaxRepeat { // caps the renders one repeat asks for; checkRepeatReach bounds what nested ones multiply to + return 0, fmt.Errorf("repeat %v exceeds the maximum %d", rv, MaxRepeat) } return int(r), nil } // weightOf reads a node's "weight" (default 1) from its raw JSON form. Only -// template objects carry weight; a present one must be finite and non-negative. +// template objects carry weight; a present one must be finite and positive. func weightOf(raw any) (float64, error) { m, ok := raw.(map[string]any) if !ok { @@ -248,16 +310,23 @@ func weightOf(raw any) (float64, error) { return 0, fmt.Errorf("weight must be a number, got %T", wv) } if w < 0 || math.IsNaN(w) || math.IsInf(w, 0) { - return 0, fmt.Errorf("weight must be finite and non-negative, got %v", w) + return 0, fmt.Errorf("weight must be finite and positive, got %v", w) + } + if w == 0 { + return 0, fmt.Errorf("weight 0 means the item is never drawn; remove the item instead") + } + if w == 1 { + return 0, fmt.Errorf("weight 1 is the default, so it has no effect; drop it") } return w, nil } // reservedInName is what a category, folder or field name may not contain: a dot // separates the segments of a path, '|' the arms of a token, '(' opens a function -// call and '}' ends the token. A name carrying one is reachable by no format, so it -// is rejected where it is authored rather than at the token that cannot reach it. -const reservedInName = ".|(}" +// call, braces delimit the token and '/' starts a reference. A name carrying one is +// reachable by no format, so it is rejected where it is authored rather than at +// the token that cannot reach it. +const reservedInName = ".|({}/" // reservedList spells reservedInName for an error message, so the two cannot drift. var reservedList = strings.Join(strings.Split(reservedInName, ""), " ") diff --git a/one_spelling_test.go b/one_spelling_test.go new file mode 100644 index 0000000..f866609 --- /dev/null +++ b/one_spelling_test.go @@ -0,0 +1,49 @@ +package fejkdata + +import ( + "strings" + "testing" +) + +func TestOneItemChoiceIsRejected(t *testing.T) { + for _, src := range []string{`["x"]`, `[{"format":"{d}","d":"x"}]`, `{"format":"{w}","w":["only"]}`} { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), "one-item choice") { + t.Errorf("compile(%s) = %v, want the one-item choice rejected", src, err) + } + } +} + +func TestRepeatedChoiceItemIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `["a", "a", "b"]`: `{ "format": "a", "weight": 2 }`, + `[{"format":"{x}","x":"1"},{"format":"{x}","x":"1"}]`: "repeats item", + `{"format":"{w}","w":["", "", "x"]}`: `{ "format": "", "weight": 2 }`, + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error naming %s", src, err, want) + } + } + if _, err := compile(parse(t, `[{"format":"a","weight":2}, "b"]`)); err != nil { + t.Errorf("compile(weighted a, b) = %v", err) + } +} + +func TestInertObjectIsRejected(t *testing.T) { + for src, want := range map[string]string{ + `{"format":"Malmö"}`: `write "Malmö"`, + `{"format":"{digits(3)}"}`: `write "{digits(3)}"`, + `[{"format":"a","weight":1},"b"]`: "weight 1", + `{"format":"{x}","x":"v","repeat":1}`: "repeat 1", + `{"format":"{x}","x":"v","separator":","}`: "separator", + `{"format":"{x}","x":"v","repeat":2,"separator":""}`: "default", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) + } + } + for _, ok := range []string{`[{"format":"a","weight":2},"b"]`, `{"format":"ab","repeat":2}`, `{"format":"{x}","x":"v"}`, `"{digits(3)}"`} { + if _, err := compile(parse(t, ok)); err != nil { + t.Errorf("compile(%s) = %v", ok, err) + } + } +} diff --git a/path.go b/path.go new file mode 100644 index 0000000..ee9b1c5 --- /dev/null +++ b/path.go @@ -0,0 +1,110 @@ +package fejkdata + +import ( + "fmt" + "sort" + "strings" +) + +// pathWalk is what one walk of a dotted path does at each kind of level: choice +// returns the variants to continue into (none stops the walk); level runs at each +// template a segment descends into; leaf runs where the tail ends. A nil action +// is skipped. +type pathWalk struct { + choice func(c *choice, rest []string) ([]node, error) + level func(t *template, rest []string) error + leaf func(n node) error +} + +// walkPath descends tail from n: a group or template by its next segment, a +// choice by w.choice, which consumes no segment. A missing segment is an error, +// so no walk reaches past what the data holds. A table-shaped dispatch, one case +// per node kind, kept whole on purpose. +func walkPath(n node, tail []string, w pathWalk) error { + if len(tail) == 0 { + if w.leaf != nil { + return w.leaf(n) + } + return nil + } + switch n := n.(type) { + case *group: + child, ok := n.children[tail[0]] + if !ok { + return fmt.Errorf("no entry %q", tail[0]) + } + return walkPath(child, tail[1:], w) + case *template: + if w.level != nil { + if err := w.level(n, tail); err != nil { + return err + } + } + child, ok := n.field(tail[0]) + if !ok { + return fmt.Errorf("no field %q", tail[0]) + } + return walkPath(child, tail[1:], w) + case *choice: + if w.choice == nil { + return nil + } + next, err := w.choice(n, tail) + if err != nil { + return err + } + for _, item := range next { + if err := walkPath(item, tail, w); err != nil { + return err + } + } + return nil + } + return fmt.Errorf("cannot descend into %T at %q", n, tail[0]) +} + +// carriedByAll is the choice rule a path that must resolve on every call obeys: +// the rest of the tail must be one every variant carries. +func carriedByAll(c *choice, rest []string) error { + if want := strings.Join(rest, "."); !c.shared[want] { + return unreachableInChoice(c, want) + } + return nil +} + +// unreachableInChoice reports that a path cannot step through this choice, listing +// what every variant does carry. It reads the precomputed set, so a failing path +// costs no more than a rendering one. +func unreachableInChoice(c *choice, want string) error { + if len(c.shared) == 0 { + return fmt.Errorf("no variant of this %d-way choice carries %q", len(c.items), want) + } + offered := make([]string, 0, len(c.shared)) + for p := range c.shared { + offered = append(offered, p) + } + sort.Strings(offered) + return fmt.Errorf("not every variant of this %d-way choice carries %q; all carry %v", len(c.items), want, offered) +} + +// checkPath proves a dotted tail resolves whichever way the draws go — a choice +// must carry the rest of the path in the set every variant shares — and that no +// level a path reads carries a repeat, which one draw of it could not apply. So a +// path that validates here resolves on every render, and a typo is a New-time +// error. +func checkPath(n node, tail []string, level string) error { + return walkPath(n, tail, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if err := carriedByAll(c, rest); err != nil { + return nil, err + } + return c.items, nil + }, + level: func(t *template, rest []string) error { + if t.repeat > 1 { + return fmt.Errorf("the level %q carries a repeat, which a path reading one draw of it cannot apply", join(level, strings.Join(tail[:len(tail)-len(rest)], "."))) + } + return nil + }, + }) +} diff --git a/path_test.go b/path_test.go new file mode 100644 index 0000000..b784793 --- /dev/null +++ b/path_test.go @@ -0,0 +1,152 @@ +package fejkdata + +import ( + "slices" + "strings" + "testing" +) + +func TestWalkPathStopsAtAMissingSegment(t *testing.T) { + n := compiled(t, `{"format":"{a}","a":{"format":"{b}","b":"leaf"}}`) + var seen []string + walk := pathWalk{ + level: func(tm *template, rest []string) error { seen = append(seen, "level:"+rest[0]); return nil }, + leaf: func(n node) error { seen = append(seen, "leaf"); return nil }, + } + if err := walkPath(n, []string{"a", "b"}, walk); err != nil { + t.Fatalf("walkPath(a.b) = %v", err) + } + if want := []string{"level:a", "level:b", "leaf"}; !slices.Equal(seen, want) { + t.Errorf("walk visited %v, want %v", seen, want) + } + seen = nil + err := walkPath(n, []string{"a", "nope", "deeper"}, walk) + if err == nil || !strings.Contains(err.Error(), `no field "nope"`) { + t.Errorf("walkPath(a.nope.deeper) = %v, want the missing segment named", err) + } + if slices.Contains(seen, "leaf") { + t.Errorf("walk reached a leaf past a missing segment: %v", seen) + } +} + +func TestWalkPathChoiceConsumesNoSegment(t *testing.T) { + n := compiled(t, `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`) + var leaves []node + err := walkPath(n, []string{"f"}, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if len(rest) != 1 || rest[0] != "f" { + t.Errorf("choice saw rest %v, want [f]", rest) + } + return c.items, nil + }, + leaf: func(n node) error { leaves = append(leaves, n); return nil }, + }) + if err != nil || len(leaves) != 2 { + t.Fatalf("walkPath through a choice = %v, %d leaves, want both variants' f", err, len(leaves)) + } +} + +func TestDeepDottedPath(t *testing.T) { + // A 5-segment path descends through alternating object/array nodes; choices + // on the path are single-variant, so it resolves deterministically. + f := engine(1) + f.categories = map[string]node{ + "deep": compiled(t, `{"format":"{a}","a":{"format":"{b}","b":{"format":"{c}","c":{"format":"{d}","d":"leaf"}}}}`), + } + if got, err := f.Fake("deep.a.b.c.d"); err != nil || got != "leaf" { + t.Fatalf("Fake(deep.a.b.c.d) = %q, %v, want leaf", got, err) + } + // Rendering the whole tree resolves the same chain. + if got, err := f.Fake("deep"); err != nil || got != "leaf" { + t.Fatalf("Fake(deep) = %q, %v, want leaf", got, err) + } +} + +func TestDescendIntoStringErrors(t *testing.T) { + f := engine(1) + f.categories = map[string]node{"greeting": compiled(t, `"hej"`)} + if _, err := f.Fake("greeting.extra"); err == nil || !strings.Contains(err.Error(), `no field "extra"`) { + t.Fatalf("Fake(greeting.extra) = %v, want a no-field error", err) + } +} + +// TestPathThroughChoice pins the rule that keeps a dotted path from rendering on +// one call and failing on the next: every variant must carry the rest of the path. +func TestPathThroughChoice(t *testing.T) { + dir := writeData(t, map[string]string{ + "every": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2"}]`, + "notall": `[{"format":"{f}","f":"1"},"plain"]`, + "some": `[{"format":"{f}","f":"1"},{"format":"{f}","f":"2","extra":"x"}]`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 200; i++ { + if got := fake(t, f, "every.f"); got != "1" && got != "2" { + t.Fatalf("every.f = %q, want 1 or 2", got) + } + } + // A path only some variants carry is reported against what all of them carry. + if _, err := f.Fake("some.extra"); err == nil || !strings.Contains(err.Error(), "all carry [f]") { + t.Errorf("Fake(some.extra) = %v, want it to name what every variant carries", err) + } + var first string + for i := 0; i < 200; i++ { + _, err := f.Fake("notall.f") + if err == nil { + t.Fatal("notall.f = nil error, want the same failure every call") + } + if i == 0 { + first = err.Error() + } else if err.Error() != first { + t.Fatalf("notall.f error varies between calls:\n %s\n %s", first, err.Error()) + } + } +} + +// TestPathKeyIsUnambiguous pins that the one shape which could collide cannot be +// written: a field literally named "a.b" and a field "a" holding "b" would both +// spell "a.b", so a dotted field name is rejected at New and a dot means a path +// wherever it appears. +func TestPathKeyIsUnambiguous(t *testing.T) { + dir := writeData(t, map[string]string{ + "cat": `[{"format":"{a.b}","a.b":"1"},{"format":"{a}","a":{"format":"{b}","b":"2"}}]`, + }) + _, err := New(WithoutShippedData(), WithDataPath(dir)) + if err == nil || !strings.Contains(err.Error(), `field "a.b" contains "."`) { + t.Fatalf("New = %v, want the dotted field name rejected", err) + } + // The same data without the dotted key is fine, and the path resolves. + f := newGenerator(t, writeData(t, map[string]string{ + "cat": `{"format":"{a.b}","a":{"format":"{b}","b":"2"}}`, + }), WithSeed(1)) + if !slices.Contains(f.List(), "cat.a.b") { + t.Error("List() omits cat.a.b, which the data carries") + } + if got := fake(t, f, "cat"); got != "2" { + t.Fatalf("cat = %q, want 2", got) + } +} + +// TestMissingFieldNamesItself keeps the precise diagnosis for the ordinary typo: a +// single-variant choice always picks the same item, so it needs no every-variant +// guard and the error can name the field that is missing. +func TestMissingFieldNamesItself(t *testing.T) { + f := newGenerator(t, "data/sv_SE", WithSeed(1)) + _, err := f.Fake("person.typo") + if err == nil || !strings.Contains(err.Error(), `no field "typo"`) { + t.Errorf("Fake(person.typo) = %v, want it to name the missing field", err) + } +} + +func TestBindingKeyIsNotAPathSegment(t *testing.T) { + dir := writeData(t, map[string]string{ + "color": `["red","blue"]`, + "name": `{"format":"{/color} {w}","w":"x"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + if slices.Contains(f.List(), "name./color") { + t.Error("List() advertises a binding key") + } + if _, err := f.Fake("name./color"); err == nil || !strings.Contains(err.Error(), `no field "/color"`) { + t.Errorf("Fake(name./color) = %v, want a no-field error", err) + } +} diff --git a/readme_test.go b/readme_test.go new file mode 100644 index 0000000..594fabc --- /dev/null +++ b/readme_test.go @@ -0,0 +1,61 @@ +package fejkdata + +import ( + "os" + "regexp" + "strings" + "testing" +) + +var jsonBlock = regexp.MustCompile("(?s)```json\n(.*?)```") + +func readme(t *testing.T) string { + t.Helper() + src, err := os.ReadFile("README.md") + if err != nil { + t.Fatal(err) + } + return string(src) +} + +func TestReadmeExamplesLoadAndRender(t *testing.T) { + n := 0 + for _, m := range jsonBlock.FindAllStringSubmatch(readme(t), -1) { + body := m[1] + if strings.Contains(body, "…") { + continue + } + f, err := New(WithDataPath(writeData(t, map[string]string{"example": body})), WithSeed(1)) + if err != nil { + t.Errorf("README example does not load: %v\n%s", err, body) + continue + } + for i := 0; i < 20; i++ { + if _, err := f.Fake("example"); err != nil { + t.Errorf("README example does not render: %v\n%s", err, body) + break + } + } + n++ + } + if n < 8 { + t.Fatalf("found only %d README examples", n) + } +} + +func TestReadmeSQLExampleOutput(t *testing.T) { + src := readme(t) + src = src[strings.Index(src, "### Your own data"):] + block := jsonBlock.FindStringSubmatch(src) + want := regexp.MustCompile(`(?m)^# (INSERT .*)$`).FindStringSubmatch(src) + if block == nil || want == nil { + t.Fatal("README lost the SQL example or its printed output") + } + f, err := New(WithDataPath(writeData(t, map[string]string{"sql": block[1]})), WithSeed(1)) + if err != nil { + t.Fatal(err) + } + if got := fake(t, f, "sql"); got != want[1] { + t.Errorf("README SQL example with seed 1 = %q, README prints %q", got, want[1]) + } +} diff --git a/reference.go b/reference.go index e449855..664521b 100644 --- a/reference.go +++ b/reference.go @@ -2,379 +2,176 @@ package fejkdata import ( "fmt" - "sort" "strings" ) -// refPrefix marks a {..path} token: a reference to a node elsewhere in the data -// root rather than a sibling field. The path is resolved across every loaded -// directory (see linkRefs), and is stricter than the one Fake takes: a reference -// binds one node, so it cannot step through a multi-variant choice even where Fake -// and List can. -const refPrefix = ".." +// A reference names a node by path rather than as a sibling field: {/a.b} from the +// data root, {.a} from the folder this file sits in, {..a} from the folder above. +func isRef(name string) bool { return strings.HasPrefix(name, ".") || strings.HasPrefix(name, "/") } -func isRef(name string) bool { return strings.HasPrefix(name, refPrefix) } +// refBinding is what a reference was bound to: the head key its category is held +// under, and the tail read into it. +type refBinding struct { + key string + tail []string +} -// linkRefs resolves every {..path} reference in the assembled tree, binding the -// target node into the referring template's fields under the token's key so the -// ordinary resolver renders it like a sibling. It runs once, after all data is -// merged, so a reference sees the final (override-resolved) tree. A path that is -// unknown, names a folder, or steps through a multi-variant choice fails here, -// keeping a bad reference a New-time error, never a random render-time one. +// refShape splits a reference into its sigil and the dotted path after it. +func refShape(name string) (sigil, rest string, err error) { + switch { + case strings.HasPrefix(name, "/"): + sigil, rest = "/", name[1:] + case strings.HasPrefix(name, ".."): + sigil, rest = "..", name[2:] + default: + sigil, rest = ".", name[1:] + } + if rest == "" { + return "", "", fmt.Errorf("reference has no path") + } + if strings.HasPrefix(rest, "/") { + return "", "", fmt.Errorf("the path after %s starts at a name, not a /; write {%s%s}", sigil, sigil, rest[1:]) + } + if strings.HasPrefix(rest, ".") { + return "", "", fmt.Errorf("a reference starts with / (the root), . (this folder) or .. (the folder above)") + } + for _, seg := range strings.Split(rest, ".") { + if seg == "" { + return "", "", fmt.Errorf("path has an empty segment") + } + } + return sigil, rest, nil +} + +// refSegments resolves a reference written in folder to a path from the root. +func refSegments(name string, folder []string) ([]string, error) { + sigil, rest, err := refShape(name) + if err != nil { + return nil, err + } + var base []string + switch sigil { + case ".": + base = folder + case "..": + if len(folder) == 0 { + return nil, fmt.Errorf("no folder above the root") + } + base = folder[:len(folder)-1] + } + return append(append([]string{}, base...), strings.Split(rest, ".")...), nil +} + +// linkRefs resolves every reference in the assembled tree. The head of the path — +// up to the category it names — is bound into the referring template's fields +// under its root path, and the rest reads into it the way a sibling path does, so +// a reference is held like a sibling and two spellings of one target are one +// draw. It runs once, after all data is merged, so a reference sees the final +// (override-resolved) tree. A path that is unknown, names a folder, or reads a +// field not every variant carries fails here, keeping a bad reference a New-time +// error, never a random render-time one. func linkRefs(root map[string]node) error { - return walkNodes(root, func(path string, n node) error { - t, ok := n.(*template) - if !ok { + return eachTemplate(root, func(folder []string, path string, t *template) error { + names := refTokens(t.format) + if len(names) == 0 { return nil } - for _, name := range refTokens(t.format) { - target, err := lookup(root, strings.Split(name[len(refPrefix):], ".")) + if t.fields == nil { + t.fields = map[string]node{} + } + t.refs = make(map[string]refBinding, len(names)) + for _, name := range names { + segments, err := refSegments(name, folder) if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } - t.fields[name] = target + head, target, tail, err := resolveRef(root, segments) + if err != nil { + return fmt.Errorf("%s: reference {%s}: %w", path, name, err) + } + key := "/" + strings.Join(head, ".") + if err := checkPath(target, tail, key); err != nil { + return fmt.Errorf("%s: reference {%s}: %w", path, name, err) + } + t.fields[key] = target + t.refs[name] = refBinding{key, tail} + } + if err := t.compileFormat(); err != nil { + return fmt.Errorf("%s: %w", path, err) } return nil }) } -// checkBoundLevelsHeld rejects every route to a held name except the ones that read -// its draw. An expansion holds one draw of that name; anything else that renders it -// draws again, and the two disagree. checkNoOverlap settles the spellings within one -// format (a token, a calc operand); this settles the rest — a reference, whether it -// sits in that format or in anything the format renders, however deep. -// -// It runs after checkNoCycles, whose guarantee is what lets the walk terminate. -func checkBoundLevelsHeld(root map[string]node) error { - return walkNodes(root, func(path string, n node) error { - t, ok := n.(*template) - if !ok || len(t.held) == 0 { - return nil - } - heads := make([]string, 0, len(t.held)) - for head := range t.held { - heads = append(heads, head) - } - sort.Strings(heads) // so which overlap is reported does not vary - for _, head := range heads { - // What one draw answers for depends on how the draw is read. A path may - // read into anything the level contains; a calc renders its operand, so - // that draw fixes exactly the value the render produces. - held := map[node]bool{} - reader, isPath := t.bound[head] - if isPath { - cover(t.fields[head], held) - } else { - operandDraw(t.fields[head], held) +// eachTemplate calls fn once per template, with the folder its category sits in +// and the dot path reaching it, folders and names in sorted order. +func eachTemplate(root map[string]node, fn func(folder []string, path string, t *template) error) error { + var inCategory func(folder []string, path string, n node) error + inCategory = func(folder []string, path string, n node) error { + if t, ok := n.(*template); ok { + if err := fn(folder, path, t); err != nil { + return err } - if len(held) == 0 { - continue // an early out: a literal head holds nothing to reach - } - // One seen set across the edges: a node that cannot reach the level - // cannot reach it by another route either, so it is walked once here. - seen := map[node]bool{} - for _, e := range renderEdges(t) { - if splitArm(e.label).key == head { - continue // a token or operand reading this draw, the routes allowed - } - if renders(e.to, held, seen) { - if isPath { - return fmt.Errorf("%s: %s renders %q, which {%s} reads a path into; name the fields you want instead", path, e.reached(), head, reader) - } - return fmt.Errorf("%s: %s renders %q, which a {calc()} also reads; reach it one way so it is drawn once", path, e.reached(), head) - } - } - } - return nil - }) -} - -// cover collects what one held draw of a level answers for: the level and -// everything contained in it, since a path may read any of it. Literals are left -// out — one fixed string cannot disagree with itself, and a literal is a value, so -// two that spell the same text are indistinguishable. -func cover(n node, into map[node]bool) { - if _, fixed := n.(literal); fixed { - return - } - into[n] = true - for _, c := range contained(n) { - cover(c.node, into) - } -} - -// operandDraw collects what one held draw of a {calc()} operand answers for: the -// operand and what rendering it settles inside itself. A calc renders its operand -// whole, so that draw fixes every value the render produced, and a second route to -// any of them disagrees with it. -// -// The walk stops at a {..path} edge, which is where the operand's own value ends -// and a shared source begins: two names referencing one category are two draws, the -// same rule {word} {word} follows. cover stops there too, by way of named, so both -// halves of the fence end at the same boundary. A literal is left out for the -// reason cover leaves one out. -func operandDraw(n node, into map[node]bool) { - if _, fixed := n.(literal); fixed { - return - } - if into[n] { - return - } - into[n] = true - for _, e := range renderEdges(n) { - if isRef(e.label) { - continue - } - operandDraw(e.to, into) - } -} - -// renders reports whether rendering n can reach anything in want, following the -// same edges expand does. seen keeps a node shared by several routes from being -// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk -// ends. -func renders(n node, want, seen map[node]bool) bool { - if want[n] { - return true - } - if seen[n] { - return false - } - seen[n] = true - for _, e := range renderEdges(n) { - if renders(e.to, want, seen) { - return true - } - } - return false -} - -// walkNodes calls fn once per contained node, passing the dot path that reaches it, -// visiting keys in sorted order so which of several broken nodes gets reported does -// not depend on map iteration. -func walkNodes(root map[string]node, fn func(path string, n node) error) error { - seen := map[node]bool{} - var visit func(string, node) error - visit = func(path string, n node) error { - if n == nil || seen[n] { - return nil - } - seen[n] = true - if err := fn(path, n); err != nil { - return err } for _, c := range contained(n) { - if err := visit(join(path, c.name), c.node); err != nil { + if err := inCategory(folder, join(path, c.name), c.node); err != nil { return err } } return nil } - for _, name := range sortedNames(root) { - if err := visit(name, root[name]); err != nil { - return err + var inFolder func(folder []string, children map[string]node) error + inFolder = func(folder []string, children map[string]node) error { + for _, name := range sortedNames(children) { + path := join(strings.Join(folder, "."), name) + if g, ok := children[name].(*group); ok { + if err := inFolder(append(folder[:len(folder):len(folder)], name), g.children); err != nil { + return err + } + continue + } + if err := inCategory(folder, path, children[name]); err != nil { + return err + } } - } - return nil -} - -// namedNode is a contained child and the segment reaching it; a choice's items carry -// no segment, matching how a dot path steps over a choice. -type namedNode struct { - name string - node node -} - -func contained(n node) []namedNode { - switch n := n.(type) { - case *group: - return named(n.children) - case *choice: - out := make([]namedNode, len(n.items)) - for i, it := range n.items { - out[i] = namedNode{node: it} - } - return out - case *template: - return named(n.fields) - default: return nil } + return inFolder(nil, root) } -// named skips a bound {..path} key: it is a render edge, not containment, so using -// it as a path segment would report a node under a path that does not reach it. Only -// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a -// group's children never carry the prefix — so this one skip serves both. -func named(m map[string]node) []namedNode { - out := make([]namedNode, 0, len(m)) - for _, name := range sortedNames(m) { - if isRef(name) { - continue - } - out = append(out, namedNode{name: name, node: m[name]}) - } - return out -} - -func sortedNames(m map[string]node) []string { - names := make([]string, 0, len(m)) - for name := range m { - names = append(names, name) - } - sort.Strings(names) - return names -} - -// lookup finds the single node a reference path names, walking groups and -// template fields by segment and descending a single-variant choice as a -// transparent wrapper. A missing segment, a folder target, or a step through a -// multi-variant choice (which has no one value to bind) is an error. -func lookup(root map[string]node, segments []string) (node, error) { +// resolveRef walks a reference path through the folders to the category it names, +// returning that head, the node, and the tail left to read into it. A descent of +// its own rather than a walkPath: it walks groups only and returns where they end, +// not a leaf. +func resolveRef(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { var n node = &group{children: root} - for i := 0; i < len(segments); i++ { - switch c := n.(type) { - case *group: - child, ok := c.children[segments[i]] - if !ok { - return nil, fmt.Errorf("no entry %q", segments[i]) - } - n = child - case *template: - child, ok := c.fields[segments[i]] - if !ok { - return nil, fmt.Errorf("no field %q", segments[i]) - } - n = child - case *choice: - if len(c.items) != 1 { - return nil, fmt.Errorf("%q steps through a %d-way choice", segments[i], len(c.items)) - } - n, i = c.items[0], i-1 // a choice consumes no segment; reprocess it unwrapped - default: - return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[i]) + i := 0 + for ; i < len(segments); i++ { + g, ok := n.(*group) + if !ok { + break } + child, ok := g.children[segments[i]] + if !ok { + return nil, nil, nil, fmt.Errorf("no entry %q", segments[i]) + } + n = child } if _, ok := n.(*group); ok { - return nil, fmt.Errorf("names a folder, not a value") + return nil, nil, nil, fmt.Errorf("names a folder, not a value") } - return n, nil + return segments[:i], n, segments[i:], nil } -// refTokens returns just the {..path} reference names among a format's field -// tokens (linkRefs binds each into the template's fields). +// refTokens returns the reference names a format reads, as tokens or as operands. func refTokens(format string) []string { var refs []string - for _, name := range fieldTokens(format) { - if isRef(name) { + seen := map[string]bool{} + for _, name := range append(fieldTokens(format), operandTokens(format)...) { + if isRef(name) && !seen[name] { + seen[name] = true refs = append(refs, name) } } return refs } - -// renderEdge is a child a node renders into, labelled by what reaches it (a field -// name, reference, or choice index) for a readable cycle report. operand marks a -// label that is a {calc()} operand name rather than a token, so an error can name -// it the way the author wrote it. -type renderEdge struct { - to node - label string - operand bool -} - -// reached names an edge as the author spelled it, the vocabulary boundReaders uses -// for the sibling fence. -func (e renderEdge) reached() string { - if e.operand { - return fmt.Sprintf("calc operand %q", e.label) - } - return "{" + e.label + "}" -} - -// renderEdges lists the children rendering n recurses into, mirroring expand: a -// choice's items, and a template's field/reference tokens plus its calc operands. -// A literal or group renders nothing, so it has no edges. -func renderEdges(n node) []renderEdge { - switch n := n.(type) { - case *choice: - es := make([]renderEdge, len(n.items)) - for i, it := range n.items { - es[i] = renderEdge{to: it, label: fmt.Sprintf("[%d]", i)} - } - return es - case *template: - var es []renderEdge - add := func(name string, operand bool) { - a := splitArm(name) - c, ok := n.fields[a.key] - if !ok { - return - } - for _, leaf := range pathLeaves(c, a.tail) { - es = append(es, renderEdge{leaf, name, operand}) - } - } - for _, name := range fieldTokens(n.format) { - add(name, false) - } - for _, name := range calcOperands(n.format) { - add(name, true) - } - return es - default: - return nil - } -} - -// pathLeaves lists what a token's dotted tail renders. A path draws the levels it -// passes through but renders only what it lands on, so the leaf is the edge — a -// bare token, whose tail is empty, lands on the field itself. A choice on the way -// contributes every variant, since any of them may be the one drawn. checkPath has -// already proved the tail resolves in every variant, so the walk drops nothing. -func pathLeaves(n node, tail []string) []node { - if len(tail) == 0 { - return []node{n} - } - if c, ok := n.(*choice); ok { - var out []node - for _, it := range c.items { - out = append(out, pathLeaves(it, tail)...) - } - return out - } - return pathLeaves(child(n, tail[0]), tail[1:]) -} - -// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself -// — directly, mutually, or through a chain — never terminates, so it must fail at -// New rather than stack-overflow at render. It is a depth-first walk of the render -// graph (renderEdges); grey marks nodes on the current path so a back-edge to one -// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped. -// Every node is a root: a field its parent's format never renders is still reachable -// by dot path, so a cycle in one would otherwise reach render and be fatal there. -func checkNoCycles(root map[string]node) error { - const ( - grey = 1 - black = 2 - ) - color := map[node]int{} - var visit func(n node, path string) error - visit = func(n node, path string) error { - switch color[n] { - case grey: - return fmt.Errorf("reference cycle: %s", path) - case black: - return nil - } - color[n] = grey - for _, e := range renderEdges(n) { - if err := visit(e.to, path+" -> "+e.label); err != nil { - return err - } - } - color[n] = black - return nil - } - return walkNodes(root, func(path string, n node) error { return visit(n, path) }) -} diff --git a/reference_test.go b/reference_test.go index 62ceada..65494d5 100644 --- a/reference_test.go +++ b/reference_test.go @@ -1,13 +1,16 @@ package fejkdata -import "testing" +import ( + "strings" + "testing" +) // TestRootReferenceAcrossFolders is the headline case: a category in one folder -// pulls a value from another via a {..path} reference resolved from the data root. +// pulls a value from another via a {/path} reference resolved from the data root. func TestRootReferenceAcrossFolders(t *testing.T) { dir := writeData(t, map[string]string{ - "en_US/person": `["Pat Smith"]`, - "sv_SE/greeting": `{"format":"Hej, {..en_US.person}!"}`, + "en_US/person": `"Pat Smith"`, + "sv_SE/greeting": `"Hej, {/en_US.person}!"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "sv_SE.greeting"); got != "Hej, Pat Smith!" { @@ -15,12 +18,11 @@ func TestRootReferenceAcrossFolders(t *testing.T) { } } -// TestReferenceIntoAField reaches a field inside a referenced category, crossing -// a single-variant choice and then a template field (..who.last). +// TestReferenceIntoAField reaches a field inside a referenced category. func TestReferenceIntoAField(t *testing.T) { dir := writeData(t, map[string]string{ - "who": `[{"format":"{first} {last}","first":["Ada"],"last":["Byron"]}]`, - "card": `{"format":"signed {..who.last}"}`, + "who": `{"format":"{first} {last}","first":"Ada","last":"Byron"}`, + "card": `"signed {/who.last}"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "card"); got != "signed Byron" { @@ -28,12 +30,12 @@ func TestReferenceIntoAField(t *testing.T) { } } -// TestReferenceInAlternation lets a reference stand as one arm of a {a|..b} +// TestReferenceInAlternation lets a reference stand as one arm of a {a|/b} // alternation, so a field and a cross-file value share one slot. func TestReferenceInAlternation(t *testing.T) { dir := writeData(t, map[string]string{ - "far": `["X"]`, - "near": `{"format":"{here|..far}","here":["H"]}`, + "far": `"X"`, + "near": `{"format":"{here|/far}","here":"H"}`, }) f := newGenerator(t, dir, WithSeed(2)) seen := map[string]bool{} @@ -48,8 +50,8 @@ func TestReferenceInAlternation(t *testing.T) { // TestReferenceCombinesLoadedPaths is the point of references over the merge // model: data layered from two dirs can point at each other through the root. func TestReferenceCombinesLoadedPaths(t *testing.T) { - a := writeData(t, map[string]string{"en_US/word": `["river"]`}) - b := writeData(t, map[string]string{"mine/slug": `{"format":"the-{..en_US.word}"}`}) + a := writeData(t, map[string]string{"en_US/word": `"river"`}) + b := writeData(t, map[string]string{"mine/slug": `"the-{/en_US.word}"`}) f := newGeneratorN(t, []string{a, b}, WithSeed(1)) if got := fake(t, f, "mine.slug"); got != "the-river" { t.Fatalf("slug = %q, want the-river", got) @@ -60,9 +62,9 @@ func TestReferenceCombinesLoadedPaths(t *testing.T) { // linking order cannot matter. func TestReferenceChain(t *testing.T) { dir := writeData(t, map[string]string{ - "a": `{"format":"{..b}"}`, - "b": `{"format":"{..c}"}`, - "c": `["deep"]`, + "a": `"{/b}"`, + "b": `"{/c}"`, + "c": `"deep"`, }) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "a"); got != "deep" { @@ -74,40 +76,40 @@ func TestReferenceChain(t *testing.T) { // fails at load, never at a random render. func TestReferenceErrors(t *testing.T) { cases := map[string]map[string]string{ - "missing target": {"card": `{"format":"{..nope.gone}"}`}, - "folder target": {"en_US/word": `["w"]`, "card": `{"format":"{..en_US}"}`}, - "multi-variant on the path": { - "who": `[{"format":"{f}","f":["1"]},{"format":"{f}","f":["2"]}]`, - "card": `{"format":"{..who.f}"}`, + "missing target": {"card": `"{/nope.gone}"`}, + "folder target": {"en_US/word": `"w"`, "card": `"{/en_US}"`}, + "a variant on the path lacks the field": { + "who": `[{"format":"{f}","f":"1"},{"format":"{g}","g":"2"}]`, + "card": `"{/who.f}"`, }, - "empty reference path": {"card": `{"format":"{..}"}`}, + "empty reference path": {"card": `"{/}"`}, // A reference that leads back to its own value never terminates at render, // so New must reject the cycle up front (direct, mutual, or chained). - "direct cycle": {"a": `{"format":"x{..a}"}`}, - "mutual cycle": {"a": `{"format":"{..b}"}`, "b": `{"format":"{..a}"}`}, - "chain cycle": {"a": `{"format":"{..b}"}`, "b": `{"format":"{..c}"}`, "c": `{"format":"{..a}"}`}, + "direct cycle": {"a": `"x{/a}"`}, + "mutual cycle": {"a": `"{/b}"`, "b": `"{/a}"`}, + "chain cycle": {"a": `"{/b}"`, "b": `"{/c}"`, "c": `"{/a}"`}, // calc renders its operands, so a cycle through one must be caught too. - "calc operand cycle": {"x": `{"format":"{calc(y)}","y":[{"format":"{..x}"}]}`}, + "calc operand cycle": {"x": `{"format":"{calc(y)}","y":"{/x}"}`}, // A field its parent's format never renders is still reachable by dot path, // so a cycle hiding in one must fail at New rather than at render. - "cycle in an unrendered field": {"cat": `{"format":"hi","x":{"format":"{..cat.x}"}}`}, + "cycle in an unrendered field": {"cat": `{"format":"hi","x":"{/cat.x}"}`}, "mutual cycle between unrendered fields": { - "cat": `{"format":"hi","x":{"format":"{..cat.y}"},"y":{"format":"{..cat.x}"}}`, + "cat": `{"format":"hi","x":"{/cat.y}","y":"{/cat.x}"}`, }, "cycle in an unrendered field of a choice arm": { - "cat": `[{"format":"hi","x":{"format":"{..cat.x}"}}]`, + "cat": `{"format":"hi","x":"{/cat.x}"}`, }, // The shipped layout puts categories in folders, so a cycle one level down // is the common case, not an edge case. - "cycle in a subfolder": {"sv_SE/a": `{"format":"x{..sv_SE.a}"}`}, - "mutual cycle within a subfolder": {"sv_SE/a": `{"format":"{..sv_SE.b}"}`, "sv_SE/b": `{"format":"{..sv_SE.a}"}`}, - "mutual cycle across two folders": {"en_US/a": `{"format":"{..sv_SE.b}"}`, "sv_SE/b": `{"format":"{..en_US.a}"}`}, + "cycle in a subfolder": {"sv_SE/a": `"x{/sv_SE.a}"`}, + "mutual cycle within a subfolder": {"sv_SE/a": `"{/sv_SE.b}"`, "sv_SE/b": `"{/sv_SE.a}"`}, + "mutual cycle across two folders": {"en_US/a": `"{/sv_SE.b}"`, "sv_SE/b": `"{/en_US.a}"`}, // ".." is reserved for bound references, so an authored key using it would // name a node nothing can reach and nothing would validate. - "field key using the reference prefix": {"cat": `{"format":"hi","..x":{"format":"{..nope}"}}`}, + "field key using the reference prefix": {"cat": `{"format":"hi","..x":"{/nope}"}`}, } for name, files := range cases { - if _, err := New([]string{writeData(t, files)}); err == nil { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { t.Errorf("%s: New = nil error, want a reference error", name) } } @@ -119,10 +121,10 @@ func TestReferenceErrors(t *testing.T) { // starts with "." — it is skipped, not rejected. func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { f := newGenerator(t, writeData(t, map[string]string{ - "sv_SE/ok": `["fine"]`, - "sv_SE/..bad": `{"format":"{..nope}"}`, - "sv_SE/..y/ct": `{"format":"{..nope}"}`, - ".git/config": `["not data"]`, + "sv_SE/ok": `"fine"`, + "sv_SE/..bad": `"{/nope}"`, + "sv_SE/..y/ct": `"{/nope}"`, + ".git/config": `"not data"`, }), WithSeed(1)) if got := f.List(); len(got) != 1 || got[0] != "sv_SE.ok" { t.Fatalf("List() = %v, want only sv_SE.ok", got) @@ -133,7 +135,7 @@ func TestDotPrefixedDataEntriesAreSkipped(t *testing.T) { // over-rejecting: a field the format never renders may point back at its own // category, which terminates, and stays renderable by path. func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { - dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":{"format":"see {..cat}"}}`}) + dir := writeData(t, map[string]string{"cat": `{"format":"hi","x":"see {/cat}"}`}) f := newGenerator(t, dir, WithSeed(1)) if got := fake(t, f, "cat"); got != "hi" { t.Fatalf("cat = %q, want hi", got) @@ -149,9 +151,9 @@ func TestReferenceFromUnrenderedFieldTerminates(t *testing.T) { func TestNewErrorIsDeterministic(t *testing.T) { cases := map[string]map[string]string{ "three bad references": { - "a": `{"format":"{..nope.one}"}`, - "b": `{"format":"{..nope.two}"}`, - "c": `{"format":"{..nope.three}"}`, + "a": `"{/nope.one}"`, + "b": `"{/nope.two}"`, + "c": `"{/nope.three}"`, }, "two bad fields in one template": { "cat": `{"format":"hi","aaa":{"no":1},"zzz":{"no":2}}`, @@ -161,7 +163,7 @@ func TestNewErrorIsDeterministic(t *testing.T) { dir := writeData(t, files) var first string for i := 0; i < 50; i++ { - _, err := New([]string{dir}) + _, err := New(WithoutShippedData(), WithDataPath(dir)) if err == nil { t.Fatalf("%s: New = nil error, want a load error", name) } @@ -178,7 +180,7 @@ func TestNewErrorIsDeterministic(t *testing.T) { } // TestNewErrorPathIsCanonical pins the node path a load error names: a choice arm -// adds no segment, and a bound {..path} reference is not a containment segment at +// adds no segment, and a bound {/path} reference is not a containment segment at // all, so a bad reference is reported against the node that holds it. func TestNewErrorPathIsCanonical(t *testing.T) { cases := []struct { @@ -188,17 +190,17 @@ func TestNewErrorPathIsCanonical(t *testing.T) { }{ { "cycle inside a choice arm", - map[string]string{"cat": `[{"format":"hi","x":{"format":"{..cat.x}"}}]`}, - "fejkdata: reference cycle: cat.x -> ..cat.x", + map[string]string{"cat": `{"format":"hi","x":"{/cat.x}"}`}, + "fejkdata: reference cycle: cat.x -> /cat.x", }, { "bad reference reached through another reference", - map[string]string{"a": `{"format":"{..b}"}`, "b": `{"format":"{..nope}"}`}, - `fejkdata: b: reference {..nope}: no entry "nope"`, + map[string]string{"a": `"{/b}"`, "b": `"{/nope}"`}, + `fejkdata: b: reference {/nope}: no entry "nope"`, }, } for _, c := range cases { - _, err := New([]string{writeData(t, c.files)}) + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files))) if err == nil { t.Errorf("%s: New = nil error", c.name) continue @@ -208,3 +210,139 @@ func TestNewErrorPathIsCanonical(t *testing.T) { } } } + +func TestReferencePathIsHeld(t *testing.T) { + dir := writeData(t, map[string]string{ + "person": `[{"format":"{first} {last}","first":"Anna","last":"Andersson"},{"format":"{first} {last}","first":"Bo","last":"Berg"}]`, + "card": `"{/person.first} {/person.last}"`, + }) + f := newGenerator(t, dir, WithSeed(3)) + seen := map[string]bool{} + for i := 0; i < 50; i++ { + got := fake(t, f, "card") + if got != "Anna Andersson" && got != "Bo Berg" { + t.Fatalf("card = %q, want one person's first and last name", got) + } + seen[got] = true + } + if len(seen) != 2 { + t.Fatalf("card only ever rendered %v", seen) + } +} + +func TestReferenceThroughChoiceNeedsEveryVariant(t *testing.T) { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "who": `[{"format":"{f}{h}","f":"1","h":"x"},{"format":"{g}{h}","g":"2","h":"y"}]`, + "card": `"{/who.f}"`, + }))) + if err == nil || !strings.Contains(err.Error(), "not every variant") { + t.Fatalf("New = %v, want the missing variant named", err) + } +} + +func TestBareReferenceDrawsEachTime(t *testing.T) { + dir := writeData(t, map[string]string{ + "die": `["1","2","3","4","5","6"]`, + "roll": `"{/die} {/die}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "roll"); got[0] != got[2] { + return + } + } + t.Fatal("two bare references always agreed; each should be its own draw") +} + +func TestReferenceOverlapIsRejected(t *testing.T) { + for name, file := range map[string]string{ + "head beside a path": `{"format":"{/cat.p} {/cat.p.first}","p":[{"format":"{first}","first":"A"},{"format":"{first}","first":"B"}]}`, + "sibling path beside a reference path": `{"format":"{p.first} {/cat.p.last}","p":[{"format":"{first}","first":"A","last":"1"},{"format":"{first}","first":"B","last":"2"}]}`, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file}))) + if err == nil || !strings.Contains(err.Error(), "reads a path into") { + t.Errorf("%s: New = %v, want the overlap rejected", name, err) + } + } +} + +func TestRelativeReferences(t *testing.T) { + dir := writeData(t, map[string]string{ + "sv_SE/username": `"bob"`, + "sv_SE/email": `"{.username}@example.com"`, + "sv_SE/deep/card": `"{..username} via {/sv_SE.username}"`, + "top": `"{.sv_SE.username}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for path, want := range map[string]string{ + "sv_SE.email": "bob@example.com", + "sv_SE.deep.card": "bob via bob", + "top": "bob", + } { + if got := fake(t, f, path); got != want { + t.Errorf("%s = %q, want %q", path, got, want) + } + } +} + +func TestRelativeAndRootSpellingsBindOneDraw(t *testing.T) { + dir := writeData(t, map[string]string{ + "sv_SE/person": `[{"format":"{first} {last}","first":"Anna","last":"Andersson"},{"format":"{first} {last}","first":"Bo","last":"Berg"}]`, + "sv_SE/card": `"{.person.first} {/sv_SE.person.last}"`, + }) + f := newGenerator(t, dir, WithSeed(3)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "sv_SE.card"); got != "Anna Andersson" && got != "Bo Berg" { + t.Fatalf("card = %q, want both spellings to read one person", got) + } + } +} + +func TestReferenceSigilErrors(t *testing.T) { + for name, files := range map[string]map[string]string{ + "no folder above the root": {"a": `"{..b}"`, "b": `"x"`}, + "three dots": {"a": `"{...b}"`, "b": `"x"`}, + "root sigil alone": {"a": `"{/}"`}, + "dot alone": {"a": `"{.}"`}, + "missing sibling": {"sv_SE/a": `"{.nope}"`}, + "slash in a field name": {"a": `{"format":"{x}","x":"1","a/b":"2"}`}, + } { + if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err == nil { + t.Errorf("%s: New = nil error, want a reference error", name) + } + } +} + +func TestSpellingsOfOneReferenceAreOneLevel(t *testing.T) { + person := `[{"format":"{first} {last}","first":"Ada","last":"Byron"},{"format":"{first} {last}","first":"Bo","last":"Ek"}]` + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ + "sv_SE/person": person, + "sv_SE/mail": `"{.person} <{/sv_SE.person.first}>"`, + }))) + if err == nil || !strings.Contains(err.Error(), "reads a path into") || !strings.Contains(err.Error(), "{.person}") { + t.Errorf("New = %v, want the bare spelling rejected beside the path spelling", err) + } + dir := writeData(t, map[string]string{ + "sv_SE/word": `["alpha","beta","gamma"]`, + "sv_SE/loud": `"{/sv_SE.word} {uppercase(.word)}"`, + }) + f := newGenerator(t, dir, WithSeed(2)) + for i := 0; i < 50; i++ { + got := strings.Fields(fake(t, f, "sv_SE.loud")) + if len(got) != 2 || strings.ToUpper(got[0]) != got[1] { + t.Fatalf("loud = %q, want one draw under both spellings", got) + } + } +} + +func TestSlashAfterAReferenceSigilIsRejected(t *testing.T) { + for name, files := range map[string]map[string]string{ + "after ..": {"sv_SE/person": `"Ada"`, "sv_SE/deep/a": `"{../person}"`}, + "after .": {"sv_SE/person": `"Ada"`, "sv_SE/a": `"{./person}"`}, + } { + _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))) + if err == nil || !strings.Contains(err.Error(), "person}") || !strings.Contains(err.Error(), "write {") { + t.Errorf("%s: New = %v, want the slash rejected naming the spelling", name, err) + } + } +} diff --git a/render.go b/render.go index d1985c2..2d998fb 100644 --- a/render.go +++ b/render.go @@ -18,6 +18,8 @@ type rng interface { // e.g. "sv_SE.address" or "sv_SE.address.street". Choices along the way are // resolved at random. A path naming a folder (no value of its own) is an error. func (f *Generator) Fake(path string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) if err != nil { return "", fmt.Errorf("fejkdata: %s: %w", path, err) @@ -31,64 +33,33 @@ func (f *Generator) Fake(path string) (string, error) { // descend walks named fields to the node a path names. It is the one render-side // step that can fail, because the path comes from the caller and may name a field // that does not exist. A choice consumes no segment, so the rest of the path must -// be one every variant carries (the set compile stored) before a variant is picked -// — a path that resolves at all resolves on every call. -func descend(s *session, n node, segments []string) (node, error) { - if len(segments) == 0 { - return n, nil - } - switch n := n.(type) { - case *group: - child, ok := n.children[segments[0]] - if !ok { - return nil, fmt.Errorf("no entry %q", segments[0]) - } - return descend(s, child, segments[1:]) - case *template: - child, ok := n.fields[segments[0]] - if !ok { - return nil, fmt.Errorf("no field %q", segments[0]) - } - return descend(s, child, segments[1:]) - case *choice: - if len(n.items) > 1 { - if want := strings.Join(segments, "."); !n.shared[want] { - return nil, unreachableInChoice(n, want) +// be one every variant carries before a variant is picked — a path that resolves +// at all resolves on every call. +func descend(s *session, root node, segments []string) (node, error) { + var found node + err := walkPath(root, segments, pathWalk{ + choice: func(c *choice, rest []string) ([]node, error) { + if err := carriedByAll(c, rest); err != nil { + return nil, err } - } - return descend(s, pick(s, n), segments) - case literal: - return nil, fmt.Errorf("a plain string has no field %q", segments[0]) - default: - return nil, fmt.Errorf("cannot descend into %T at %q", n, segments[0]) - } -} - -// unreachableInChoice reports that a path cannot step through this choice, listing -// what every variant does carry. It reads the precomputed set, so a failing path -// costs no more than a rendering one. -func unreachableInChoice(c *choice, want string) error { - if len(c.shared) == 0 { - return fmt.Errorf("no variant of this %d-way choice carries %q", len(c.items), want) - } - offered := make([]string, 0, len(c.shared)) - for p := range c.shared { - offered = append(offered, p) - } - sort.Strings(offered) - return fmt.Errorf("not every variant of this %d-way choice carries %q; all carry %v", len(c.items), want, offered) + return []node{pick(s, c)}, nil + }, + leaf: func(n node) error { found = n; return nil }, + }) + return found, err } // render evaluates a compiled node to a string. compile validates every node up // front, so rendering a compiled tree cannot fail. func render(s *session, n node) string { switch n := n.(type) { - case literal: - return string(n) case *choice: return render(s, pick(s, n)) case *template: if n.repeat == 1 { + if n.fixed { + return n.lit + } return expand(s, n) } var b strings.Builder @@ -117,21 +88,6 @@ func pick(r rng, c *choice) node { return c.items[i] } -// classChar randomises one character-class rune: '0' digit 0-9, '1' digit 1-9, -// 'A' letter A-Z, 'a' letter a-z. -func classChar(s *session, c rune) byte { - switch c { - case '0': - return byte('0' + s.IntN(10)) - case '1': - return byte('1' + s.IntN(9)) - case 'A': - return byte('A' + s.IntN(26)) - default: // 'a' - return byte('a' + s.IntN(26)) - } -} - // expand renders a template's compiled ops. compile validated every token, so this // cannot fail. func expand(s *session, t *template) string { @@ -151,8 +107,6 @@ func expand(s *session, t *template) string { switch o.kind { case 'l': b.WriteString(o.lit) - case 'c': - b.WriteByte(classChar(s, o.r)) case 'f': b.WriteString(readField(s, t, held, o.arms[s.IntN(len(o.arms))])) case 'b': @@ -161,8 +115,8 @@ func expand(s *session, t *template) string { var operands []string if len(o.operands) > 0 { operands = make([]string, len(o.operands)) - for j, name := range o.operands { - operands[j] = readField(s, t, held, arm{name: name, key: name}) + for j, a := range o.operands { + operands[j] = readField(s, t, held, a) } } b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far @@ -170,76 +124,3 @@ func expand(s *session, t *template) string { } return b.String() } - -// draws is what an expansion has already drawn for its held names: the variant each -// was drawn as, so every path under it reads one row, and the value each read, so -// the same name read twice reads one value. -type draws struct { - variant map[string]node - value map[string]string -} - -// readField renders one arm of a token. An arm's key is a sibling field or a -// {..path} reference, which linkRefs bound into fields too. A name the expansion -// holds — a level some token addresses by dotted path, or a sibling a {calc()} -// reads — is drawn once and kept, so {place.postal-code} and {place.locality} read -// one row, either read twice gives one value, and a shown operand is the operand -// computed. Every other name is drawn afresh, so {word} {word} still draws twice. -// checkTokens, checkPath and linkRefs prove every step, so this cannot fail. -func readField(s *session, t *template, held *draws, a arm) string { - if !t.held[a.key] { - return render(s, t.fields[a.key]) - } - if v, read := held.value[a.name]; read { - return v - } - n, drew := held.variant[a.key] - if !drew { - n = drawn(s, t.fields[a.key]) - held.variant[a.key] = n - } - // Hold the draw at every level passed through, so two paths sharing a prefix - // share it. - for i, seg := range a.tail { - if i < len(a.steps) { - step, drew := held.variant[a.steps[i]] - if !drew { - step = drawn(s, child(n, seg)) - held.variant[a.steps[i]] = step - } - n = step - continue - } - n = child(n, seg) - } - v := render(s, n) - held.value[a.name] = v - return v -} - -// child is the node one path segment names below an already-drawn node. It holds -// while checkPath and the set a choice shares (see sharedPaths) agree with this -// walk: both prove the segment exists and that drawn leaves a template here. Each -// way that can break panics naming the segment, so a slip in that agreement -// reports where it happened rather than surfacing a nil node a level later. -func child(n node, seg string) node { - t, ok := n.(*template) - if !ok { - panic(fmt.Sprintf("fejkdata: %q under %T, which carries no fields", seg, n)) - } - c, ok := t.fields[seg] - if !ok { - panic(fmt.Sprintf("fejkdata: no field %q under a drawn level", seg)) - } - return c -} - -// drawn resolves a choice to one variant, so a bound head is a concrete node the -// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not -// another set to pick from. -func drawn(s *session, n node) node { - for c, ok := n.(*choice); ok; c, ok = n.(*choice) { - n = pick(s, c) - } - return n -} diff --git a/shipped_data_test.go b/shipped_data_test.go new file mode 100644 index 0000000..be776c1 --- /dev/null +++ b/shipped_data_test.go @@ -0,0 +1,115 @@ +package fejkdata + +import ( + "errors" + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/fstest" +) + +func TestNewDefaultsToShippedData(t *testing.T) { + f, err := New(WithSeed(1)) + if err != nil { + t.Fatalf("New() = %v", err) + } + paths := f.List() + for _, p := range []string{"sv_SE.person", "en_US.address", "misc.uuid"} { + if !slices.Contains(paths, p) { + t.Errorf("List() omits shipped %q", p) + } + } + if got := fake(t, f, "sv_SE.person"); got == "" { + t.Error("sv_SE.person rendered empty") + } +} + +func TestWithDataPathLayersOverShipped(t *testing.T) { + dir := writeData(t, map[string]string{"sv_SE/word": `"only-mine"`, "greeting": `"hej"`}) + f, err := New(WithDataPath(dir), WithSeed(1)) + if err != nil { + t.Fatalf("New = %v", err) + } + if got := fake(t, f, "sv_SE.word"); got != "only-mine" { + t.Errorf("sv_SE.word = %q, want the layered file to win", got) + } + if got := fake(t, f, "sv_SE.person"); got == "" { + t.Error("sv_SE.person should still come from the shipped data") + } + if got := fake(t, f, "greeting"); got != "hej" { + t.Errorf("greeting = %q, want hej", got) + } +} + +func TestUserDataMayReferenceShipped(t *testing.T) { + dir := writeData(t, map[string]string{"greeting": `"Hej {/sv_SE.person}!"`}) + f, err := New(WithDataPath(dir), WithSeed(1)) + if err != nil { + t.Fatalf("New = %v", err) + } + if got := fake(t, f, "greeting"); !strings.HasPrefix(got, "Hej ") || !strings.HasSuffix(got, "!") { + t.Errorf("greeting = %q", got) + } +} + +func TestWithoutShippedDataNeedsASource(t *testing.T) { + _, err := New(WithoutShippedData()) + if err == nil || !strings.Contains(err.Error(), "WithDataPath") || !errors.Is(err, ErrNoData) { + t.Fatalf("New(WithoutShippedData()) = %v, want ErrNoData naming WithDataPath", err) + } +} + +func TestWithoutShippedDataListsOnlyOwn(t *testing.T) { + dir := writeData(t, map[string]string{"greeting": `"hej"`}) + f := newGenerator(t, dir, WithSeed(1)) + if got := f.List(); !reflect.DeepEqual(got, []string{"greeting"}) { + t.Errorf("List() = %v, want only the loaded dir", got) + } +} + +func TestWithDataFS(t *testing.T) { + fsys := fstest.MapFS{ + "greeting.json": {Data: []byte(`"hej"`)}, + "nested/x.json": {Data: []byte(`{"format":"{y}","y":"z"}`)}, + ".hidden.json": {Data: []byte(`"ignored"`)}, + "broken/no.json": {Data: []byte(`"x"`)}, + } + f, err := New(WithoutShippedData(), WithDataFS(fsys), WithSeed(1)) + if err != nil { + t.Fatalf("New(WithDataFS) = %v", err) + } + if got := fake(t, f, "greeting"); got != "hej" { + t.Errorf("greeting = %q, want hej", got) + } + if got := fake(t, f, "nested.x.y"); got != "z" { + t.Errorf("nested.x.y = %q, want z", got) + } + bad := fstest.MapFS{"broken.json": {Data: []byte(`{ not json`)}} + if _, err := New(WithoutShippedData(), WithDataFS(bad)); err == nil || !strings.Contains(err.Error(), "broken.json") { + t.Errorf("New(bad fs) = %v, want an error naming the file", err) + } +} + +func TestFakeIsSafeForConcurrentUse(t *testing.T) { + f, err := New(WithSeed(1)) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + if _, err := f.Fake("sv_SE.person"); err != nil { + t.Error(err) + return + } + f.List() + } + }() + } + wg.Wait() +} diff --git a/template.go b/template.go index 2ca0fb6..32f5f38 100644 --- a/template.go +++ b/template.go @@ -2,45 +2,37 @@ package fejkdata import ( "fmt" - "sort" "strings" ) -// This file is the {token} grammar of a format string: how it is scanned -// (eachToken), how a function token is parsed (funcCall) and validated -// (checkFunc, checkTokens), and the builtin contract its functions implement. -// render.go evaluates these tokens; node.go compiles the surrounding JSON; -// reference.go binds {..path} tokens across the tree. - -// ftoken is one unit of a scanned format string: a literal rune to emit, a class -// char to randomise ('0' '1' 'A' 'a'), or the body of a {…} token. +// ftoken is one unit of a scanned format string: a literal rune or the body of a +// {…} token. type ftoken struct { - kind byte // 'l' literal rune, 'c' class char, 'b' brace body - r rune // for kinds 'l' and 'c' + kind byte // 'l' literal rune, 'b' brace body + r rune body string } // eachToken scans a format string once and calls fn for each unit, the single -// source of truth for how '#' escapes and {…} braces are read — compileOps, -// checkTokens, fieldTokens and refTokens all drive off it so the grammar can't -// drift between the validator and the renderer. '#' escapes the next char to a -// literal ("#0" -> '0', "##" -> '#'); a '{' must reach a '}' (else an error); an -// unmatched '}' is an ordinary literal. The error stops the scan early. +// source of truth for how braces are read: "{{" and "}}" are literal braces, a "{" +// opens a token that must reach its "}", and a lone "}" is an error. A table-shaped +// scanner, one case per rune kind, kept whole on purpose. func eachToken(format string, fn func(ftoken) error) error { rs := []rune(format) for i := 0; i < len(rs); i++ { var t ftoken switch c := rs[i]; c { - case '#': - t.kind, t.r = 'l', '#' - if i++; i < len(rs) { - t.r = rs[i] - } - case '0', '1', 'A', 'a': - t.kind, t.r = 'c', c case '{': + if i+1 < len(rs) && rs[i+1] == '{' { + t.kind, t.r = 'l', '{' + i++ + break + } end := i + 1 for end < len(rs) && rs[end] != '}' { + if rs[end] == '{' { + return fmt.Errorf("'{' inside a token in %q; a literal brace is written {{", format) + } end++ } if end >= len(rs) { @@ -48,6 +40,13 @@ func eachToken(format string, fn func(ftoken) error) error { } t.kind, t.body = 'b', string(rs[i+1:end]) i = end + case '}': + if i+1 < len(rs) && rs[i+1] == '}' { + t.kind, t.r = 'l', '}' + i++ + break + } + return fmt.Errorf("lone '}' in %q; a literal brace is written }}", format) default: t.kind, t.r = 'l', c } @@ -61,18 +60,18 @@ func eachToken(format string, fn func(ftoken) error) error { // builtin is a format-string function invoked as {name(args)}. It receives the // session (its rng, and the {seq()} counters), the output emitted so far in the // current expansion (for derivations such as a checksum over preceding digits), and -// the values of the operands it named (only calc names any). -// Almost all are pure over (rng, emitted, args) — no wall-clock, no crypto/rand — -// so seeding stays reproducible; a time-based id derives its time from the rng. -// seq is the one exception: it advances per-session counter state, which is itself -// deterministic (1, 2, 3 …). arity is the exact arg count, or -1 for variadic -// (then check does all the validation). The optional check validates args at -// compile time (their values, beyond the count). The registry lives in builtins.go. +// the values of the operands it named (only calc names any). All must stay pure +// over (rng, emitted, args) so seeded output is reproducible; seq advances +// per-session counter state, which is itself deterministic. arity is the exact arg +// count, or -1 for variadic (then check does all the validation). type builtin struct { arity int // prep parses validated args once, at compile time, into the closure expand calls. prep func(args []string) callFn check func(fields map[string]node, args []string) error + // operands names the fields the call reads, which expand renders for it; nil + // for a builtin that reads none. + operands func(args []string) []string } // funcCall splits a "{token}" body shaped name(args) into its parts; ok is false @@ -136,28 +135,14 @@ func checkTokens(format string, fields map[string]node) error { names := strings.Split(t.body, "|") for _, name := range names { if isRef(name) { - if name == refPrefix { - return fmt.Errorf("token {%s}: reference has no path", t.body) + if _, _, err := refShape(name); err != nil { + return fmt.Errorf("token {%s}: %w", t.body, err) } - continue // a root reference; its target is checked at New (see linkRefs) + continue // its target is checked at New (see linkRefs) } - a := splitArm(name) - if err := checkSegments(a); err != nil { + if err := checkArm(name, fields); err != nil { return fmt.Errorf("token {%s}: %w", t.body, err) } - head, ok := fields[a.key] - if !ok { - if a.key == "" { - return fmt.Errorf("token {%s}: a name is never empty, so this token can name no field", t.body) - } - if isOption(a.key) { - return fmt.Errorf("token {%s}: %q is an option and can never be a field", t.body, a.key) - } - return fmt.Errorf("token {%s}: no field %q", t.body, a.key) - } - if err := checkPath(head, a.tail, a.key); err != nil { - return fmt.Errorf("token {%s}: field %q: %w", t.body, a.key, err) - } } // Last, so an arm broken on its own terms is reported as that: a repeat is // the consequence of such a mistake, not the mistake itself. @@ -165,11 +150,57 @@ func checkTokens(format string, fields map[string]node) error { }) } -// checkNoRepeatedArm rejects {a|a|b}. An alternation picks its arms evenly, so a -// repeated one skews the odds — a third spelling of what weight is for, and one an -// author is far likelier to have typed by accident than meant. The error names the -// spelling that does skew a pick. It runs over the arms as written, so a repeated -// reference arm ({..a|..a}) is caught alongside a repeated sibling. +// checkArm validates one sibling name or path against a template's fields. +func checkArm(name string, fields map[string]node) error { + a := splitArm(name, nil) + if err := checkSegments(a); err != nil { + return err + } + head, ok := fields[a.key] + if !ok { + if a.key == "" { + return fmt.Errorf("a name is never empty, so this token can name no field") + } + if isOption(a.key) { + return fmt.Errorf("%q is an option and can never be a field", a.key) + } + return fmt.Errorf("no field %q", a.key) + } + if err := checkPath(head, a.tail, a.key); err != nil { + return fmt.Errorf("field %q: %w", a.key, err) + } + return nil +} + +// tokenOperands lists the fields one {token} body reads as operands, empty for a +// field token or a builtin that reads none. +func tokenOperands(body string) []string { + name, args, ok := funcCall(body) + if !ok { + return nil + } + b, known := builtins[name] + if !known || b.operands == nil { + return nil + } + return b.operands(args) +} + +// operandTokens lists every operand the builtins in a format read. +func operandTokens(format string) []string { + var names []string + _ = eachToken(format, func(t ftoken) error { + if t.kind == 'b' { + names = append(names, tokenOperands(t.body)...) + } + return nil + }) + return names +} + +// checkNoRepeatedArm rejects {a|a|b}: an alternation picks its arms evenly, so a +// repeated one is a second spelling of weight. The error names the spelling that +// does skew a pick. func checkNoRepeatedArm(body string, names []string) error { if len(names) < 2 { return nil @@ -198,88 +229,60 @@ func fieldTokens(format string) []string { return names } -// arm is one alternative of a {a|b} token, split into the key naming the node in -// a template's fields (a sibling field, or the whole "..path" string a reference is -// bound under) and the tail of a dotted path into it. A non-empty tail is what makes -// the arm a bound draw: its head is drawn once per expansion (see compileOps). +// arm is one alternative of a {a|b} token or one operand, split into the key +// naming the node in a template's fields (a sibling field, or the head a +// reference is bound under) and the tail of a dotted path into it. A non-empty +// tail is what makes the arm a bound draw: its head is drawn once per expansion +// (see compileOps). type arm struct { - name string // as written, and the key a bound draw's value is held under + name string // as written, for messages key string tail []string steps []string // key per level passed through; the head and leaf hold their own + path string // key and tail, the one spelling every way of writing this read shares } -// splitArm splits one token alternative into key and tail. A reference keeps its -// dots — linkRefs binds it whole — so only a sibling name reads as a path. -func splitArm(name string) arm { +// splitArm splits one name into key and tail. refs maps a reference to what +// linkRefs bound it to; before linking, a reference is whole. +func splitArm(name string, refs map[string]refBinding) arm { if isRef(name) { - return arm{name: name, key: name} + b, bound := refs[name] + if !bound || len(b.tail) == 0 { + key := name + if bound { + key = b.key + } + return arm{name: name, key: key, path: key} + } + return pathArm(name, b.key, b.tail) } head, tail, dotted := strings.Cut(name, ".") if !dotted { - return arm{name: name, key: name} + return arm{name: name, key: name, path: name} } - segs := strings.Split(tail, ".") + return pathArm(name, head, strings.Split(tail, ".")) +} + +func pathArm(name, key string, segs []string) arm { var steps []string for i := 0; i < len(segs)-1; i++ { // every level except the leaf's own - steps = append(steps, head+"."+strings.Join(segs[:i+1], ".")) + steps = append(steps, key+"."+strings.Join(segs[:i+1], ".")) } - return arm{name: name, key: head, tail: segs, steps: steps} + return arm{name: name, key: key, tail: segs, steps: steps, path: key + "." + strings.Join(segs, ".")} } -// checkNoOverlap rejects a format that both renders a level and reads a path into -// it — {p} beside {p.first}, or {p.addr} beside {p.addr.city}. The two spell one -// draw two ways: the path reads the level's held draw, while rendering the level -// expands it afresh, so their values disagree. One spelling, so there is nothing -// to get wrong. Names are compared in sorted order, so which pair is reported -// does not depend on where the tokens sit. -func checkNoOverlap(format string, bound map[string]string) error { - names := boundReaders(format, bound) - // Stable over one format-order scan, so two readers of one name (a token and a - // calc operand both naming "p") are reported as the format writes them. - sort.SliceStable(names, func(i, j int) bool { return names[i].name < names[j].name }) - for i, level := range names { - for _, path := range names[i+1:] { - if strings.HasPrefix(path.name, level.name+".") { - return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name) - } - } +// splitArms splits a token body's '|' alternatives. +func splitArms(body string, refs map[string]refBinding) []arm { + parts := strings.Split(body, "|") + arms := make([]arm, len(parts)) + for i, p := range parts { + arms[i] = splitArm(p, refs) } - return nil + return arms } -// reader is one way a format reaches a bound field, and how to name that spelling. -type reader struct{ name, label string } - -// boundReaders lists every way a format reaches a bound field, in the order the -// format writes them. A calc operand renders its field, so it names a level exactly -// as a token does; one scan finds both, which is what puts them in one order. -func boundReaders(format string, bound map[string]string) []reader { - var names []reader - _ = eachToken(format, func(t ftoken) error { - if t.kind != 'b' { - return nil - } - if _, _, isFunc := funcCall(t.body); isFunc { - for _, operand := range calcTokenOperands(t.body) { - if _, isBound := bound[operand]; isBound { - names = append(names, reader{operand, fmt.Sprintf("calc operand %q", operand)}) - } - } - return nil - } - for _, a := range splitArms(t.body) { - if _, isBound := bound[a.key]; isBound { - names = append(names, reader{a.name, "token {" + a.name + "}"}) - } - } - return nil - }) - return names -} - -// checkSegments rejects an unfinished path: "{a.}", "{.b}" and "{a..b}" each have -// a segment naming nothing. A field really named "" would otherwise make them +// checkSegments rejects an unfinished path: "{a.}" and "{a..b}" each have a +// segment naming nothing. A field really named "" would otherwise make them // resolve, so a typo would read as a path that worked. func checkSegments(a arm) error { if len(a.tail) == 0 { @@ -296,59 +299,87 @@ func checkSegments(a arm) error { return nil } -// splitArms splits a token body's '|' alternatives. -func splitArms(body string) []arm { - parts := strings.Split(body, "|") - arms := make([]arm, len(parts)) - for i, p := range parts { - arms[i] = splitArm(p) - } - return arms -} - // callFn is a builtin bound to one call site: its args already parsed. It reads the // output emitted so far in the current expansion (a derivation's payload) and the // values of the operands it named, which expand read for it. type callFn func(s *session, emitted string, operands []string) string -// op is one compiled unit of a format string: a literal run, a class char, a field -// alternation, or a builtin already bound to its args. compile builds these so -// render never re-scans the format. +// op is one compiled unit of a format string: a literal run, a field alternation, +// or a builtin already bound to its args. compile builds these so render never +// re-scans the format. type op struct { - kind byte // 'l' literal run, 'c' class char, 'f' field alternation, 'b' builtin + kind byte // 'l' literal run, 'f' field alternation, 'b' builtin lit string // kind 'l' - r rune // kind 'c' arms []arm // kind 'f': the '|' alternatives, split into key and path once call callFn - // operands names the sibling fields a {calc()} reads, in the order calcVars - // fixed; expand reads them before the call. nil for every other builtin. - operands []string + // operands are the fields the builtin reads, in the order its operands func + // fixed; expand reads them before the call. nil for a builtin that reads none. + operands []arm } -// compileOps turns a format string into ops, and returns the smallest output it can -// produce (literals plus one byte per class char) to size the render buffer, plus -// two sets. bound is the levels the format addresses by dotted path, each mapped to -// the first path reading it, which is what the overlap fences name. held is every -// name drawn once per expansion — those levels, plus the siblings a {calc()} reads, -// so an operand shown is the operand computed. Both are nil when the format needs -// neither, so data that uses neither carries no render-time cost. Call checkTokens -// first: it is what proves the scan and every token are valid. -func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { - var ops []op - var lit strings.Builder - var bound map[string]string - var held map[string]bool - grow := 0 - hold := func(name string) { - if held == nil { - held = map[string]bool{} - } - held[name] = true +// formatOps is a compiled format: its ops, the size of its literal text (to size +// the render buffer), and the names drawn once per expansion. bound maps each level +// a path reads into to the first such path; held is every such level plus the +// fields an operand reads; holder maps each held name to the first reader holding +// it, for error messages. The maps are nil when the format holds nothing, so data +// that holds nothing carries no render-time cost. +type formatOps struct { + ops []op + grow int + bound map[string]string + held map[string]bool + holder map[string]string +} + +func (c *formatOps) hold(a arm, label string) { + if c.held == nil { + c.held = map[string]bool{} + c.holder = map[string]string{} } + c.held[a.key] = true + if _, named := c.holder[a.key]; !named { + c.holder[a.key] = label + } + if len(a.tail) > 0 { + if c.bound == nil { + c.bound = map[string]string{} + } + if _, named := c.bound[a.key]; !named { + c.bound[a.key] = a.name + } + } +} + +func (c *formatOps) function(body string, refs map[string]refBinding) { + name, args, _ := funcCall(body) + var operands []arm + for _, operand := range tokenOperands(body) { + a := splitArm(operand, refs) + c.hold(a, fmt.Sprintf("%s operand %q", name, operand)) + operands = append(operands, a) + } + c.ops = append(c.ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) +} + +func (c *formatOps) field(body string, refs map[string]refBinding) { + arms := splitArms(body, refs) + for _, a := range arms { + if len(a.tail) > 0 { + c.hold(a, "token {"+a.name+"}") + } + } + c.ops = append(c.ops, op{kind: 'f', arms: arms}) +} + +// compileOps compiles a format string. Call checkTokens first: it is what proves +// the scan and every token are valid. +func compileOps(format string, refs map[string]refBinding) formatOps { + var c formatOps + var lit strings.Builder flush := func() { if lit.Len() > 0 { - grow += lit.Len() - ops = append(ops, op{kind: 'l', lit: lit.String()}) + c.grow += lit.Len() + c.ops = append(c.ops, op{kind: 'l', lit: lit.String()}) lit.Reset() } } @@ -356,36 +387,16 @@ func compileOps(format string) ([]op, int, map[string]string, map[string]bool) { switch t.kind { case 'l': lit.WriteRune(t.r) - case 'c': - flush() - grow++ - ops = append(ops, op{kind: 'c', r: t.r}) case 'b': flush() - if name, args, ok := funcCall(t.body); ok { - operands := calcTokenOperands(t.body) - for _, operand := range operands { - hold(operand) // a calc renders its operand, so the expansion holds that draw - } - ops = append(ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands}) + if _, _, isFunc := funcCall(t.body); isFunc { + c.function(t.body, refs) } else { - arms := splitArms(t.body) - for _, a := range arms { - if len(a.tail) > 0 { - if bound == nil { - bound = map[string]string{} - } - if _, named := bound[a.key]; !named { - bound[a.key] = a.name // the first path reading it, for error messages - } - hold(a.key) - } - } - ops = append(ops, op{kind: 'f', arms: arms}) + c.field(t.body, refs) } } return nil }) flush() - return ops, grow, bound, held + return c } diff --git a/template_stability_test.go b/template_stability_test.go index cf16d86..e0e584f 100644 --- a/template_stability_test.go +++ b/template_stability_test.go @@ -7,25 +7,25 @@ import "testing" // cannot quietly shift the rng stream that reproducibility depends on. func TestSeededOutputIsStable(t *testing.T) { dir := writeData(t, map[string]string{ - "alt": `{"format":"{a|b}","a":["A"],"b":["B"]}`, + "alt": `{"format":"{a|b}","a":"A","b":"B"}`, "calc": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, - "classes": `{"format":"00-11-AA-aa"}`, - "escapes": `{"format":"#0#1#A#a##{x}","x":["!"]}`, - "funcs": `{"format":"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"}`, - "nested": `{"format":"{outer}","outer":[{"format":"{inner}-00","inner":["i"]}]}`, - "ref": `{"format":"see {..alt}"}`, + "classes": `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`, + "escapes": `{"format":"01Aa#{x}","x":"!"}`, + "funcs": `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`, + "nested": `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`, + "ref": `"see {/alt}"`, "repeat": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`, - "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":["012345678901234"],"e":["123456789012"],"m":["12345678"]}`, - "weights": `[{"format":"big","weight":9},{"format":"tiny","weight":1}]`, + "sums": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`, + "weights": `[{"format":"big","weight":9},"tiny"]`, }) want := map[string][]string{ - "alt": {"A", "B", "A", "A"}, + "alt": {"A", "A", "B", "A"}, "calc": {"100.00 x 3 = 300.00", "100.00 x 3 = 300.00", "100.00 x 7 = 700.00", "5.00 x 3 = 15.00"}, "classes": {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"}, "escapes": {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"}, "funcs": {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"}, - "nested": {"i-89", "i-67", "i-47", "i-27"}, - "ref": {"see A", "see A", "see B", "see A"}, + "nested": {"i-73", "i-23", "i-67", "i-85"}, + "ref": {"see A", "see A", "see A", "see B"}, "repeat": {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"}, "sums": {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"}, "weights": {"big", "big", "big", "big"}, diff --git a/template_test.go b/template_test.go index ee56174..74dbc30 100644 --- a/template_test.go +++ b/template_test.go @@ -8,7 +8,13 @@ import ( ) // engine builds a seeded generator with no loaded categories, for rendering tests. -func engine(seed uint64) *Generator { return &Generator{rand: newRand(seed, true)} } +func engine(seed uint64) *Generator { + s, err := newRand(seed, true) + if err != nil { + panic(err) + } + return &Generator{rand: s} +} // parse unmarshals a JSON template fragment into its dynamic form. func parse(t *testing.T, s string) any { @@ -35,19 +41,33 @@ func mustRender(t *testing.T, f *Generator, s string) string { return render(f.rand, compiled(t, s)) } -func TestLiteralStringIsVerbatim(t *testing.T) { - // A bare string is a literal, never formatted: 'a'/'A' must survive. - if got := mustRender(t, engine(1), `"Malmö"`); got != "Malmö" { - t.Fatalf("literal = %q, want Malmö", got) +func TestStringIsAFormat(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `"Malmö"`: "Malmö", + `"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0", + `"{{x}}"`: "{x}", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } + if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) { + t.Errorf(`render("{digits(3)}") = %q, want three digits`, got) + } + if _, err := compile(parse(t, `"{x}"`)); err == nil || !strings.Contains(err.Error(), `no field "x"`) { + t.Errorf(`compile("{x}") = %v, want a no-field error`, err) } } -func TestCharacterClasses(t *testing.T) { +func TestClassBuiltins(t *testing.T) { cases := map[string]*regexp.Regexp{ - `{"format":"0"}`: regexp.MustCompile(`^[0-9]$`), - `{"format":"1"}`: regexp.MustCompile(`^[1-9]$`), - `{"format":"A"}`: regexp.MustCompile(`^[A-Z]$`), - `{"format":"a"}`: regexp.MustCompile(`^[a-z]$`), + `"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`), + `"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`), + `"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`), + `"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`), + `"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`), + `"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`), } f := engine(7) for tmpl, re := range cases { @@ -59,15 +79,39 @@ func TestCharacterClasses(t *testing.T) { } } -func TestEscapeAndLiteralChars(t *testing.T) { - // '#' escapes the next char; non-class chars (7, x, -) are literal. - if got := mustRender(t, engine(1), `{"format":"#0#1#A#a## x7-z"}`); got != "01Aa# x7-z" { - t.Fatalf("escape = %q, want \"01Aa# x7-z\"", got) +func TestTextIsLiteral(t *testing.T) { + if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" { + t.Fatalf("text = %q, want it verbatim", got) + } +} + +func TestBraceEscapes(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `{"format":"{{","x":"v"}`: "{", + `{"format":"}}","x":"v"}`: "}", + `{"format":"{{x}}","x":"v"}`: "{x}", + `{"format":"{{{x}}}","x":"v"}`: "{v}", + `{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } + for src, want := range map[string]string{ + `"x}y"`: "}}", + `"}"`: "}}", + `{"format":"{a{b}","a":"Q"}`: "'{'", + `"{x"`: "unterminated", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want) + } } } func TestTokenSubstitution(t *testing.T) { - if got := mustRender(t, engine(1), `{"format":"{x}sson","x":["Erik"]}`); got != "Eriksson" { + if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" { t.Fatalf("token = %q, want Eriksson", got) } } @@ -76,19 +120,17 @@ func TestAlternationPicksOneField(t *testing.T) { f := engine(3) seen := map[string]bool{} for i := 0; i < 100; i++ { - seen[mustRender(t, f, `{"format":"{a|b}","a":["A"],"b":["B"]}`)] = true + seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true } if !seen["A"] || !seen["B"] || len(seen) != 2 { t.Fatalf("alternation produced %v, want both A and B", seen) } } -func TestWeightZeroNeverChosen(t *testing.T) { - f := engine(5) - for i := 0; i < 200; i++ { - if got := mustRender(t, f, `[{"format":"X","weight":0},"Y"]`); got != "Y" { - t.Fatalf("weight-0 variant chosen: %q", got) - } +func TestWeightZeroIsRejected(t *testing.T) { + _, err := compile(parse(t, `[{"format":"X","weight":0},"Y"]`)) + if err == nil || !strings.Contains(err.Error(), "weight 0") || !strings.Contains(err.Error(), "remove") { + t.Fatalf("compile(weight 0) = %v, want it rejected naming the fix", err) } } @@ -96,7 +138,7 @@ func TestWeightSkewsDistribution(t *testing.T) { f := engine(9) heavy := 0 for i := 0; i < 1000; i++ { - if mustRender(t, f, `[{"format":"H","weight":10},{"format":"L"}]`) == "H" { + if mustRender(t, f, `[{"format":"H","weight":10},"L"]`) == "H" { heavy++ } } @@ -118,7 +160,7 @@ func TestRepeatRendersFormatNTimes(t *testing.T) { f, re := engine(7), regexp.MustCompile(`^[0-9]{4}$`) seen := map[string]bool{} for i := 0; i < 50; i++ { - got := mustRender(t, f, `{"format":"0","repeat":4}`) + got := mustRender(t, f, `{"format":"{digits(1)}","repeat":4}`) if !re.MatchString(got) { t.Fatalf("repeat-4 = %q, want 4 digits", got) } @@ -135,10 +177,10 @@ func TestFunctionTokenLuhn(t *testing.T) { // buffer, so a value is never re-rendered. Bodies are escaped to fix input. f := engine(1) cases := map[string]string{ - `{"format":"#8#1#1#2#1#8#9#8#7{luhn()}"}`: "8112189876", // personnummer body - `{"format":"#7#9#9#2#7#3#9#8#7#1{luhn()}"}`: "79927398713", // classic Luhn vector - `{"format":"#8#1#1#2#1#8-#9#8#7{luhn()}"}`: "811218-9876", // '-' skipped, kept - `{"format":"{n}{luhn()}","n":["811218987"]}`: "8112189876", // over a rendered token + `"811218987{luhn()}"`: "8112189876", // personnummer body + `"7992739871{luhn()}"`: "79927398713", // classic Luhn vector + `"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept + `{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token } for tmpl, want := range cases { if got := mustRender(t, f, tmpl); got != want { @@ -151,7 +193,7 @@ func TestRecursionHasNoDepthLimit(t *testing.T) { // Build {format:{a}, a:[{format:{a}, a:[ ... "deep" ]]}} 50 levels deep. tmpl := `"deep"` for i := 0; i < 50; i++ { - tmpl = `{"format":"{a}","a":[` + tmpl + `]}` + tmpl = `{"format":"{a}","a":` + tmpl + `}` } if got := mustRender(t, engine(1), tmpl); got != "deep" { t.Fatalf("deep recursion = %q, want deep", got) @@ -162,45 +204,68 @@ func TestCompileErrors(t *testing.T) { // Every structural problem is caught up front, at compile/New time, never // deferred to a random render that happens to hit the bad branch. for _, bad := range []string{ - `{"x":["Q"]}`, // object without "format" - `{"format":"{y}","x":1}`, // a field is a bare number - `[1, 2]`, // a choice of numbers - `5`, // unsupported node type - `[]`, // empty choice - `{"format":"{x"}`, // unterminated brace - `{"format":"{y}","x":["Q"]}`, // token names a missing field - `{"format":"{}"}`, // empty token name - `{"format":"{a|}","a":["Q"]}`, // empty alternation segment - `[{"format":"A","weight":-1},{"format":"B"}]`, // negative weight + `{"x":"Q"}`, // object without "format" + `{"format":"{y}","x":1}`, // a field is a bare number + `[1, 2]`, // a choice of numbers + `5`, // unsupported node type + `[]`, // empty choice + `"{x"`, // unterminated brace + `"x}y"`, // a lone } must be written }} + `["x"]`, // a one-item choice is its item + `["a","a"]`, // a repeated item is a weight + `{"format":"x"}`, // an object holding only a format is a string + `[{"format":"a","weight":1},"b"]`, // weight 1 is the default + `{"format":"{x}","x":"v","repeat":1}`, // repeat 1 is the default + `{"format":"{a{b}","a":"Q"}`, // a brace inside a token + `"{x}"`, // a bare string has no fields to name + `"{digits(0)}"`, // count must be positive + `"{upper(x)}"`, // count must be an integer + `"{lower()}"`, // wrong arity + `{"format":"{y}","x":"Q"}`, // token names a missing field + `"{}"`, // empty token name + `{"format":"{a|}","a":"Q"}`, // empty alternation segment + `[{"format":"A","weight":-1},"B"]`, // negative weight `[{"format":"A","weight":0},{"format":"B","weight":0}]`, // weights sum to zero `[{"format":"A","weight":1e308},{"format":"B","weight":1e308}]`, // weights overflow to +Inf - `[{"format":"A","weight":"heavy"}]`, // non-numeric weight + `{"format":"A","weight":"heavy"}`, // non-numeric weight `{"format":"x","repeat":0}`, // repeat below 1 `{"format":"x","repeat":-2}`, // negative repeat `{"format":"x","repeat":1.5}`, // non-integer repeat `{"format":"x","repeat":"two"}`, // non-numeric repeat `{"format":"x","repeat":2,"separator":5}`, // non-string separator - `{"format":"{nope()}"}`, // unknown function - `{"format":"{luhn(x)}"}`, // function given args it takes none of - `{"format":"{luhn(}"}`, // malformed function token - `{"format":"{int(1)}"}`, // wrong arity - `{"format":"{int(a,b)}"}`, // non-integer args - `{"format":"{int(5,1)}"}`, // min > max - `{"format":"{hex(0)}"}`, // count must be positive - `{"format":"{nanoid(-1)}"}`, // negative count - `{"format":"{base64(0)}"}`, // count must be positive - `{"format":"{float(1,2)}"}`, // wrong arity - `{"format":"{float(1,2,-1)}"}`, // negative decimals - `{"format":"{iban(US)}"}`, // unsupported country - `{"format":"{seq(a,b)}"}`, // seq takes at most one name - `{"format":"{calc()}"}`, // calc needs an expression - `{"format":"{calc(1 +)}"}`, // dangling operator - `{"format":"{calc((1 + 2)}"}`, // unbalanced parenthesis - `{"format":"{calc(1 2)}"}`, // two operands, no operator - `{"format":"{calc(price)}"}`, // operand names no field - `{"format":"{calc(1, 2, 3)}"}`, // too many args - `{"format":"{calc(1, x)}"}`, // decimals arg not an integer - `{"format":"{calc(1, -1)}"}`, // decimals negative + `"{nope()}"`, // unknown function + `"{luhn(x)}"`, // function given args it takes none of + `"{luhn(}"`, // malformed function token + `"{int(1)}"`, // wrong arity + `"{int(a,b)}"`, // non-integer args + `"{int(5,1)}"`, // min > max + `"{hex(0)}"`, // count must be positive + `"{nanoid(-1)}"`, // negative count + `"{base64(0)}"`, // count must be positive + `"{float(1,2)}"`, // wrong arity + `"{float(1,2,-1)}"`, // negative decimals + `"{float(NaN,NaN,2)}"`, // bounds must be finite + `"{float(Inf,Inf,2)}"`, // same-sign infinities + `"{float(1,NaN,2)}"`, // one NaN bound + `"{float(-Inf,1,2)}"`, // one infinite bound + `"{digits(+5)}"`, // a count is a plain integer + `"{digits(05)}"`, // no leading zero + `"{int(+1,5)}"`, // a bound is a plain integer + `"{int(5,5)}"`, // a constant is written as text + `"{float(1,1,2)}"`, // a constant is written as text + `{"format":"{x}","x":"v","repeat":2,"separator":""}`, // separator "" is the default + `"{calc(1/0)}"`, // a constant zero divisor + `{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor + `"{iban(US)}"`, // unsupported country + `"{seq(a,b)}"`, // seq takes at most one name + `"{calc()}"`, // calc needs an expression + `"{calc(1 +)}"`, // dangling operator + `"{calc((1 + 2)}"`, // unbalanced parenthesis + `"{calc(1 2)}"`, // two operands, no operator + `"{calc(price)}"`, // operand names no field + `"{calc(1, 2, 3)}"`, // too many args + `"{calc(1, x)}"`, // decimals arg not an integer + `"{calc(1, -1)}"`, // decimals negative } { if _, err := compile(parse(t, bad)); err == nil { t.Errorf("compile(%s) = nil error, want error", bad) @@ -211,7 +276,7 @@ func TestCompileErrors(t *testing.T) { func TestFakePathNavigation(t *testing.T) { f := engine(1) f.categories = map[string]node{ - "addr": compiled(t, `[{"format":"{street}","street":["Main"]}]`), + "addr": compiled(t, `{"format":"{street}","street":"Main"}`), } if got, err := f.Fake("addr"); err != nil || !strings.Contains(got, "Main") { t.Fatalf("Fake(addr) = %q, %v", got, err) @@ -235,15 +300,16 @@ func TestGrowIsALowerBound(t *testing.T) { for _, format := range []string{ "", "plain literal", - "00-11-AA-aa", - "#0#1#A#a## literal", + "{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}", + "01Aa# literal", + "{{}} {{{x}}}", "Ö dag åäö 日本語", "{x}{x}{x}", "{hex(8)}-{int(10,99)}-{nanoid(5)}", "9{d}{luhn()}", "{a|b} and {a|b}", } { - src := `{"format":` + quote(format) + `,"x":["1"],"a":["A"],"b":["B"],"d":["012345678901234"]}` + src := `{"format":` + quote(format) + `,"x":"1","a":"A","b":"B","d":"012345678901234"}` tmpl, ok := compiled(t, src).(*template) if !ok { t.Fatalf("format %q did not compile to a template", format) @@ -263,3 +329,53 @@ func quote(s string) string { } return string(b) } + +func TestHashIsLiteral(t *testing.T) { + f := engine(1) + cases := map[string]string{ + `{"format":"","x":"v"}`: "", + `{"format":"#","x":"v"}`: "#", + `{"format":"##","x":"v"}`: "##", + `{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a", + `{"format":"#{x}","x":"v"}`: "#v", + } + for tmpl, want := range cases { + if got := mustRender(t, f, tmpl); got != want { + t.Errorf("render(%s) = %q, want %q", tmpl, got, want) + } + } +} + +func TestMultibyteFormat(t *testing.T) { + // Scanning is rune-aware: multibyte literals coexist with class chars and + // tokens without corrupting indices. + got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`) + if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) { + t.Fatalf("multibyte format = %q", got) + } +} + +func TestAlternationThreeWay(t *testing.T) { + f := engine(4) + seen := map[string]bool{} + for i := 0; i < 200; i++ { + seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true + } + if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 { + t.Fatalf("3-way alternation produced %v, want A, B and C", seen) + } +} + +func TestArgErrorsNameTheSpelling(t *testing.T) { + for src, want := range map[string]string{ + `"{float(1,2,02)}"`: "write 2", + `{"format":"{calc(a,02)}","a":"1"}`: "write 2", + `"{digits(+5)}"`: "write 5", + `"{hex(99999999999999999999)}"`: "exceeds the maximum", + `"{int(007,9)}"`: "write 7", + } { + if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("compile(%s) = %v, want an error saying %q", src, err, want) + } + } +} diff --git a/transform_test.go b/transform_test.go new file mode 100644 index 0000000..0272856 --- /dev/null +++ b/transform_test.go @@ -0,0 +1,59 @@ +package fejkdata + +import "testing" + +func TestTransforms(t *testing.T) { + f := engine(1) + for src, want := range map[string]string{ + `{"format":"{uppercase(x)}|{lowercase(x)}|{ascii(x)}","x":"Åsa Ödegård-Nuñez"}`: "ÅSA ÖDEGÅRD-NUÑEZ|åsa ödegård-nuñez|Asa Odegard-Nunez", + `{"format":"{lowercase(ascii(x))}","x":"Åsa"}`: "asa", + `{"format":"{ascii(x)}","x":"Ærø ß 日本"}`: "AEro ss ", + } { + if got := mustRender(t, f, src); got != want { + t.Errorf("render(%s) = %q, want %q", src, got, want) + } + } +} + +func TestTransformReadsTheHeldDraw(t *testing.T) { + f := engine(2) + src := `{"format":"{p.first}={lowercase(p.first)}","p":[{"format":"{first}","first":"Anna"},{"format":"{first}","first":"Bo"}]}` + for i := 0; i < 50; i++ { + if got := mustRender(t, f, src); got != "Anna=anna" && got != "Bo=bo" { + t.Fatalf("render = %q, want the transform over the same draw", got) + } + } +} + +func TestTransformOverAReference(t *testing.T) { + dir := writeData(t, map[string]string{ + "person": `[{"format":"{first} {last}","first":"Åsa","last":"Öberg"},{"format":"{first} {last}","first":"Bo","last":"Ek"}]`, + "email": `"{/person.first} {/person.last} <{lowercase(ascii(/person.first))}.{lowercase(ascii(/person.last))}@example.com>"`, + }) + f := newGenerator(t, dir, WithSeed(5)) + for i := 0; i < 50; i++ { + if got := fake(t, f, "email"); got != "Åsa Öberg " && got != "Bo Ek " { + t.Fatalf("email = %q, want a record from one draw", got) + } + } +} + +func TestTransformArgs(t *testing.T) { + for _, bad := range []string{ + `{"format":"{lowercase()}","x":"v"}`, + `{"format":"{lowercase(nope)}","x":"v"}`, + `{"format":"{ascii(x,y)}","x":"v","y":"w"}`, + `{"format":"{lowercase(hex(2))}","x":"v"}`, + `{"format":"{lowercase(x)} {x.a}","x":{"format":"{a}","a":"1"}}`, + } { + if _, err := compile(parse(t, bad)); err == nil { + t.Errorf("compile(%s) = nil error, want it rejected", bad) + } + } +} + +func TestAsciiKeepsDEL(t *testing.T) { + if got := mustRender(t, engine(1), `{"format":"{ascii(x)}","x":"a\u007fb"}`); got != "a\u007fb" { + t.Errorf("ascii over DEL = %q, want it kept: DEL is ASCII", got) + } +}