diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..02a87f0 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,162 @@ +name: Packages + +on: + workflow_call: + inputs: + tag: + type: string + required: true + secrets: + TAP_GITHUB_TOKEN: + required: true + PACKAGES_GPG_KEY: + required: true + PACKAGES_RSA_KEY: + required: true + DOCKERHUB_TOKEN: + required: true + workflow_dispatch: + inputs: + tag: + description: The released tag to package, such as v0.1.0 + type: string + required: true + +permissions: + contents: read + +concurrency: + group: packages + cancel-in-progress: false + +jobs: + packages: + name: Binaries, packages and image for ${{ inputs.tag }} + runs-on: ubuntu-26.04 + permissions: + contents: write + packages: write + outputs: + latest: ${{ steps.latest.outputs.latest }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag }} + path: src + fetch-depth: 0 + persist-credentials: false + # Check out the packaging config at the workflow's commit, so a tag cut before the + # config existed can still be packaged. + - uses: actions/checkout@v7 + with: + path: tooling + persist-credentials: false + - name: The Go version the gate tests + id: go + run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@v7 + with: + go-version: ${{ steps.go.outputs.version }} + cache: false + - name: Channels holding one version + id: latest + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + run: | + latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)" + if [ "$latest" = "$TAG" ]; then + echo "latest=true" >> "$GITHUB_OUTPUT" + else + echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT" + echo "OLDER_TAG=1" >> "$GITHUB_ENV" + fi + - uses: docker/setup-buildx-action@v4 + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: goreleaser/goreleaser-action@v7 + with: + version: v2.18.3 + workdir: src + args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }} + - name: The built binary and the published image print the tag + env: + TAG: ${{ inputs.tag }} + run: | + test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG" + test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG" + + dockerhub: + name: Docker Hub image for ${{ inputs.tag }} + needs: packages + runs-on: ubuntu-26.04 + permissions: + packages: read + steps: + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Copy the GHCR image to Docker Hub, and check it prints the tag + env: + LATEST: ${{ needs.packages.outputs.latest }} + TAG: ${{ inputs.tag }} + run: | + tags=(-t "docker.io/larvit/fejkdata:${TAG#v}") + if [ "$LATEST" = true ]; then + tags+=(-t docker.io/larvit/fejkdata:latest) + fi + docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}" + test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG" + + repos: + name: apt, dnf and apk repositories for ${{ inputs.tag }} + needs: packages + if: needs.packages.outputs.latest == 'true' + runs-on: ubuntu-26.04 + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: The release's Linux packages + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk' + - uses: docker/setup-buildx-action@v4 + - name: Install fejkdata from each repository, then export them + env: + PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }} + PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }} + TAG: ${{ inputs.tag }} + run: | + build() { + docker buildx build -f release-tooling/package-repos.Dockerfile \ + --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \ + --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" . + } + build --target checks --output type=local,dest=checks + build --target site --output type=local,dest=site + - uses: actions/upload-pages-artifact@v5 + with: + path: site + - id: deploy + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7569186..92f4179 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,9 @@ on: pull_request: push: branches: [main] + workflow_dispatch: + schedule: + - cron: '17 5 * * *' permissions: contents: read @@ -11,9 +14,10 @@ permissions: jobs: test: name: vet + fmt + tests - runs-on: ubuntu-24.04 + if: github.event_name != 'schedule' + runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: persist-credentials: false fetch-depth: 0 @@ -26,11 +30,16 @@ jobs: else base='${{ github.event.before }}' fi - if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then + if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then echo "code=false" >> "$GITHUB_OUTPUT" else echo "code=true" >> "$GITHUB_OUTPUT" fi + if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then + echo "packaging=false" >> "$GITHUB_OUTPUT" + else + echo "packaging=true" >> "$GITHUB_OUTPUT" + fi echo "base=$base" >> "$GITHUB_OUTPUT" - name: CHANGELOG entry for data changes run: | @@ -48,22 +57,84 @@ jobs: # failure is version-specific. - name: Lowest supported Go if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }} - run: docker build --build-arg GO_VERSION=1.22.12 . + run: docker build --build-arg GO_VERSION=1.22.12 --target portable . + - name: Package repositories, signed with throwaway keys + if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }} + run: | + docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" . + PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \ + docker build -f release-tooling/package-repos.Dockerfile --target checks \ + --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY . - release: - name: Gitea release from CHANGELOG.md - needs: test - if: github.event_name == 'push' - runs-on: ubuntu-24.04 + vulnerabilities: + name: Known vulnerabilities in this Go version and the required modules + runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: + actions: write contents: write + pull-requests: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: persist-credentials: false - - run: python3 release-tooling/publish_release.py + - name: Fetch govulncheck's report from vuln.go.dev + run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" . + - name: File each one in todo.md + id: file + run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md + - name: Propose the filed items + if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main' env: - GITEA_API_URL: ${{ github.api_url }} - GITEA_REPOSITORY: ${{ github.repository }} - GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} + run: | + if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then + exit 0 + fi + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git switch -C vulnerabilities + git commit -qam "File the known vulnerabilities govulncheck reports" + git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities + gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \ + --body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it." + # A pull request the workflow's token opens starts no workflow. + gh workflow run test.yml --ref vulnerabilities + + release: + name: Release from CHANGELOG.md + needs: [test, vulnerabilities] + if: github.event_name == 'push' + runs-on: ubuntu-26.04 + permissions: + contents: write + outputs: + tag: ${{ steps.publish.outputs.tag }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - id: publish + run: python3 release-tooling/publish_release.py + env: + FORGE_API_URL: ${{ github.api_url }} + FORGE_REPOSITORY: ${{ github.repository }} + FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }} SHA: ${{ github.sha }} + + packages: + needs: release + if: needs.release.outputs.tag != '' + permissions: + contents: write + packages: write + pages: write + id-token: write + uses: ./.github/workflows/packages.yml + with: + tag: ${{ needs.release.outputs.tag }} + secrets: + TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }} + PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }} + PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index e2b0b93..e08dd3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,27 +1,137 @@ # Rules -- Go runs only through `docker compose run --rm `; the merge gate is `docker build .` plus CI's changelog check. +- Go runs only through `docker compose run --rm `; the merge gate is what the README's Development section lists. - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit. - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table. - One-line commit messages: no ticket prefix, no repo name, no authorship trailers. -- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it - offers fails with "is it still open?", as does a merge whose required checks are - still pending, so read the checks before believing the style is the problem. Use - `tea api /repos/{owner}/{repo}/pulls//merge -f Do=fast-forward-only`. -- Gitea queues Actions runs that `/repos/{owner}/{repo}/actions/tasks` does not - list, so an empty task list says nothing about whether CI ran; read - `/repos/{owner}/{repo}/commits//status`. -- README goal 2 is gated at 7.0 on the `comprehension-panel` skill, and below it - nothing else merges — no feature, no category, no data — bar a security fix and a - dependency bump. `todo.md` carries the round: the nine-seat findings run at depth 1, - a PR per item it names, then the run again, until the score passes. At or above 7.0 +- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward + only, `gh pr merge --rebase` where the branch is already on top of main. +- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check + its status once when a PR is marked ready, answer a review if one is there, and never + wait for one. +- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it + nothing else merges — no feature, no category, no data — bar a security fix, a + dependency bump, a repair of behaviour the README documents, the infrastructure + the round itself runs on, and a chunk the comprehension-floor decision lists as + shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat + findings at depth 1 or a restructure the maintainer approved, then the nine-seat run + again, until the score passes. At or above 7.0 every pull request scores with the four-seat run and answers it in one run. No merge - lowers the last score. + lowers the last score its own panel recorded: the nine-seat round and the four-seat + run are two ratchets, never compared with each other. A lower score blocks the merge + even where it may be panel noise: fix what the seats name until the score rises. A PR + that only records a run's scores and files what its seats name merges even when a + score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse. - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble. -- One spelling per result: reject the other at `New`, and let the error name the spelling to use. -- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. - Prose naming a source states what that source states: read the register's own field before paraphrasing it, and name the register you read before claiming none publishes a spelling. - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. -- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed. +- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole. + +# Decisions + +In [docs/decisions.md](docs/decisions.md): + +- GitHub is canonical, and the module path names it +- The Gitea copy stays, as a pull mirror of GitHub +- The vocabulary sits below `doc.go`'s package clause, not in the package doc +- Options and fields share one namespace +- `{a|b}` stays beside nested choices +- Flags follow getopt_long +- A struct tag is a template by its shape +- The CLI renders exactly the template on its stdin, and adds nothing +- The CLI reads only the library's public API +- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129 +- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string +- `FakeTemplate` and `NewTemplate` both stay +- The shipped data is embedded, not discovered +- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0 +- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace` +- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads +- Every `{…}` draws afresh, and only a name keeps a pick +- Reference sigils follow the filesystem +- A change to what exists is a major; a minor only adds +- An error is a contract by what it names, not its bytes +- A load reports every mistake at once +- Formats are checked after all data is loaded, so their errors name the category +- Raising the lowest supported Go is a major +- The format check runs on the latest Go only +- The changelog heading is the one spelling of a release; CI cuts the tag +- A `--data-path` override rebinds every reference to the category it replaces +- The repeat cap bounds renders, not bytes +- 64-bit targets only +- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf` +- Samples say what they emit, transforms what they do +- A record is a template seen as columns; a Go struct is the one second schema +- A struct's records follow Go's field access, and compile on first use +- A category may reference itself, and a record's fences run at load +- A record's column set is fixed before the first draw +- Null is a `null` item, not a rate +- A typed column holds one value, not composed text +- A column that only reads one reference or name is the column it reads +- A typed column's calc is refused unless proven +- `Column` carries text, not a Go value +- A unit takes the stem of what it is, and a file the stem of the units it holds +- One name, one meaning +- The performance gate asserts allocations, not wall-clock time +- Rows live in a TSV, the shape in JSON +- A selector is bracketed, and a dot inside it is literal +- `parent` names the link column and the table alike +- After a row, a path names a column or a linked table +- A selected row is fixed, not drawn +- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick +- A table selects by one key column, a code, never a free-form name +- A parent row with no child row is a load error +- A table is a record of string columns +- The key map is built at load, the rest on first draw +- Two categories may name one TSV +- A cell may read a reference, and each row its own +- A path is walked once without drawing before it is walked for real +- A country's postal codes and streets are siblings under its locality +- A locale package imports its country's geo package, and its `Modules()` returns both +- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten +- A locale's `address` binds its country's record once and prints it whole +- A postort's kommun comes from its name, its tätort or its codes, never from distance +- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places +- `--list` stays a plain list of paths +- A layout is always quoted +- A title is a table under `sex` +- A table owns the spelling of a selector on it +- No builtin reads the clock, so a date is bounded by days, never by an age +- `misc` is what every locale shares +- A register's canonical spelling loses to the one its domain writes +- `misc.territory` is the spine, and a `misc` table naming a territory links to it +- A table whose register publishes no frequency draws evenly +- `misc.timezone` weighs a zone by the people living in it +- `misc.car` is one flat table, not a make linked to its models +- `misc.territory` names its sovereign in a column, and there is no `misc.country` table +- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code +- `misc.tld` keys carry the leading dot, where other tables key on a bare code +- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column +- `misc.territory` carries a currency code, it does not link to `misc.currency` +- An extension may name two media types +- The Swedish ids draw Skatteverket's test series +- The US given names come from a mirror of the SSA file +- `List` advertises direct descents only +- A path draws through its compiled steps +- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read +- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0 +- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/` +- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape +- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}` +- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany +- A country with no open address register covering it whole ships its places and postal codes without streets +- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only +- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows +- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge + +# Scoring run + +The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`: + +- `{language}`: Go +- `{kind}`: a Go library and CLI that renders fake data from JSON templates +- `{domain}`: fake test-data generation and template languages +- `{domain docs}`: the documentation of any fake-data library, such as Faker +- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another diff --git a/CHANGELOG.md b/CHANGELOG.md index de8b1c8..c1568d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,85 +8,99 @@ replacement, and each removed path, column or flag. ### Added -- First release: the CLI, the library and the shipped data set. -- Table categories: a category JSON naming a `rows` TSV beside it, with the options - `key`, `name`, `weight` and `parent`; a path selects a row by key or name, - `misc.territory[SE]`, and descends to a linked table by name; linked tables draw - consistently within one render and draw group. `rows` is an option, so no - template may carry a field of that name. A `name` without a `key` resolves - inside the table's `parent`. Refused at `New`: a `name` without a `key` or a - `parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any - table above it, a table whose format or cell references its own family, and, - within one render and draw group, a path drawing a table another path selects a - row of, or two paths pinning different rows of one table. -- `New` refuses a root choice of templates sharing one format and one set of string - fields, naming the rows TSV to write instead. -- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and - `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has - a capital, a currency and a TLD, with the columns `calling-code`, `capital`, - `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country` - holding the alpha-2 code of the sovereign state it belongs to and its own where it - is one; `misc.currency` the - current ISO 4217 currencies with a minor unit, with `decimals` and `numeric` - added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a - 639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each - code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each - type mime-db records as IANA-registered and gives a filename extension. - `DATA-LICENSES.md` lists each source. -- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone` - is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the - zone's standard UTC offset and `territory` linking to `misc.territory`, so - `misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory - `misc.territory` does not ship, Antarctica's among them, and the constant `UTC` - are gone; spell that one as the text `"UTC"`. A `weight` column carries the - population GeoNames records in each zone, so a draw favours the zones people live - in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`, - `device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a - `model` column. -- `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries as tables. - `misc.httpmethod` is the eight methods RFC 9110 defines and PATCH, carrying - `safe` and `idempotent` as `true` or `false`; `misc.protocol` the keyword, its - `number` and its `name`, selectable by either; `misc.port` renders a TCP port number, - with the IANA `service` name beside it, keeping the first service the registry - describes for a port so a number selects one row. -- `misc.tld` is the IANA root zone as a table: every delegated TLD keyed with its - leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode` - column holding the form the register displays, so `misc.tld[.рф]` selects the row - `.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship. -- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes, - `err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by - either, with the `severity` RFC 5424 spells beside it. -- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`, - `locality`, `postal-code` and `street`, weighted by population and address counts - and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an - `address` record over one consistent draw of them. `sv_SE.address` and - `en_US.address` read those records, so `en_US.address.street` no longer carries - `name` and `suffix`, and a locale folder loads only beside `geo`. -- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or - within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`. - `sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so - `date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`, - `time.sec` and `time.ampm` are no longer paths — a part of a date is now its own - `{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant. -- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by - bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a - name both sexes carry is a row under each. `person` reads them, so its columns are - `first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are - no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead. - `en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms` - without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the - same shape. -- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series - from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose - `ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the - ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so - `--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new - and carries the same three. `person.prefix` is null where a person has no title, - where it used to be an empty string, so `--format sql` writes `NULL` and a `string` - struct field reading it becomes `*string`. -- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can - answer from the rest. -- An error names a spelling that runs: a layout is named single-quoted and free of - its own quotes, a row of a table with no key is named as the path that selects it, - `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record - that gives it one. +- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt, + dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so + the CLI installs without Go. 0.1.0 ships them too; the README's + [Install](https://github.com/larvit/fejkdata#install) lists each command. + +## [0.1.0] - 2026-10-13 + +The first release: a Go library and a CLI that render locale-aware fake data from JSON +templates, and the data they ship with. The +[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part. +Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the +library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break, +and any release may change what a seed draws, so pin one exact version +([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)). + +### Added + +- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing: + `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed` + repeats a run's output while the version and data stay the same, and without it each + run takes a new seed; `-n` renders the template that many times, and `--list` + prints the paths the data offers. + `--data-path` layers a directory of your own data over the shipped set, and + `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes + a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime + error such as data that fails to load, and `2` on misuse. +- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target. + `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing + from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`; + without data, it fails and names `WithDataFS`. The same seed repeats the draws + while the version and data stay the same. `Fake` renders a path, and `FakeTemplate` + and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and + `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and + `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's + `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only, + and `IsTemplate` tells an inline template from a path. Data that fails to load gives + an error matching `ErrLoad`, and a record with no fields gives one matching + `ErrNoColumns`. +- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command: + `cli.Main(modules...)` runs it over the modules given, so + `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data. + That CLI names itself by the file it runs from, and its `--version` prints its own + module's version before fejkdata's. +- The template language + ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)). + Every value in a data file is a string, a choice, a template, a table, a number or a + boolean. A `{…}` token reads a sibling field, a name, or a reference from the data + root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`), + or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a + pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check + digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`, + `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and + transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the + grammar, could mean two things or cannot render a valid value. +- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)): + a template seen as columns. A column is a string unless it declares a `datatype`, + `integer`, `number` or `boolean`, and a `null` item draws it as null. +- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a + category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and + `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a + linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row, + `{/misc.timezone[Europe/Oslo]..territory.name}`. +- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)): + a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json` + manifest at a source's root may carry a `namespace`, the dot path the source mounts + under, and an `index`, with which each category loads on the first call reaching it. +- The shipped data, in Go packages that each export `FS` and `Modules()`: + `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`), + `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from + `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all. + [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md) + names each source. + - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`, + `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and + the Census Bureau. A person's first name and sex agree, and so does an `en_US` + person's title; `person.prefix` is null where a person has no title. + `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series. + `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the + columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may + assign. + - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the + HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR + symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's + zones weighted by the people living in each, user agents and cars, all as tables; + and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses, + ObjectIds and UUIDs. + - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`, + `locality`, `postal-code` and `street`, weighted by population and address counts + and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata, + and an `address` record that draws a street and a postal code inside one + locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort + (postal town) the import can place ships with all its streets. A US place ships + only if it is incorporated and holds most of some postal code's land. Of those + places, every state capital but Honolulu ships, as does every place of 100,000 + people or more and a tenth of all of them drawn by population. Each US place + carries a quarter of its streets, at least ten. diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 8da2f33..c49f799 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -9,12 +9,12 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` | | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` | | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` | -| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` | +| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` | | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` | | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` | | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` | | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` | -| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 12) | — | — | — | +| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — | | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` | | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | diff --git a/Dockerfile b/Dockerfile index 8f04a4a..767d5c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,43 @@ -# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI -# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source, -# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test -# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 . -ARG GO_VERSION=1.26.4 -FROM golang:${GO_VERSION} +ARG GO_VERSION=1.27.2 +FROM golang:${GO_VERSION} AS portable WORKDIR /app -# Module layer cached separately from source. COPY go.mod ./ RUN go mod download COPY . . RUN go vet ./... && \ go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \ - { unformatted="$(gofmt -l .)"; test -z "$unformatted" || \ - { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \ go test -race ./... + +FROM golang:${GO_VERSION} AS vulnerabilities +WORKDIR /app +COPY . . +RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json + +FROM scratch AS vulnerability-report +COPY --from=vulnerabilities /report.json / + +FROM python:3.14.8-slim AS release-tooling +COPY release-tooling/ /release-tooling/ +RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested + +FROM python:3.14.8-slim AS vulnerability-filing +COPY vulnerability-filing/ /vulnerability-filing/ +# The filer's test files an item into the plan itself. +COPY todo.md /todo.md +RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested + +FROM goreleaser/goreleaser:v2.18.3 AS packaging +WORKDIR /check +COPY release-tooling/goreleaser.yaml ./ +RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested + +FROM portable +# BuildKit skips a stage nothing copies from. +COPY --from=release-tooling /tested /tmp/release-tooling-tested +COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested +COPY --from=packaging /tested /tmp/packaging-tested +RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \ + { echo "unformatted files:"; echo "$unformatted"; exit 1; } diff --git a/LICENSE b/LICENSE index 3502de6..0fb135f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ MIT License Copyright (c) 2026 Timewave +Copyright (c) 2026 Larvit Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 32464e6..d39d351 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,136 @@ # fejkdata -Locale-aware fake data for tests and fixtures, generated from JSON templates. Use -it as a Go library or the CLI — no data on disk, no dependencies, and a seed makes output -reproducible. +[![Go Reference](https://pkg.go.dev/badge/github.com/larvit/fejkdata.svg)](https://pkg.go.dev/github.com/larvit/fejkdata) +[![Tests](https://github.com/larvit/fejkdata/actions/workflows/test.yml/badge.svg)](https://github.com/larvit/fejkdata/actions/workflows/test.yml) +[![Release](https://img.shields.io/github/v/release/larvit/fejkdata)](CHANGELOG.md) + +Locale-aware fake data for tests and fixtures, from JSON templates: a Go library and a CLI +with the data built in and no dependencies. ```sh -go install gitea.larvit.se/larvit/fejkdata/cmd/fejkdata@latest -fejkdata sv_SE.person # Sara Eriksson +go install github.com/larvit/fejkdata/cmd/fejkdata@latest +echo '{/sv_SE.person}' | fejkdata # Sara Eriksson ``` +- **Data built in:** the `sv_SE` and `en_US` locales, with names weighted by how common + they are. Every Swedish postal town and a sample of US places ship, and each address + takes its street and postal code from one town ([Data](#data)). +- **Valid ids:** personnummer and samordningsnummer from Skatteverket's test series, which + no real person holds; SSNs and ITINs in the ranges the SSA and the IRS assign, so a + drawn one may be a real person's; Luhn-valid card numbers; and IBANs for seven + European countries. +- **Registers:** ISO 3166-1 territories, ISO 4217 currencies, ISO 639 languages, IANA + time zones, TLDs, ports, HTTP status codes, media types and more, as tables you + select a row from. +- **Bulk output:** records as JSON, NDJSON, CSV or SQL `INSERT`s, streamed. +- **Go structs:** `FakeStruct` fills a struct from its `fake:"…"` tags. +- **Your own data:** each JSON file in a folder you pass adds a path, such as + `{/acme.customer}`, and a mistake in it fails at load, never mid-render. +- **Reproducible:** the same seed, version and data give the same output, when one goroutine + draws it. + +[Install](#install) · [CLI](#cli) · [Library](#library) · [Data](#data) · [Data format (templates)](#data-format) · +[Functions](#functions) · [Versioning](#versioning) · [Audience](#audience) · +[Goals](#goals) · [Development](#development) · [License](#license) + +## Install + +```sh +brew install larvit/tap/fejkdata # macOS, Linux +scoop bucket add larvit https://github.com/larvit/scoop-bucket && scoop install fejkdata # Windows +docker run --rm -i larvit/fejkdata < template.json # any OS with Docker +go install github.com/larvit/fejkdata/cmd/fejkdata@latest # any OS with Go +``` + +On shared CI runners, pull `ghcr.io/larvit/fejkdata` instead: Docker Hub limits anonymous +pulls per IP address. + +Debian, Ubuntu, Fedora, RHEL and Alpine install from fejkdata's signed repositories, which +hold the latest release: + +```sh +# Debian, Ubuntu +sudo curl -fsSLo /etc/apt/keyrings/fejkdata.asc https://larvit.github.io/fejkdata/fejkdata.asc +echo 'deb [signed-by=/etc/apt/keyrings/fejkdata.asc] https://larvit.github.io/fejkdata/deb stable main' | sudo tee /etc/apt/sources.list.d/fejkdata.list +sudo apt update && sudo apt install fejkdata + +# Fedora, RHEL +sudo curl -fsSLo /etc/yum.repos.d/fejkdata.repo https://larvit.github.io/fejkdata/rpm/fejkdata.repo +sudo dnf install fejkdata + +# Alpine, as root +wget -qO /etc/apk/keys/fejkdata.rsa.pub https://larvit.github.io/fejkdata/apk/fejkdata.rsa.pub +echo https://larvit.github.io/fejkdata/apk >> /etc/apk/repositories +apk add fejkdata +``` + +Each [release](https://github.com/larvit/fejkdata/releases) also carries the binaries for +Linux, macOS and Windows on amd64 and arm64, and the `.deb`, `.rpm` and `.apk` packages. +The binaries are unsigned. macOS refuses to run one a browser downloaded until you run +`xattr -d com.apple.quarantine fejkdata`. + ## CLI ```sh -fejkdata sv_SE.person # Sara Eriksson -fejkdata sv_SE.person.last # Eriksson -fejkdata --seed 42 sv_SE.address # the same address every run -fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol -fejkdata --list # every path the data offers -fejkdata 'misc.territory[SE].capital' # Stockholm — a table's row, selected by key or name -fejkdata 'geo.SE.locality[Lund].street' # Fjelievägen — a linked table, drawn inside the row -fejkdata --data-path ./mydata sv_SE.word # layer a directory over the shipped data -fejkdata --no-shipped-data -d ./mydata --list # only your data -fejkdata 'name: {/sv_SE.person.last}' # name: — an inline template -fejkdata "{date(1990-01-01,2010-12-31,'2006-01-02')}" # 2003-11-27 — the argument in "…", the layout in '…' -fejkdata '{"format":"name: {x}","x":["bosse","lina"]}' # name: bosse or name: lina +echo '{/sv_SE.person}' | fejkdata # Sara Eriksson +echo '{/sv_SE.person.last}' | fejkdata # Eriksson +echo '{/sv_SE.person} hihi' | fejkdata # Sara Eriksson hihi +echo '{/sv_SE.address}' | fejkdata --seed 42 # the same address every run +echo '{/sv_SE.word}' | fejkdata -n 3 # nät, barn and sol, a line each +printf '%s' '{/sv_SE.word}' | fejkdata -n 3 --separator ', ' # nät, barn, sol +fejkdata --list # every path the data offers +echo '{/misc.territory[SE].capital}' | fejkdata # Stockholm — a table's row, selected by key or name +echo '{/geo.SE.locality[Lund].street}' | fejkdata # Fjelievägen — a linked table, drawn inside the row +echo '{/sv_SE.word}' | fejkdata --data-path ./mydata # layer a directory over the shipped data +fejkdata --no-shipped-data -d ./mydata --list # only your data +echo '{"format":"name: {x}","x":["bosse","lina"]}' | fejkdata # name: bosse or name: lina, with no trailing newline; see What stdin renders +fejkdata < template.txt # a template kept in a file +fejkdata <<'EOF' # born 2003-11-27 — a quoted heredoc passes ', $ and \ as written +born {date(1990-01-01,2010-12-31,'2006-01-02')} +EOF ``` -A path names a category, or a field inside one: each dot segment descends one -level — folders, then the category (a JSON file), then fields — and `[SE]` after a -[table](#table) selects its row. An argument that is -a JSON object, array or string, or that carries a `{` token, is instead an -**inline template**: a format string or a JSON value compiled and rendered on the -spot. Its tokens reach the data by reference from the root — -`{/sv_SE.person.last}`, so shipped and `--data-path` categories are alike -available. A path never contains a brace or a quote, and a bracket only as a selector -after a name, so the two spellings cannot collide; which spellings an inline template -rejects, and what each names instead, is under [Decisions](#decisions). +fejkdata renders the template on its stdin: a format string, or a JSON object, array, string or `null`. Its `{…}` tokens reach the data by reference from the root, +`{/sv_SE.person.last}`, so shipped and `--data-path` categories are alike available, +and everything outside them prints as written. A reference names a category, or a field +inside one: each dot segment descends one level — folders, then the category (a JSON +file), then fields — and `[SE]` after a [table](#table) selects its row. `--list` prints +these paths; a template reads one as `{/path}`. | Flag | | |------|--| | `-d`, `--data-path D` | a directory to layer over the shipped data; repeatable, the last wins a name clash | | `--no-shipped-data` | load only the `--data-path` directories | -| `-s`, `--seed N` | reproducible output | -| `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | -| `--separator S` | between repeated values (default a newline) | +| `-s`, `--seed N` | same seed, version and data: identical output (default: a random seed) | +| `-n`, `--repeat N` | render the template N times (0 to 1048576), each an independent draw, streamed; `0` prints nothing, `[]` under `--format json`, or the header under `--format csv` | +| `--separator S` | between repeated renders (default empty); refused with a record `--format`, which has its own separators | | `--format F` | `text` (default), `json`, `ndjson`, `csv` or `sql` — a record's columns, one record per row (json frames them as an array) | -| `--table T` | the INSERT target for `--format sql` (default: the path's last segment, or `records` for an inline template) | -| `--list` | print every path, then exit | +| `--table T` | the INSERT target for `--format sql`, ignored under any other (default: a lone reference's last segment, such as `users` for `{/users}`, else `records`) | +| `--list` | print every path, then exit; `--repeat`, `--separator`, `--format` and `--table` do nothing here | | `--version`, `-h`, `--help` | print, then exit | `--name value` and `--name=value` both work, a short flag's value attaches or follows (`-n3`, `-n 3`) and short flags bundle (`-hn 3`) — see -[Decisions](#decisions); flags go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing -dir, unknown path), `2` misuse — a bad flag, an argument that names neither a -template nor a path, or an inline template that does not compile. From a checkout: +[Decisions](docs/decisions.md#flags-follow-getopt_long); flags go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (data +that fails to load, a missing `-d` folder included, a lone reference to nothing under `--format`), `2` misuse — a bad flag, an argument +other than a flag, an unreadable stdin, nothing piped in, or a +template that does not compile. From a checkout: `go run ./cmd/fejkdata …`. +### What stdin renders + +Under `--format text`, the default, what you send is what renders, and fejkdata adds nothing. A format string keeps every +byte, so `echo`, a file or a heredoc ends each render with its newline, and `printf '%s'` +sends none; `-n` joins renders with `--separator`, empty by default. A JSON object, array +or string drops the whitespace around it, and `null` alone renders nothing; `42` or `true` +alone is a format string, so it prints as written, newline included. An empty or blank stdin is a format string like +any other, and renders as sent. A JSON template that should end +each render in a newline writes `\n` in its format, from a quoted heredoc or a file. A +lone reference, one `{/…}` and nothing else, is the record it names under `--format`, one +newline after it allowed. Some shells' `echo` reads a backslash as an escape, so pipe a +template holding one from a quoted heredoc or a file. The rest of stdin's rules are under +[Decisions](docs/decisions.md#the-cli-renders-exactly-the-template-on-its-stdin-and-adds-nothing). + ### Your own data A category is a JSON file in a directory. Save this as `mydata/sql.json`: @@ -73,9 +148,9 @@ A category is a JSON file in a directory. Save this as `mydata/sql.json`: ``` ```sh -fejkdata --seed 1 --data-path ./mydata sql +echo '{/sql}' | fejkdata --seed 1 --data-path ./mydata # INSERT INTO users VALUES('zoom'),('wahoo'),('blip'); -fejkdata --repeat 100 --data-path ./mydata sql > seed.sql +echo '{/sql}' | fejkdata --repeat 100 --data-path ./mydata > seed.sql ``` That `format` string is the free-form spelling — you hand-write the whole row. @@ -98,17 +173,17 @@ the whole. `--format json|ndjson|csv|sql` writes the records; the library's ``` ```sh -fejkdata --seed 1 --data-path ./mydata --format json users # [{"first":"Bo","last":"Lovelace"}] -fejkdata --seed 1 --data-path ./mydata --format ndjson users # {"first":"Bo","last":"Lovelace"} -fejkdata --seed 1 --data-path ./mydata --format csv users # first,last → Bo,Lovelace -fejkdata --seed 1 --data-path ./mydata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace'); -fejkdata --seed 1 --data-path ./mydata --format sql --table people users # INSERT into another table +echo '{/users}' | fejkdata --seed 1 -d ./mydata --format json # [{"first":"Bo","last":"Lovelace"}] +echo '{/users}' | fejkdata --seed 1 -d ./mydata --format ndjson # {"first":"Bo","last":"Lovelace"} +echo '{/users}' | fejkdata --seed 1 -d ./mydata --format csv # first,last → Bo,Lovelace +echo '{/users}' | fejkdata --seed 1 -d ./mydata --format sql # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace'); +echo '{/users}' | fejkdata --seed 1 -d ./mydata --format sql --table people # INSERT into another table ``` `--repeat` streams that many records — `json` frames them as one array document, `ndjson` writes one object per line, `csv` a row after a header, `sql` one INSERT -per line. Only a category-level template is a record; a field, choice or folder -errors, and so does a `repeat` on +per line. Only a category-level template is a record, or a one-item choice holding one; a +field, a choice of more items or a folder errors, and so does a `repeat` on the template itself, which composes the format into one string rather than projecting columns — ask for more records with `--repeat`. A `repeat` on a column is fine. @@ -130,48 +205,223 @@ valid identifier. PostgreSQL and SQLite take it as written; MySQL and MariaDB ne string and a backslash as an escape. A record written only to emit columns still needs a `format` — the grammar's one -required key — so `"format": ""` carries the fields with an inert format: it -renders nothing by `Fake`, and is compiled only so the tree's fences still run. -The columns are the point, and their facts stay together: a record is one render, so -columns that read a path into one category — `{/currency.code}` and -`{/currency.symbol}` — read one draw of it ([References](#references)), and a -[draw group](#draw-group) draws a column apart. That one draw is also why the columns of one -record may not overlap — `{/cat.a}`, or a bare `{/cat}`, beside -`{/cat.a.b}` is refused, naming the fields to write instead, as -[One draw, one spelling](#one-draw-one-spelling) refuses that pair inside a single -format. Both fences run at load, so a category that loads renders as either shape. A -category never references itself — `{/users.first}` or a bare `{/users}` inside `users` -describes a draw other than the fields beside it — so read a sibling as a path, and put -a value two fields share in its own category and reference that. A field hold, transform or -operand ties fields together within one column as always (see -[Correlated fields](#correlated-fields) and [Decisions](#decisions)). +required key — so `"format": ""` carries the fields with an inert format: `Fake` renders +it as nothing. + +A record is one render, so its columns read one pick of each [name](#names) its category +binds: `"code": "{/currency as c}{c.code}"` beside `"symbol": "{c.symbol}"` is one +currency, while `{/currency.code}` and `{/currency.symbol}` in two columns may name two. A +column the format binds to a name, as `{place as p}` binds `place`, holds that same pick, so it +agrees with the columns reading `p`; a column bound only in part, `{place.town as t}`, draws +afresh, so bind it whole and read `{p.town}`. + +A reference into a category's own fields, `{/users.first}` inside `users`, is a draw of its +own, apart from the fields beside it; to share one draw, read the sibling as a field, or bind +it to a name and read the name +([Decisions](docs/decisions.md#a-category-may-reference-itself-and-a-records-fences-run-at-load)). + +## Library + +```sh +go get github.com/larvit/fejkdata # Go 1.22+, 64-bit targets +``` + +```go +import ( + "github.com/larvit/fejkdata" + svse "github.com/larvit/fejkdata/data/sv_SE" +) + +f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(42)) +if err != nil { + log.Fatal(err) +} +v, err := f.Fake("sv_SE.address") // "Trekungagatan 43\n442 42 Kungälv" +paths := f.List() // the paths Fake renders, sorted; direct descents only +v, err = f.FakeTemplate("name: {/sv_SE.person.last}") // compile + render in one call +t, err := f.NewTemplate(`{"format":"name: {x}","x":["bosse","lina"]}`) // compile once +v = t.Fake() // render many times, no re-parse +r, err := f.FakeRecord("users") // one record: each field a column +s := r.JSON() // {"first":"Ada","last":"Lovelace"} +r, err = f.FakeRecordTemplate(`{"format":"{x}","x":["a","b"]}`) // compile + render inline +rt, err := f.NewRecordTemplate(`{"format":"","id":{"format":"{seq(users)}","datatype":"integer"}}`) // compile once +r = rt.Fake() // render a row; loop this for bulk output +err = f.FakeStruct(&user) // fill a struct's fake:"…" tagged fields +ok, err := fejkdata.IsTemplate(arg) // an inline template by its shape, else a path +``` + +| Option | | +|--------|--| +| `WithSeed(n)` | required; the same seed, version and data give the same sequence. For a new seed each run, set `n := rand.Uint64()` (`math/rand/v2`), and log `n` to replay the run | +| `WithDataPath(dir)` | layer a directory; repeat to layer several, the last wins a clash | +| `WithDataFS(modules...)` | layer modules in order, each an `fs.FS`: a shipped package's `Modules()...`, or your own `embed.FS`; `WithDataFS()` loads none | + +A shipped package's `Modules()` returns its own `FS` and every module it reads, such as a +locale's country's places ([Data](#data)); `misc.*` comes from `misc.Modules()`, in +`github.com/larvit/fejkdata/data/misc`. Your program links only the packages it imports. +`data.Modules()`, from `github.com/larvit/fejkdata/data`, is every shipped module, and +links all of them. Two locales, with `enus` from `github.com/larvit/fejkdata/data/en_US`, +are `WithDataFS(append(svse.Modules(), enus.Modules()...)...)`. To stand in your +own places, pass the locale's `FS` alone beside a tree providing `geo.SE.address`, a record +with a `format` and the columns `sv_SE.address` reads, `street`, `street-number`, +`postal-code` and `locality`: `WithDataFS(svse.FS, myPlaces)`. `New` fails where no +`WithDataFS` or `WithDataPath` names the data to load. + +`New` refuses data that breaks the grammar, could mean two things or cannot render a valid +value, and `NewTemplate` refuses such a template. A category of an [indexed](#data) source is +checked on the first call reaching it instead. Data that fails to load, in `New` or on +that first call, gives an error matching `ErrLoad`. On a loaded generator: + +- `Fake` fails only for a path that names nothing, could name two things or reads one + draw of a level carrying a `repeat`, or for data that fails to load (`ErrLoad`), with + the same error every call. +- `FakeStruct` fails only for a non-struct argument, a type its tags do not describe, or + data that fails to load (`ErrLoad`), with the same error every call. +- `Template.Fake` cannot fail. + +A path, wherever a call takes one, may start with one `/`, as a reference does. + +A `*Record` carries its columns via `Columns()` — each a `Column` of `Name`, +`DataType`, rendered `Value` and `Null` — and serializes them with `JSON()` +(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the shapes the +CLI's `--format` writes, each without a trailing newline. `FakeRecord` and +`FakeRecordTemplate` take a record; a path or template that is not one — a bare string, a +choice of more than one item, or a folder — errors. + +```go +type User struct { + ID int64 `fake:"{seq()}"` + Last string `fake:"sv_SE.person.last"` + Age uint8 `fake:"{int(18,99)}"` + Nick *string `fake:"[null, \"{/sv_SE.username}\"]"` + Home Address // filled from Address's own tags +} + +type Address struct { + Street string `fake:"{/sv_SE.address as a}{a.street} {a.street-number}"` + PostalCode string `fake:"{a.postal-code}"` // the same address's code + Locality string `fake:"{a.locality}"` +} +``` + +`FakeStruct` fills a struct through a pointer: each exported field tagged `fake:"…"` is +a column of one record, its tag a path or an inline template — told apart by +`IsTemplate` — and its Go type the column's +[datatype](#datatype): a string, bool, integer or float kind, or a pointer to one. A +column that can be null needs a pointer, which a [`null`](#null) item leaves nil. An +integer stays within int64 whatever its kind, and a value the kind cannot hold, such as +`{int(0,300)}` in a `uint8`, is refused naming a kind that holds it. The tags are a record's columns, so they share its +[names](#names): `First` tagged `{/sv_SE.person as p}{p.first}` and `Sex` tagged `{p.sex}` +describe one person, while two path tags into `sv_SE.person` are two draws. The fields an +embedded struct promotes are columns of the same record; a named struct field, or a +pointer to one, fills from its own tags as a record of its own, so its names and picks are +its own. `fake:"-"` leaves a field unfilled. +Untagged fields keep their values, and so does a pointer back to a struct already being filled; a type +whose fields reach more than 1024 structs is refused, naming `fake:"-"` to cut it. The +first call for a type compiles its tags and reports what they get wrong, with the same +error on every later call; a `datatype` in a tag other than the one its Go type sets is +refused, naming that type. + +A `*Generator` is safe for concurrent use; a sequence is reproducible only +when drawn from one goroutine. Editing a template shifts what it draws, and +everything drawn after it. + +### Your own CLI + +Package `cli` is the `fejkdata` command, and its `Main` takes the modules the command +loads, so a CLI carrying your own data is a `main` package of your own: + +```go +package main + +import ( + "embed" + "os" + + "github.com/larvit/fejkdata/cli" + "github.com/larvit/fejkdata/data" +) + +//go:embed *.json +var mydata embed.FS + +func main() { os.Exit(cli.Main(append(data.Modules(), mydata)...)) } +``` + +The embed's folders are the paths: `//go:embed acme` serves `acme/customer.json` as +`{/acme.customer}`. Add `*.tsv` to the embed where your data holds a table. The CLI loads +its data only when it runs. To catch broken data at `go test`, call +`fejkdata.New(fejkdata.WithDataFS(...), fejkdata.WithSeed(1))` over the same modules, and +render each path once: a module with an [index](#data) checks a category only when a call first +reaches it. The CLI takes `fejkdata`'s flags and exit codes, and names itself by the file +it runs from in its help and errors. `--no-shipped-data` leaves out the modules passed to +`Main`, and `--version` prints your module's version beside the fejkdata version it is built with. ## Data -The shipped set under [`data/`](data) — one folder per locale (`en_US`, `sv_SE`) -plus a locale-neutral `misc` folder — is embedded, so the CLI and the library -work with no data on disk. A directory is a namespace: each JSON file is a -category named after the file, each subdirectory a dot-path segment, so -`mydata/sv_SE/person.json` is `sv_SE.person` and replaces the shipped one. +The shipped data comes in modules, each a Go package in the folder under [`data/`](data) +its namespace spells, a dot a slash: one per locale (`data/en_US`, `data/sv_SE`), one per +country's tree of places (`data/geo/SE`, `data/geo/US`), and the locale-neutral +`data/misc`. A package's Go name is its namespace in lower case without separators: +`svse`, `enus`, `geose`, `geous`, `misc`. Each embeds its files, so nothing is read from +disk. The CLI carries every module, and a library loads those it passes to `WithDataFS` +([Library](#library)). A directory is a namespace: each JSON file is a category named +after the file, each subdirectory a dot-path segment, so `mydata/sv_SE/person.json` is +`sv_SE.person` and replaces the shipped one. Sources merge in order; matching folders combine, any other clash is won by the -last loaded. Names may not use `.`, `|`, `(`, `{`, `}`, `[`, `]`, `"` or `/`, nor be -`-`, which a struct tag reserves; dot-prefixed entries are skipped, so a data directory can also be a checkout. +last loaded. Names may not use `.`, `|`, `(`, `{`, `}`, `[`, `]`, `"`, `/` or ` as `, +which binds a [name](#names), nor be `-`, which a struct tag reserves. A dot-prefixed +entry is never a category or a folder, so a data directory can also be a checkout, though +a category may name a hidden rows file beside it. + +A source whose root holds `.fejkdata.json` with an `index` loads each category on the +first call reaching it, so `New` does not check them: load each category once in a test, +as CI does for the shipped set. Any other source loads in `New`, with the indexed +categories it reads. An indexed category reading one that another source replaced loads +on its own first reach, and fails there if the replacement breaks it. The index names +every category of its source, so a category file it leaves out never loads. + +The index maps each category's dot path to an entry: `parent`, the table it links to, +which needs its own entry in the same folder; and `paths`, every path `List` advertises +below the category, `""` for the category itself. Each shipped module's manifest, such as +[`data/sv_SE/.fejkdata.json`](data/sv_SE/.fejkdata.json), carries one, so in a copy you +edit, delete its `index` and keep its `namespace`, or keep the index current. +A table whose parent column differs from its entry's `parent` fails at first reach, and a +table linking under another source's table needs a source without an index. `//go:embed` +of a directory leaves the manifest out, and the source then loads whole in `New` and at +the root, with no `namespace`; name the manifest in the pattern too. + +A manifest's `namespace` is the dot path its source's root sits under: a folder holding +`person.json` with `"namespace": "sv_SE"` provides `sv_SE.person`, its subfolders adding +segments below that. It needs no index. Each shipped module carries one, so +`data/sv_SE/person.json` is `sv_SE.person`. Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name`, `ip`, `last-name`, `person`, `phone`, `price`, `sentence`, `sex`, `time`, `url`, -`username`, `version` and `word`, formatted per locale; `sv_SE` adds -`personnummer` and `samordningsnummer`, `en_US` adds `ssn` and `itin`. `misc` -carries `car`, `coordinate`, `creditcard` (Luhn-valid), `currency` (ISO 4217), -`datetime` (RFC 3339), `emoji`, `httpmethod`, `httpstatus`, `language` (ISO 639-1, -with its 639-2/T code), `loglevel` (syslog severity), `mac`, `mimetype`, `objectid`, -`port`, `protocol`, `territory` (ISO 3166-1), `timezone` (IANA), `tld` (IANA root -zone), `useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, -`misc.territory.alpha2`, `misc.httpstatus.code` — which `--list` shows. `car`, -`currency`, `httpmethod`, `httpstatus`, `language`, `loglevel`, `mimetype`, `port`, -`protocol`, `territory`, `timezone`, `tld` and `useragent` are [tables](#table), so +`username`, `version` and `word`, formatted per locale; `sv_SE` adds `personnummer` +and `samordningsnummer`, `en_US` adds `ssn`, `itin` and `phone-exchange` (the three +digits after the area code). `en_US.phone` draws its exchange from `en_US.phone-exchange`, +which draws evenly from the codes the North American Numbering Plan Administration (NANPA) +may assign: `200` to `999`, except `211`, `311` … `911`, `555`, `950`, `958`, `959` and +`976`. `misc` carries `car`, `coordinate`, `creditcard` +(Luhn-valid), `currency` (ISO 4217), `datetime` (RFC 3339), `emoji`, `httpmethod`, +`httpstatus`, `language` (ISO 639-1, with its 639-2/T code), `loglevel` (syslog +severity), `mac`, `mimetype`, `objectid`, `port`, `protocol`, `territory` (ISO +3166-1), `timezone` (IANA), `tld` (IANA root zone), `useragent` and `uuid` (v4). Many +carry sub-fields — `misc.currency.symbol`, `misc.territory.alpha2`, +`misc.httpstatus.code` — which `--list` shows. `car`, `currency`, `httpmethod`, +`httpstatus`, `language`, `loglevel`, `mimetype`, `port`, `protocol`, `territory`, +`timezone`, `tld` and `useragent` are [tables](#table), so `misc.territory[SE].capital` and `misc.currency[Euro].symbol` select a row; `car` and `useragent` carry no key or name, so they are drawn from rather than selected in. +Where a script under [`data-import/`](data-import) reads a source, the facts come from +that source, and the script drops a row only by a rule it states; elsewhere a small hand-written +set ships, and [`todo.md`](todo.md) carries the step that replaces it (goal 13). Copy +that states no fact, such as `word` and `sentence`, stays hand-written. Where a category +has no path that draws only values reaching nothing real (goal 15), write a fixed value such +as `example.com` into the template; `todo.md` plans the path. + `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries. `misc.httpmethod` is the eight methods RFC 9110 defines and PATCH; the register's other entries, WebDAV and DeltaV among them, do not ship. Its `safe` and @@ -221,109 +471,62 @@ code for a sovereign one, or for one the register names no state for. SSA and the Census Bureau. `first-name` links to `sex`, so `sv_SE.sex[f].first-name` draws a woman's name, and a name both sexes carry is a row under each, so `en_US.sex[m].first-name[Taylor]` names the one a `first-name[Taylor]` alone cannot. -`person` reads one draw of the three, so its `first` and `sex` columns agree, and so -does a `personnummer` in the same render: its birth number, `sv_SE.birth-number` -under `sex`, is Skatteverket's test series, 238 for a woman and 239 for a man, which no -real person is ever given. `en_US.title` links to `sex` too, so a person's prefix -never contradicts it. +`person` binds one first name and reads its sex through it, `{.first-name as n}` and +`{n..sex.name}`, so its `first` and `sex` columns agree. `en_US.title` links to `sex` +too, so a person's prefix never contradicts it. A `personnummer`'s birth number, +`sv_SE.birth-number` under `sex`, is Skatteverket's test series, 238 for a woman and 239 +for a man, which no real person is ever given. A person of a chosen sex is assembled from the tables — `sex[f].first-name` beside -`last-name` — while a shipped `personnummer` agrees with the sex its own render -*drew*, not with one a path selects. That test series is also small: a personnummer -is one of about 70,000 values, a day in 1930–2025 against the two birth numbers, so a -fixture past a few hundred rows repeats one and a `UNIQUE` column needs a category of -your own. `sv_SE.date` and `en_US.date` are uniform over 1970-01-01 to 2029-12-31, -`misc.datetime` over 2000-01-01 to 2029-12-31. What the two locales do not share: -`en_US.address` carries a `region` column the Swedish one has no use for, and -`sv_SE.title` has no `parent`, so it is selected as `sv_SE.title[dr]` rather than -inside a sex. +`last-name`. `person` and `personnummer` draw apart, so where a name and a number must +agree, read both through one bound `sex`, `{/sv_SE.sex[f] as s}` for a woman: -A `geo` folder holds one tree per country under its alpha-2 code: five -[linked tables](#linked-tables) named alike, and an `address` record over one -consistent draw of them, which the locale's `address` reads. +```json +{ "format": "{first} {last}, {pnr}", + "first": "{/sv_SE.sex as s}{s.first-name}", + "last": "{/sv_SE.last-name}", + "pnr": "{date(1930-01-01,2025-12-31,'060102')}-{s.birth-number.number}{luhn()}" } +``` + +Renders e.g. `Anne Grönberg, 011012-2389`. + +Skatteverket's test series is small: a personnummer is one of about 70,000 values, a day +in 1930–2025 against the two birth numbers, so a fixture past a few hundred rows repeats +one and a `UNIQUE` column needs a category of your own. `sv_SE.date` and `en_US.date` +are uniform over 1970-01-01 to 2029-12-31, `misc.datetime` over 2000-01-01 to 2029-12-31. +What the two locales do not share: `en_US.address` carries a `region` column the Swedish +one has no use for, and `sv_SE.title` has no `parent`, so it is selected as +`sv_SE.title[dr]` rather than inside a sex. + +The `geo` namespace holds one tree per country under its alpha-2 code: five +[linked tables](#linked-tables) named alike, and an `address` record binding one locality +and drawing its street and postal code inside it. The locale's `address` prints that +record, and each of its columns describes the address printed. Do not assert that a Swedish +street lies in its postort: the import counts each road segment toward the shipped postort +owning the postal code whose centre point lies nearest, so a street may sit under a +neighbouring one. | Table | `geo.SE` | `geo.US` | Weight | |-------|----------|----------|--------| | `region` | län, by code or name | state, by USPS abbreviation or name; `code` is the FIPS code | population | | `municipality` | kommun, by code or name | county, by FIPS code or name | population | -| `locality` | postort, by name | incorporated place of 25,000 people or more with a postal code of its own, by GEOID or name; Hawaii has none | tätort population, the kommun's where the postort names it, else 200; place population | +| `locality` | postort, by name; [`geo-se.py`](data-import/geo-se.py)'s `localities` picks the candidates, and one with no street is dropped | incorporated place, by GEOID or name; Hawaii has none | tätort population, the kommun's where the postort names it, else 200; place population | | `postal-code` | postnummer with street delivery, by code | ZCTA, by code | one; address ranges | -| `street` | gatunamn, the ten with most road segments per postort | street name, the ten with most address ranges per place | segments; address ranges | +| `street` | gatunamn, every one [`geo-se.py`](data-import/geo-se.py)'s `street_name` accepts | street name, per place the quarter with most address ranges, at least ten | segments; address ranges | `geo.SE.region[Skåne län].municipality` draws a kommun in Skåne, `geo.SE.locality[Lund].street` a street in Lund, and -`geo.US.region[IL].locality[Springfield]` settles which Springfield. A region row +`geo.US.region[IL].locality[Springfield]` settles which Springfield. The US places come +from the incorporated places that hold most of some postal code's land: a tenth of them, +drawn with odds by population, every state capital but Honolulu, and every place of +100,000 or more. A smaller city may be missing, so look one up in +[`locality.tsv`](data/geo/US/locality.tsv) before selecting it. A region row carries its `timezone`, the state's predominant zone, and a locality its `lat` and `lon`. What ports across countries is the five table names, the `name` column, selection by name, and the `address` record's columns `street`, `street-number`, `postal-code` and `locality`; every other column is the country's own, `code` on a Swedish region but `abbr` on a US one. -## Library - -```sh -go get gitea.larvit.se/larvit/fejkdata # Go 1.22+ -``` - -```go -f, err := fejkdata.New(fejkdata.WithSeed(42)) -if err != nil { - log.Fatal(err) -} -v, err := f.Fake("sv_SE.address") // "Järvedsvägen 43\n891 77 Järved" -paths := f.List() // every path Fake accepts, sorted -v, err = f.FakeTemplate("name: {/sv_SE.person.last}") // compile + render in one call -t, err := f.NewTemplate(`{"format":"name: {x}","x":["bosse","lina"]}`) // compile once -v = t.Fake() // render many times, no re-parse -r, err := f.FakeRecord("users") // one record: each field a column -s := r.JSON() // {"first":"Ada","last":"Lovelace"} -r, err = f.FakeRecordTemplate(`{"format":"{x}","x":["a","b"]}`) // compile + render inline -err = f.FakeStruct(&user) // fill a struct's fake:"…" tagged fields -ok, err := fejkdata.IsTemplate(arg) // an inline template by its shape, else a path -``` - -| Option | | -|--------|--| -| `WithSeed(n)` | same seed, same data: identical sequence | -| `WithDataPath(dir)` | layer a directory; repeat to layer several, the last wins a clash | -| `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | -| `WithoutShippedData()` | load only what you give | - -A `*Record` carries its columns via `Columns()` — each a `Column` of `Name`, -`DataType`, rendered `Value` and `Null` — and serializes them with `JSON()` -(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the shapes the -CLI's `--format` writes. `FakeRecord` and `FakeRecordTemplate` take a record; a -path or template that is not one — a bare string, a choice, or a folder — errors. - -```go -type User struct { - ID int64 `fake:"{seq()}"` - Last string `fake:"sv_SE.person.last"` - Age uint8 `fake:"{int(18,99)}"` - Nick *string `fake:"[null, \"{/sv_SE.username}\"]"` - Home Address // filled from Address's own tags -} -``` - -`FakeStruct` fills a struct through a pointer: each exported field tagged `fake:"…"` is -a column of one record, its tag a path or an inline template — told apart by -`IsTemplate`, as the CLI tells an argument — and its Go type the column's -[datatype](#datatype): a string, bool, integer or float kind, or a pointer to one, -which a [`null`](#null) item leaves nil. An integer stays within int64 whatever its -kind, and a value the kind cannot hold, such as `{int(0,300)}` in a `uint8`, is refused -naming a kind that holds it. The fields an embedded struct promotes are columns of the -same record; a named struct field, or a pointer to one, fills from its own tags as a -record of its own, so its references draw apart from its parent's. `fake:"-"` leaves a -struct field, embedded or named, or a pointer to one, unfilled. Untagged fields keep -their values, and so does a pointer back to a struct already being filled; a type -whose fields reach more than 1024 structs is refused, naming `fake:"-"` to cut it. The -first call for a type compiles its tags and reports what they get wrong, with the same -error on every later call; a `datatype` in a tag names the Go type that already sets it. - -A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only -when drawn from one goroutine. Changing how a value is composed shifts the seeded -stream for that value and everything drawn after it. - ## Data format Every value is a **node**, nestable without limit: @@ -334,6 +537,12 @@ Every value is a **node**, nestable without limit: | choice | `["a", "b", …]` | one item, picked at random | | template | `{"format": "…", …}` | its format, with `{name}` tokens rendering the named fields | | table | `{"format": "…", "rows": "x.tsv", …}` | its format over one row of the TSV beside it ([Table](#table)) | +| number | `5`, `1.50` | its text as written | +| boolean | `true` | its text as written | + +In a record column a bare number or boolean is refused, since it could be the text `"5"` +or the typed value `{"format": "5", "datatype": "integer"}`. In a struct tag, the +field's Go type decides. ### Format string @@ -341,21 +550,25 @@ Every character is literal except a `{…}` token: | Token | Renders | |-------|---------| -| `{name}` | the sibling field `name` | -| `{name.field}` | `field` of one draw of `name` ([Correlated fields](#correlated-fields)) | +| `{name}` | the sibling field `name`, or the pick bound to the name ([Names](#names)) | +| `{x as name}` | nothing: binds one pick of `x`, a reference, a sibling field or a path into one, to `name` ([Names](#names)) | +| `{name.field}` | `field` of a fresh draw of the sibling `name`, or of the pick bound to it ([Correlated fields](#correlated-fields)) | | `{a\|b}` | one of the named fields, even odds | | `{fn(args)}` | a builtin ([Functions](#functions)) | | `{/path}`, `{.path}`, `{..path}` | a node reached from the data root, this file's folder, or the folder above ([References](#references)) | | `{{`, `}}` | a literal `{` or `}` | +Each `{…}` draws afresh, so `{w} {w}`, `{p.a} {p.b}` and `{/person.first} {/person.last}` +are two draws each. Only a [name](#names) keeps one pick for every read of it. + ```json "100 Main St, Apt {int(1,9)}{upper(1)} — tel {digits(3)}-{digits(4)}" ``` Renders e.g. `100 Main St, Apt 4B — tel 555-0199`. Random characters come from the sample functions, so a phone pattern is `070-{digits(3)} {digits(2)} {digits(2)}`. -A lone `}` is a load error naming `}}`; the arms of `{a|b}` must differ, a -repeated arm being a second spelling of [weight](#weight). +A lone `}` is a load error naming `}}`. A repeated arm counts as written, so +`{a|a|b}` draws `a` two times in three. ### Weight @@ -370,29 +583,30 @@ An item of a choice may carry a `weight` (default `1`) to skew its odds: ``` Renders `070-412 38 91` ten times as often as `08-…`. A string item is weighted by -writing it as `{ "format": "AB", "weight": 3 }`. Rejected at load: a weight that -is negative, non-numeric, `0` (never drawn — remove the item), `1` (the default) -or outside a choice, and a repeated item — `["a", "a", "b"]` is `[{ "format": "a", "weight": 2 }, "b"]`, -and the error says so. +writing it as `{ "format": "AB", "weight": 3 }`. A repeated item counts as written, +so `["a", "a", "b"]` draws `a` two times in three, and an item of weight `0` is never +drawn. A weight outside a choice does nothing, and so does one above `0` on the item of +a one-item choice. Rejected at load: a weight that is negative, not a number, or too +close to `0` to tell from it, such as `1e-400`; and a choice whose every weight is `0`, a one-item choice of weight `0` among them. ### Repeat -A template may carry `repeat` (an integer above `1`) to render its format that -many times — each an independent draw — joined by `separator` (default `""`): +A template may carry `repeat` (an integer of 0 or more, default `1`; `0` renders nothing) to render its format +that many times — each an independent draw — joined by `separator` (default `""`): ```json { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -Renders e.g. `bar foo baz`. Rejected at load: a `separator` without a `repeat`, -a `separator` of `""` (the default), and a `repeat` that multiplies to more than -1 048 576 renders along any path of nested repeats. +Renders e.g. `bar foo baz`. A `separator` without a `repeat` above `1` does nothing. +Rejected at load: a `repeat` that multiplies to more than 1 048 576 renders along any +path of nested repeats. ### Datatype A record column may declare `datatype` — `integer`, `number` or `boolean` — so `json` -writes `42` rather than `"42"` and `sql` a bare literal; a column without one is a -string: +writes `42` rather than `"42"` and `sql` a bare literal; a column without one, or with +`string`, is a string: ```json { "format": "", @@ -402,23 +616,24 @@ string: ``` Writes e.g. `{"id":1,"paid":true,"total":59.97}`. A column is a field of the top-level -template, or an item of a choice standing in for one; `datatype` anywhere else is a -load error. A typed column holds one value, alone in its format: a literal, one -`{int()}`, `{float()}`, `{seq()}` or `{calc()}` call, or a read that lands only on such -values. `integer` is an int64 written `0|-?[1-9][0-9]*` — `{float()}` prints one at +template, or an item of a choice standing in for one; `datatype` anywhere else does +nothing. A typed column holds one value, alone in its format: a +literal, one `{int()}`, `{float()}`, `{seq()}` or `{calc()}` call, or a read that lands only on such +values. `integer` is an int64 written `-?(0|[1-9][0-9]*)` — `{float()}` prints one at `0` decimals within int64 — `number` a JSON number, `boolean` `true` or `false`. A value its datatype cannot hold is a load error naming it: ```text order.id: datatype integer: {digits(3)} prints text, not an integer -order.id: datatype integer: "1{digits(2)}" is not one value; write one literal or one {int()}, {float()}, {seq()} or {calc()}, or read one +order.id: datatype integer: "1{digits(2)}" is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one ``` -A column of one reference alone to another record's column — `"score": "{/src.score}"` -— is that column: it takes the column's datatype and is null where the column is, and -a struct field tagged `src.score` is nil there. A `datatype` of its own types the -column's values where they prove it, and one restating the datatype it takes is refused. -Any other read renders the column's text, a null as `""`. +A column whose format is one read of a record's column and nothing else — +`"score": "{/src.score}"`, `"score": "{s.score}"` with `s` bound to `/src`, or `"a": "{p}"` +beside `{place as p}` in its own record — is that column: it takes the column's datatype and +is null where the column is, and a struct field tagged `src.score` is nil there. A +`datatype` of its own types the column's values where they prove it, and `"string"` +reads them as text. Any other read renders the column's text, a null as `""`. A typed column's `{calc()}` must be proven to print a number: each operand a number literal, an `{int()}`, `{float()}`, `{seq()}` or `{digits()}` call, a calc, or a read of @@ -439,9 +654,9 @@ renders a null as `""`. The other items' weights skew its odds: { "format": "", "deleted_at": null, "middle": [null, { "format": "{n}", "n": ["Ann", "Eva"], "weight": 3 }] } ``` -`deleted_at` is null every draw, `middle` a name three draws in four. Rejected at -load: `null` anywhere but a column, naming `""`, and a column whose items hold -different datatypes. +`deleted_at` is null every draw, `middle` a name three draws in four. Anywhere but a +column, `null` renders `""`. Rejected at load: a column whose items hold different +datatypes. ### Table @@ -461,16 +676,16 @@ SE Sweden 10500000 ``` ```sh -fejkdata -d ./mydata country # Sweden (SE), about half the time -fejkdata -d ./mydata country.alpha2 # NO -fejkdata -d ./mydata 'country[SE]' # Sweden (SE) -fejkdata -d ./mydata 'country[Norway].alpha2' # NO -fejkdata -d ./mydata --format csv country # alpha2,name,population → SE,Sweden,10500000 +echo '{/country}' | fejkdata -d ./mydata # Sweden (SE), about half the time +echo '{/country.alpha2}' | fejkdata -d ./mydata # NO +echo '{/country[SE]}' | fejkdata -d ./mydata # Sweden (SE) +echo '{/country[Norway].alpha2}' | fejkdata -d ./mydata # NO +echo '{/country}' | fejkdata -d ./mydata --format csv # alpha2,name,population → SE,Sweden,10500000 ``` `rows` names the TSV beside the category file; `key` names the column a path -selects a row by, `name` a column it also selects by, `weight` a column of positive -numbers that skews the draw, and `parent` the table a column links to +selects a row by, `name` a column it also selects by, `weight` a column of numbers +of 0 or more that skews the draw, and `parent` the table a column links to ([Linked tables](#linked-tables)). The format's `{tokens}` read the columns, and the columns are the [record](#records)'s columns, so `--format csv` writes the rows and `--list` shows `country.alpha2`. A cell is a string node: `1{digits(2)} {digits(2)}` @@ -478,19 +693,22 @@ in a cell draws digits and `{/misc.uuid}` reads a reference, while `{name}` in a is refused, since a cell has no sibling. Each cell may select its own row of another table, `{/misc.currency[SEK].symbol}` on one row and `{/misc.currency[EUR].symbol}` on the next, since only one row renders. `New` proves the header, the options and every -cell token, and refuses a TSV no category names, a key that is empty or repeats, a -weight that is not a positive number, and a key or name holding `[`, `]`, `{`, `}`, -`"` or `|`, which a selector cannot spell; the rows are indexed on the first draw that -selects one. A `name` needs a `key`, since a name naming several rows is reported by +cell token, and refuses: + +- a JSON member other than `format`, `rows`, `key`, `name`, `weight` and `parent`, which + could be meant as a field or as an option a table does not take; +- a key cell that is empty or repeats; +- a weight that is negative, not a number, or too close to `0` to tell from it; +- every weight `0`, across the table or inside one parent row; +- a key or name holding `[`, `]`, `{`, `}`, `"` or `|`, which a selector cannot spell. + +`New` maps the keys, and a name or parent column is mapped on the first draw that selects +by name or descends through the table. A `name` needs a `key`, since a name naming several rows is reported by their keys, or a `parent`, inside whose row a name names one row, so `first-name[Kim]` is settled by the `sex` selected before it and a name repeating inside one parent row is refused; a name spelling another row's key is refused, since the key would -select first and the name never. The table's options are its own — `rows`, `key`, -`name`, `weight` and `parent` — so a column may be named `name`, as one usually is. - -A choice of templates sharing one format and one set of string fields is a table -written by hand, and `New` refuses it in a data file naming the TSV to write; an -inline template has no file beside it, so there it stays a choice. +select first and the name never. A table's options are its own, so a column may be named +`name`, as one usually is. ### Row selection @@ -500,8 +718,7 @@ A name naming several rows is an error listing their keys, unless a row selected before it settles which ([Linked tables](#linked-tables)). A selector is part of the path, so it works wherever a path does: `Fake`, `FakeRecord`, a `{/misc.territory[SE].capital}` reference and a struct tag. A dot inside the brackets belongs to the key or name, so -`city[St. Louis]` selects it. A path starts with a name, and `[` still opens a JSON -array at the start of a CLI argument, so `'[SE]'` alone names nothing. +`city[St. Louis]` selects it. ### Linked tables @@ -522,47 +739,63 @@ Stockholm SE 990000 ``` ```sh -fejkdata -d ./mydata 'country[SE].city' # Stockholm or Göteborg -fejkdata -d ./mydata country.city.name # a country drawn, then a city inside it -fejkdata -d ./mydata 'city[Oslo].country' # NO — the link column's cell -fejkdata -d ./mydata '{/city.name}, {/country.name}' # Oslo, Norway — one consistent draw +echo '{/country[SE].city}' | fejkdata -d ./mydata # Stockholm or Göteborg +echo '{/country.city.name}' | fejkdata -d ./mydata # a country drawn, then a city inside it +echo '{/city[Oslo].country}' | fejkdata -d ./mydata # NO — the link column's cell +echo '{/city as c}{c.name}, {c..country.name}' | fejkdata -d ./mydata # Oslo, Norway — one city and its country ``` A path descends from a row to a linked table by name, at any depth, and `--list` -advertises each direct step. Within one render and [draw group](#draw-group), linked -tables agree: the first table a reference path reads pins its ancestors, and a -descendant read after it is drawn inside them, so `{/city.name}` and -`{/country.name}` are a city and its country whichever is read first. A selected row -pins the render the same way, so every reference path into one family of linked -tables in one render and group selects the same rows: one that selects none beside -one that does is refused naming the spelling that does, `{/country[SE].city.name}` -beside `{/country[SE].name}`, and two selecting different rows are refused naming a -`drawGroup` to draw them apart in. A bare `{/city}` beside a path into its family is -refused too, since a bare reference draws each time, and so is a path that draws a -table another path in the group selects a row of, `{/city.name}` beside -`{/country[SE].name}`, which the first token rendered would otherwise decide. `New` -also refuses a link cell that is no key of the parent, a parent row no child links -to, a chain of parents that closes, a table named like a column of any table above -it, and a cell or format of a table that references a table of its own family, through -any template, a `repeat` or a `drawGroup` included, since a row rendered whole would -draw the family apart from itself: read the family from a template beside it, or add -the value as a column. +advertises each direct step. One path reads one chain of rows: `country.city.name` draws a +country by its weights, then a city inside it by theirs. Two paths are two draws, so +`{/city.name}, {/country.name}` may name a city and another country. The reads of one +[name](#names) agree instead, and the table it binds decides whose weights govern: +`{/city as c}` draws a city by city weight and `{c..country.name}` is its country, while +`{/country as k}` draws a country by country weight and `{k.city.name}` a city inside it. +A name keeps every row its reads draw, so `{k.city.name}` and `{k.city.population}` describe +one city, and `{k.name}` its country. + +`New` refuses a link cell that is no key of the parent, a parent row no child links to, a +chain of parents that closes, and a table named like a column of any table above it. + +### Step up + +`..` after a row steps up to the row its link column names. Name the parent table right +after the `..`: + +```sh +echo '{/city[Oslo]..country.name}' | fejkdata -d ./mydata # Norway +echo '{/city..country}' | fejkdata -d ./mydata # a city drawn, then its country: Sweden (SE) +echo '{/city[Göteborg]..country.city}' | fejkdata -d ./mydata # Stockholm or Göteborg, drawn afresh +echo '{/city as c}{c}, {c..country.name}' | fejkdata -d ./mydata # Oslo, Norway +``` + +It steps up from a selected row, a drawn row or a [name](#names) bound to a row, one +level per `..`: `geo.SE.locality..municipality..region.name`. `.country` after a row +still reads the link column's cell, `NO`. + +After a `..`, `.` steps down again and draws afresh inside the row stepped up to, so +`city[Göteborg]..country.city` may draw Göteborg again. Under a name, the paths stepping +down from one `..` read one draw: `{c..country.city.name}` and +`{c..country.city.population}` describe one city. + +A `..` that names anything but the parent table is an error naming the parent. A selector +after a `..` is an error naming the table to select from, since it could name a row outside +the one stepped up to. `--list` advertises no path with a `..`. At the start of a +reference, `..` is the folder above ([References](#references)). ### Options and fields -`format`, `weight`, `repeat`, `separator`, `datatype` and `drawGroup` are the only options; -**any other key is a field** (see [Decisions](#decisions)), and `rows` makes a category -a [table](#table), so no template carries a field of that name. An object that does nothing a -string can't — only a `format` — is rejected naming the string, as is a one-item -choice naming its item. +`format`, `weight`, `repeat`, `separator` and `datatype` are the only options; +**any other key is a field** (see [Decisions](docs/decisions.md#options-and-fields-share-one-namespace)), and `rows` makes a category +a [table](#table), so no template carries a field of that name. A [name](#names) shares the +namespace too. ### Functions A `{name(args)}` token calls a builtin. Arguments are checked at `New`: a bad -count, range, country or expression fails fast; an integer is written plain -(`5`, not `+5` or `05`); bounds are finite; a sample that could only ever emit one -value (`int(5,5)`, `float(1,1,2)`) is rejected naming the text to write instead; -and a length, count or decimal place beyond a sane maximum is rejected, so a +count, range, country or expression fails fast; an integer may carry a sign or leading +zeros (`+5` and `05` are `5`); a count of `0` renders nothing; bounds are finite; and a length, count or decimal place beyond a sane maximum is rejected, so a fat-fingered `hex(2000000000)` never tries to allocate gigabytes. Every builtin draws only from the seed — a time-based id takes its timestamp from the rng, not the clock — so seeded output stays reproducible. @@ -585,7 +818,7 @@ stays reproducible. | `{date(from,to,'layout')}` | sample | a second between two `YYYY-MM-DD` days, both included, in a quoted Go layout: `'2006-01-02'`, `'January 2, 2006'`, `'060102'`, `'2006-01-02T15:04:05Z'` | | `{time('layout')}` | sample | a second within a day: `'15:04'`, `'3:04 PM'` | | `{seq()}`, `{seq(name)}` | counter | next integer from 1 in this generator; `name` selects an independent counter | -| `{calc(expr)}`, `{calc(expr,dp)}` | computation | an arithmetic expression over sibling fields ([Computation](#computation)) | +| `{calc(expr)}`, `{calc(expr,dp)}` | computation | an arithmetic expression over sibling fields and names ([Computation](#computation)) | | `{lowercase(x)}`, `{uppercase(x)}`, `{ascii(x)}` | transform | a field's value rewritten ([Transforms](#transforms)) | A derivation reads what is to its left, so place it after its payload; the @@ -604,42 +837,47 @@ between the two days is reachable, so a layout with a clock draws the time too, `from` may equal `to`, which is that one day. The instant is UTC, so a zone in the layout prints `UTC` or `Z`. The quotes delimit a layout outside a selector only, so `[O'Fallon]` in an -argument stays a name. Rejected at `New`: a bound that is no calendar date, or not -before the other; an unquoted layout, naming the single-quoted one; a layout naming no field, which is -text, as is one day in a layout with no clock; for `date` a layout naming no date -field, naming `time`; and for `time` a layout naming a date field, naming `date`. `{seq()}` spans `Fake` +argument stays a name. A layout naming no field prints itself, `date` with a clock-only +layout prints the drawn instant's clock, and `date` over one day with no clock field prints +that day. +Rejected at `New`: a bound that is no calendar date, or after the other; an unquoted +or double-quoted layout, naming the single-quoted one; and `time` with a layout naming a +date field, naming `date`, since `time` draws no date. `{seq()}` spans `Fake` calls and `repeat`, resets with a new generator, and is the natural primary key for -the SQL example above. +the SQL example above; give each record its own counter, `{seq(orders)}`, where one +generator fills several. ### Computation `{calc(expr)}` evaluates `+ - * /`, parentheses and unary minus over number -literals and sibling field names, each rendered then read as a number; a second -argument rounds to that many decimals. A hyphen is always subtraction, so a -hyphenated field can't be an operand. +literals, sibling fields and [names](#names), each rendered then read as a number; +a second argument rounds to that many decimals. A hyphen is always subtraction, so a +hyphenated field or name can't be an operand. + +A calc computes from the values its format shows where both read one [name](#names): ```json -{ "format": "{net} x {qty} = {calc(net * qty, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } +{ "format": "{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}", "net": ["19.99", "5.00"], "qty": ["3", "7"] } ``` -Renders e.g. `19.99 x 3 = 59.97`. A result that rounds to zero prints unsigned — `0`, -`0.00` — as `{float()}`'s does. An operand that can never be a number (`"abc"`, -or a choice of such) is rejected at load, as is a division by a constant zero -(`1/0`, or a fixed `"0"` field); an operand that sometimes is not a number yields +Renders e.g. `19.99 x 3 = 59.97`. + +A result that rounds to zero prints unsigned — `0`, `0.00` — as `{float()}`'s does. An +operand that can never be a number (`"abc"`, or a choice of such) is rejected at load, as +is a division by a constant zero (`1/0`, or a fixed `"0"` field); an operand that sometimes is not a number yields `NaN`, and a division by one that is not constant `Inf` — both print rather than fail, except in a [typed column](#datatype), which must prove neither happens. ### Transforms `{lowercase(x)}`, `{uppercase(x)}` and `{ascii(x)}` rewrite the value of `x` — a -field, a path or a `..path` — and nest. `ascii` folds Latin letters (`Åsa Öberg` -→ `Asa Oberg`) and drops any other non-ASCII rune. `x` is held -([One draw, one spelling](#one-draw-one-spelling)), so an email built from a name -matches the name beside it: +field, a path, a `..path` or a [name](#names) — and nest. `ascii` folds Latin letters (`Åsa Öberg` +→ `Asa Oberg`) and drops any other non-ASCII rune. An email +matches the name beside it where both read one name: ```json -{ "format": "{p.first} {p.last} <{lowercase(ascii(p.first))}.{lowercase(ascii(p.last))}@example.com>", - "p": [ +{ "format": "{person as p}{p.first} {p.last} <{lowercase(ascii(p.first))}.{lowercase(ascii(p.last))}@example.com>", + "person": [ { "format": "{first} {last}", "first": "Åsa", "last": "Öberg" }, { "format": "{first} {last}", "first": "Bo", "last": "Ek" } ] } @@ -654,58 +892,72 @@ A reference renders a node from elsewhere in the data — the path `Fake` takes, across every loaded source — so one category borrows another. The sigil says where the path starts, as in a filesystem: `{/en_US.person}` from the data root, `{.username}` from the folder this file sits in, `{..username}` from the folder -above. `data/sv_SE/email.json` can therefore read its own locale's `username` +above. `sv_SE.email` can therefore read its own locale's `username` without naming `sv_SE`: ```json "Hej, {/en_US.person}!" ``` -Renders e.g. `Hej, Pat Smith!`. A reference path into a category is held like a -[correlated](#correlated-fields) path, but for the whole render — one `Fake`, or one -record — rather than one format: `{.person.first} {.person.last}` name one -person, as do the same two references in sibling fields or a nested template, and -`{lowercase(.person.first)}` reads that same draw. Each `repeat` iteration is -a render of its own, in no group, so it draws anew, and a [draw group](#draw-group) holds a -draw apart. A bare reference names no field and makes its own picks each time — -`{/misc.uuid} {/misc.uuid}` is two draws — while the reference paths inside what it -renders still read the render's draws. Rejected at `New`: a path that is +Renders e.g. `Hej, Pat Smith!`. Each reference draws afresh ([Format string](#format-string)), +so `{.person.first} {.person.last}` may name two people. Bind the person to a [name](#names) to read one: `{.person as p}{p.first} +{p.last}`. Rejected at `New`: a path that is unknown, names a folder, has no folder above, reads a field not every variant -of a choice carries, or names the category the reference sits in, and a reference -that leads back to its own value, directly, mutually or through a chain. +of a choice carries, and a reference that leads back to its own value, directly, +mutually or through a chain. -### Draw group +### Names -A template may carry `drawGroup` to hold its reference draws apart: every reference path -it renders, however deep short of a `repeat` or a nested `drawGroup`, reads the draw of -that group, and the templates of one category naming one group in one render read one -draw. A name is local to its category, so a category another one references never joins -its groups by name; the unnamed group spans them all. +`{x as n}` binds one pick of `x`, a reference, a sibling field or a path into one, to the name +`n` and prints nothing. `{n}` then +prints that pick, and `{n.path}` reads a path through it, so every read of `n` describes one +row: ```json -{ "format": "{payer} pays {payee}; signed {signature}", - "payer": { "format": "{/sv_SE.person.first} {/sv_SE.person.last}", "drawGroup": "payer" }, - "payee": "{/sv_SE.person.first} {/sv_SE.person.last}", - "signature": { "format": "{/sv_SE.person.last}", "drawGroup": "payer" } } +"{/misc.territory as t}{t.flag} {t}: capital {t.capital}, calling code +{t.calling-code}" ``` -Renders e.g. `Sara Eriksson pays Ebba Lind; signed Eriksson`: the signature reads the -payer's draw, while the payee is drawn apart. Rejected at load, each naming nothing: a -`drawGroup` of `""` (the default); one naming the draw group its template already draws -in; one on a template that renders no reference path — a bare reference to a -[table](#table) counts, since the group answers for the family it draws in — short of a -`repeat` or a nested `drawGroup`, on a `repeat` itself — each iteration renders in no -draw group — or on an inline template's root, which nothing references. So is a path -reading into a level that carries a `drawGroup`. +Renders e.g. `🇳🇴 Norway: capital Oslo, calling code +47`. A name keeps what each `{n.path}` +lands on and every level it passes, so `{n}` and `{n.path}` agree, and so does a +[transform](#transforms) of `n.path`. Everything else under the name draws as it would +anywhere: if `n`'s category has the format `{w}-{w}`, it draws `w` twice, and `{n.w}` beside +`{n}` is refused, since it could read either draw. A sibling field is bound the same way, so +`{place as p}{p.postal-code} {p.locality}` reads one place ([Correlated fields](#correlated-fields)). + +A name is a pick of its own: `{t.name}` and `{/misc.territory.name}` beside it are two draws, +and so are two names bound to one category. + +A name is drawn on its first read, and lives in the category binding it: any field of the +category may read it, and a record's columns read one pick. Each render of the category picks +anew, and so does each bare reference to it. A name a `repeat` binds picks again on every +iteration, and one bound outside the repeat keeps its pick on every line. A name a choice's +item binds lives in that item, and picks anew each time the item is drawn. A category a +reference enters does not see the name; bind it in the category that reads it. + +Refused at `New`, each error naming what to write instead: + +- a name bound twice outside any `repeat`, twice in one `repeat`, or both inside a `repeat` + and outside it; +- a name that is a field or an option; +- a binding of anything but a reference, or a field of the template binding it or a path into + one; +- a read of a name inside the field bound to it: read it outside that field; +- reading a name outside the choice item that binds it, since the other items leave it + unbound; +- a path through a name that selects a row: `{r.municipality[0180]}` could mean municipality + 0180 whatever region `r` picked, or 0180 only where `r` picked the region holding it; +- `{n}` beside `{n.w}` where `n`'s category reads `w` twice, as in `{w}-{w}` or + `{w.a} {w.b}`; +- a binding in a table's format or cell. ### Correlated fields -`{name.field}` reads a path into a sibling, which holds the sibling to one draw -([One draw, one spelling](#one-draw-one-spelling)), so several tokens read one -row — a locality and the postal code that really covers it: +`{name.field}` reads a path into a sibling. Several tokens read one draw of it through a +[name](#names): bind the sibling, and every read of +the name reads that one row — a locality and the postal code that really covers it: ```json -{ "format": "{street} {int(1,99)}\n{place.postal-code} {place.locality}", +{ "format": "{place as p}{street} {int(1,99)}\n{p.postal-code} {p.locality}", "street": ["Kungsgatan", "Storgatan"], "place": [ { "format": "{locality}", "locality": "Stockholm", "postal-code": "1{digits(2)} {digits(2)}", "weight": 975 }, @@ -714,66 +966,57 @@ row — a locality and the postal code that really covers it: ``` Renders e.g. `Kungsgatan 35` / `176 99 Stockholm`, never a Stockholm code beside -Tranås; each row's `weight` says how often it appears. A path is held at every -level it passes through: `{p.geo.town.name} {p.geo.town.zip}` share the town. -Every variant of a choice on the path must carry the rest of it, so a row missing -a field is named at load: +Tranås; each row's `weight` says how often it appears. `{p.geo.town.name} +{p.geo.town.zip}` share the town too, as a name keeps every level its reads pass. Every variant of a choice +on the path must carry the rest of it, so a row missing a field is named at load: ```text -token {place.postal-code}: field "place": not every variant of this 2-way choice carries "postal-code"; all carry [locality] +token {p.postal-code}: not every variant of this 2-way choice carries "postal-code"; all carry [locality] ``` The sub-fields stay addressable — `Fake("address.place.locality")` renders, and -`List` advertises it. A path may not read into a level carrying a `repeat`. - -### One draw, one spelling - -A name any token reads as a path (`{p.first}`) or as an operand (`{calc(net * 2)}`, -`{uppercase(w)}`) is drawn **once per expansion**, a reference path (`{/cat.p.first}`) -**once per render** in its [draw group](#draw-group), and every other route to either — a bare -`{p}`, a second bare `{w}`, `{/cat.net}`, a nested template rendering `{/cat.p.last}` -beside `{p.first}`, a bare `{/cat}` beside `{/cat.p.first}`, at any -depth — is a load error naming the spelling to use. A name nothing reads that way is -drawn each time: `{word} {word}` differs. An expansion is one render of one format, so -each nested template draws its own names again; a render is one `Fake` or one record, -and each `repeat` iteration is an expansion and a render of its own. - -```text -token {p} renders a level that {p.first} reads a path into; name the fields you want instead -token {w} is repeated, and uppercase operand "w" holds "w" to one draw per expansion; write {w} once -``` +`List` advertises it. A path may not read into a level carrying a `repeat`, and `List` +advertises no such path; read that level whole. ### Performance -Each file is parsed, validated and weight-indexed once, in `New`. Proving the draw -fences adds one pass over the loaded tree, and walks what a render reads only where -data binds a reference, so a set that binds none pays for the pass alone. A `Fake` call -then costs about what its output costs: an unweighted pick is O(1) whatever the -list's length, a weighted one O(log n), and long formats, deep nesting and many -tokens add cost in proportion to the output. +The target is about a microsecond per value (goal 12). + +A source with an [index](#data), as the shipped set has, loads nothing in `New`: a +category loads on the first `Fake`, `FakeRecord`, `FakeStruct` or template that reads +it, with every category it reads and its whole table family. Each file is parsed, +validated and weight-indexed once, bar a table's name and parent columns, which are +mapped on the first `Fake` that selects by name or descends through it. A `Fake` call then +costs about what its output costs: an unweighted pick is O(1) whatever the list's length, +a weighted one O(log n), and long formats, deep nesting and many tokens add cost in +proportion to the output. ## Versioning Semver tags on `main`, `v0.1.0` first; one version covers the shipped data, the library and the CLI, and [`CHANGELOG.md`](CHANGELOG.md) names what each release -changed. A consumer's data, code and scripts keep working across a minor or a patch: -a minor only adds, and a major is the only release that changes what exists. +changed. A consumer's data, code and scripts keep working across a minor or a patch. A +minor only adds, and a major is the only release that changes what exists. Seeded output +is the exception: any release, a patch too, may change what a seed draws. Where you +compare seeded output, install one exact version +(`go install github.com/larvit/fejkdata/cmd/fejkdata@vX.Y.Z`, the image +`larvit/fejkdata:X.Y.Z` or a release's binary; `go.mod` already pins one), +and regenerate seeded fixtures and expected values when you raise that version. | Surface | Major | Minor | |---------|-------|-------| | Shipped data | remove or rename a path; change a category's format; remove a value, or change a weight or a repeat; add a reference from one shipped category into another; change a table's key, name, weight or parent column, or remove a row | a path outside a record's columns, a locale, a value in a list, a row | | Records (a table is one) | remove, rename, retype or add a column; let a column be null | a record, as a new category | -| Data format | a fence: a spelling `New` rejects that it accepted; a template option, since it reserves a field name | a builtin | +| Data format | a fence: a spelling a load rejects that it accepted, in a category or a manifest; a template option, since it reserves a field name | a builtin | | CLI | remove or rename a flag, or change its default; change what an exit code means; change the framing a `--format` writes (header, quoting, statement shape), the `--list` layout, or what an error names | a flag, a format | | Library | change or remove an exported name; raise the lowest supported Go | an exported name, a `With…` option | +| Data packages | rename or remove a package; move a category to another package; make a package read another it did not | a package | A patch changes no row of this table: performance, docs, or a fix inside a promised -behaviour that changes no value, path, format or spelling. +behaviour that adds or removes no value and changes no path, format or spelling. -Seeded output is a promise within one version: same seed, same version, same -data, same output. Any release may shift a stream, since a value added to a list -moves every draw after it, so pin fixtures per version. An error's wording may -improve in a minor; the path, rejected spelling and replacement it names may not. +An error's wording may improve in a minor; the path, rejected spelling and replacement +it names may not. Before `v1.0.0` a minor is the breaking unit: `0.(x+1).0` may carry a major's changes, each named in the changelog, and a `0.x.y` patch may not. `v1.0.0` is @@ -791,537 +1034,142 @@ renamed or retyped line is a major. App developers writing tests and fixtures, in Go and at a shell: -- a **bulk fixture author**, thousands of rows into CSV or SQL +- a **bulk fixture author**, millions of rows into CSV or SQL - a **Go test author**, filling a struct with `FakeStruct` -- a **hand fixture author**, one value at a shell +- a **hand fixture author**, a few values at a shell - a **validator-facing author**, who needs a value a real checker accepts +- a **data author**, who writes categories under `--data-path` and meets every load + error the fences raise +- a **custom-CLI builder**, who ships a CLI like `fejkdata` with data and functions of + their own and a **contributor**, who reads [`todo.md`](todo.md), [`AGENTS.md`](AGENTS.md) and -the Development section below, and who ships a register the four above then draw from. +the Development section below, and who ships a register the six above then draw from. ## Goals -1. **The best fake-data tool available, and used because it is** — a developer - choosing a seed-data library or CLI, in any language, should find this the one - worth switching to, and every goal below it is what that costs. Merit is only - half the goal: once the grammar, the data and the CLI are complete and stable, - it ships where those developers install from and is announced where they read. -2. **Understanding the code is part of the product** — it is a compiler with static - fences behind a small API, it will grow, and the people who extend it will not be - the people who wrote it. So reading cost is judged alongside behaviour: reaching - the unit behind a symptom without asking a person, changing one piece without - holding the rest, names that tell the truth, and a file that stands with no second - document open beside it. A simulated reader panel scores those four, the score is a - ratchet no release lowers, and [`AGENTS.md`](AGENTS.md) says what each pull request - owes it. -3. **Valid by construction** — every value passes the check its real consumer - applies; facts that belong together come from one draw, within a value and - across categories. -4. **Text means what it says** — a format renders as written; only `{…}` varies, - random characters included (`{digits(3)}`). One spelling per result; the wrong - one is a load error naming the right one. -5. **Every mistake is a load error** — `New` rejects the data and `NewTemplate` - the inline template; on a loaded generator `Fake` fails only for an unknown - path, `FakeStruct` only for a non-struct argument or a type its tags do not - describe, with the same error every call, and `Template.Fake` cannot fail at all. -6. **Zero to a value in one command** — `go install`, then `fejkdata sv_SE.person`: - no checkout, no flag. Flags are GNU-form (`--seed 42`, `-n 3`) in any position; - the first custom template needs no escape and no option. -7. **Data lives in JSON** — a builtin only for what data can't express. -8. **Reproducible** — seed in, same stream out; no builtin reads a clock. -9. **Zero dependencies** — standard library only. -10. **Docs index the grammar** — every syntax feature is a heading; every example - runs under test and shows its output; a rule is stated once. -11. **Fast enough to be free** — a value renders in about a microsecond and `New` - parses and validates the whole set once upfront, so generating fixtures stays - noise against a test's own runtime. -12. **Data is sourced, or on its way there** — a shipped fact, a name, place, - code, id or classification, is read from a register or open dataset by a - [`data-import/`](data-import) script wherever one exists to read; where none - does yet a small hand-written set ships and [`todo.md`](todo.md) carries the - step that replaces it. Only non-factual copy stays authored. A sourced table - holds the rows its source holds: none is added by hand, and one is dropped - only by a rule the script states. -13. **Breadth follows what most systems store** — a category is added in proportion - to how many real schemas hold it: names, addresses, phones, ids, money and - timestamps before anything domain-specific, and a catalogue serving one niche - waits behind everything serving many. -14. **Realism is the default, inertness is selectable** — where a value could reach - something real, a domain anyone may register or an account a bank could issue, - the realistic breadth ships *and* so does the subset that provably reaches - nothing, each on its own path. A fixture that looks nothing like production - tests nothing; the caller who needs a value that can touch nothing asks for it - by name. -## Decisions +Where the project is heading; the sections before Audience document what ships today. -- **Options and fields share one namespace.** `format`, `weight`, `repeat`, - `separator`, `datatype` and `drawGroup` are reserved; every other key is a field. Nesting fields under a - key, or prefixing options, would tax every template to guard against a - misspelt option. -- **`{a|b}` stays beside nested choices.** `[[…], […]]` picks the same way, but - its arms are anonymous; `{female|male}` keeps `person.female` addressable. -- **Flags follow getopt_long.** `--name value` and `--name=value` both work; a - short flag's value attaches or follows (`-s42`, `-s 42`) and short flags bundle - (`-hn 3`), as every shell user expects. A single-dash long flag is rejected - naming the double-dash spelling, and `-s=42` is rejected naming both short - spellings: `=` belongs to the long form, and reading `=42` as the value would - make `-d=./x` a directory named `=./x`. -- **An argument is a template by its shape, not by a flag.** A JSON object, array - or string, or a string carrying a `{` token, is an inline template; anything else - is a path. A name may not contain a brace, a bracket or a quote, so a path can - never collide with any of those spellings, and the leading `[` or `"` is gated on - valid JSON so a stray copied bracket never swallows an argument — it names - nothing, and says so. No `--template` flag is needed. Reserving the characters - whole — though only a leading one could collide — keeps one simple name rule - instead of a leading-position special case. The JSON string is what makes the - library's own advice reachable: the error for an object holding only a format - names `"…"`, and that spelling has to work where it is printed. An argument or struct - tag of one reference alone, `{/users}`, is refused naming the path `users`: both - render the same text, and only the path names a record. A folder-relative `{.name}` - or `{..name}` is refused naming `{/name}`, since an inline template sits in no - folder. `IsTemplate` exports the - rule, so the CLI, struct tags and any other caller read one. -- **An inline template skips the cycle fence.** `New` proves the loaded tree - acyclic, an inline node is a finite tree of its own, and nothing in the tree can - reference it, so no render of it reaches itself. Every other fence runs over both, - from one `checkScope`, except that struct tags leave column agreement to their Go - types. -- **An inline template that does not compile is misuse (exit 2), including a - reference that resolves to nothing** — the whole argument is the spelling under - test, and `NewTemplate` compiles, links and validates as one step. An unknown - *path* stays a runtime error (exit 1): there the argument is well-formed and only - the data is absent. -- **A padded JSON argument is rejected, not trimmed.** Padding is the one place the - two readings disagree — a format string renders it, JSON drops it — so the - spelling that renders is named rather than silently chosen. -- **`FakeTemplate` and `NewTemplate` both stay.** They reach the same value but not - at the same cost: `NewTemplate` pays the compile and validation once and renders - many times, `FakeTemplate` is the one-shot call, and `--repeat` is exactly the - case that needs the first. The pair is `regexp.MustCompile` and `regexp.Match`, - not two spellings of one result. -- **The shipped data is embedded, not discovered.** A directory a machine happens - to have would make `--seed 42` machine-dependent. Data still lives in `data/` - as JSON; `--data-path` layers over it. -- **A bare reference draws each time; a reference path is held.** `{/p} {/p}` - is two draws, as `{word} {word}` is, while every `{/p.first}` in one render reads - one draw, and a bare `{/p}` beside them is a load error: a bare token - is by contract an independent draw, a path pins its level, and a fresh draw of a - pinned level could show another row. A builtin's operand holds what it reads for - its expansion, references included, so `{uppercase(/p)} {/p}` is one draw — the - rule every operand follows. -- **Reference sigils follow the filesystem.** `/` is the root, `.` this file's - folder, `..` the folder above — what those spellings already mean to anyone who - has typed a path. A locale's files reach each other without naming the locale, - so a folder renames and copies without editing its references. -- **A change to what exists is a major; a minor only adds.** Data files, the CLI - and the Go API are the public API, and a consumer must be able to take a minor - without an edit — so an added column is a major, since it changes the CSV header - and the `INSERT` column list, as is a removed value, which changes what a fixture - holds, and a new option, which reserves a field name. One spelling per result - grows by tightening, so every fence invalidates some file. Each such release - names the rejected spelling and its replacement in the changelog and in the load - error, and that is the whole migration: a fence rejects one spelling with one - replacement, so the fix is local to each site. A fence that would need a - non-local rewrite ships a converter with its release instead. Before `v1.0.0` a - minor carries what a major would. -- **Seeded output is promised within one version.** Any edit to a category shifts - its stream and everything drawn after it, so a promise across versions would - freeze every shipped list; a fixture is re-pinned on a bump, as this repo's own are. -- **An error is a contract by what it names, not its bytes.** A script branches on - the exit code and reads the named path or spelling, so those hold; wording improves - in a minor. -- **Raising the lowest supported Go is a major.** A consumer building on it breaks, - which is the one test every rule above applies; Go's convention of a minor is not - followed. -- **The changelog heading is the one spelling of a release; CI cuts the tag.** A - tag pushed by hand is served by `go get` at once, so a tag whose commit lacks its - heading is burnt, not fixed. The heading on a gate-passed `main` commit is the - trigger instead: the tag can land only there, and the Gitea release the same job - publishes keeps one text as its body and is where prebuilt binaries will attach. -- **A `--data-path` override rebinds every reference to the category it - replaces.** References bind against the merged tree, so once shipped data uses - `{.person}`, a consumer's `sv_SE/person.json` is what every shipped reference - into `person` reads, and `New` fails on shipped data the consumer never wrote - when that file lacks a field those references read. Accepted: overriding is the - point of layering, the error names the reference and the field, and the fix is - the consumer's file carrying the fields the shipped tree reads. -- **The repeat cap bounds renders, not bytes.** A repeat, alone or nested, may - ask for at most 1 048 576 renders; how large each render is stays what the data - asked for, so `{hex(1048576)}` repeated to the cap is a terabyte, loaded without - complaint. A byte estimate would need every builtin to declare a width to fence - a shape no data comes near, and the harm lands on the author who wrote it. -- **64-bit targets only.** The gate builds amd64, and the buffer sizing a render - pre-computes (renders × bytes) assumes a 64-bit int; on a 32-bit target it could - overflow and panic. -- **A constant zero divisor is a load error; in a string column a divisor that is not - constant prints `Inf`.** `1/0` and a fixed `"0"` field are decidable, so they join - the never-numeric operand as a load error; the fold stops where an operand varies, - so `a/(b*c)` with `b` fixed at `0` and `c` varying loads and prints `Inf` every - draw — catching it needs zero-absorbing algebra for a shape nobody writes. A - [typed column](#datatype) bounds its operands instead and refuses a divisor it - cannot keep from zero. -- **In data, a default written out and a constant spelled as a sample are load - errors.** `weight: 1`, `repeat: 1`, `separator: ""`, `datatype: "string"`, - `int(5,5)`, `float(1,1,2)`, - `+5` and `05` each spell what a shorter form already spells, so each is rejected - naming that form. The CLI's numbers follow the shell instead: `--seed 007` and - `--repeat +3` are 7 and 3, as every command line reads them. -- **Samples say what they emit, transforms what they do.** `{upper(2)}` is two - letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on - whether the argument looks like a number. -- **A record is a template seen as columns; a Go struct is the one second schema.** A - template's `format` composes its fields into one string; `FakeRecord` and - `--format` project the same fields as columns. Two views of one dataset, so a - record author writes the same JSON they already know, and a column is the same - field `Fake` renders by dotted path. The `format` is inert to a record — a - record-only template writes `"format": ""` — but it is compiled and fenced, so - a template that loads renders as whichever shape is asked for. `FakeStruct` takes - its columns from a struct instead, because a Go caller has already written that - schema: the fields name the columns and their types are the datatypes, so a tag - says only what to draw, and a `datatype` in it would be a second spelling of the - type. The Go type is a struct column's one datatype, so its items need not agree on - one among themselves; each must only hold that type. -- **A struct's records follow Go's field access, and compile on first use.** The - fields an embedded struct promotes are the struct's own — `e.First`, as - `encoding/json` and SQL mappers read them — so they are columns of its record and - share its draws; a tagged field that another field hides is refused, not dropped. A - named struct field is another entity and a record of its own. `fake:"-"` leaves a - struct field, embedded or named, unfilled, so no name may be `-`; a pointer back to - a struct already being filled is left alone, since filling it would never end. `New` cannot - see a caller's types, so the first `FakeStruct` for a type compiles its tags and the - answer, error included, is kept per type: a test's first call is its load, and no - `NewStruct` handle is needed, as the cache already compiles once. -- **A render shares one reference draw per category, per group.** Every reference - path into a category in one `Fake`, or one record, reads one draw of it, so a - value's facts agree across its fields, nested templates and columns alike — - `{/currency.code}` in one field and `{/currency.symbol}` in another name one - currency, whichever view renders them. A `repeat` iteration is a render of its - own, since repeating asks for another entity, and a [draw group](#draw-group) names further - entities within one render, so a payer and a payee are two groups over one - `person` rather than two copies of it. Only references share: a sibling field is - local to its own expansion, so a `first` column does not silently bind to a - `first` in the column next to it. -- **A draw group name is local to its category.** A category's groups are its own - entities, so a caller naming a group the same way never joins them by accident, - and renaming a group inside one file changes no render elsewhere. The unnamed - group still spans categories, since facts that belong together across categories - must agree. -- **The expansion hold and the render's draws are two fences.** One proves a sibling - path or an operand is reached only by its readers within an expansion, the other - does the same for reference paths across a render and its draw groups. They pin - different things — an operand pins the value its own render produced and stops at a - reference, a path pins every level it passes through — so one walk would carry both - rules and both scopes anyway, and tell them apart at every step. -- **A record makes its draw maps up front, a `Fake` on its first read.** A record's - columns always read through the render's draws, so making the maps where the set is - declared keeps them on that frame's stack. A `Fake` often reads no reference path at - all, and making them anyway cost about a fifth of the cheapest render, so it makes - them on the first read instead, at two heap allocations for a render that does share - a draw. The allocation gate over a repeat of a reference path and over a named draw - group prices that, and pins the two measures that keep a draw set off the heap. -- **A category never references itself, and a record's fences run at load.** A category - is one unit: a reference back into it — `{/users.first}` inside `users` — describes a - draw other than the fields beside it, so `New` refuses it and the sibling path stays - the one spelling for a field of one's own. A value two fields share goes in its own - category, which both reference. That settled, a record's column fences run at `New` - too, so a category that loads renders as whichever shape is asked for, and a reference - reaching back into a category through another one is refused there as the overlap it - is. Which reads those fences weigh differs on purpose: a record-only template's inert - format renders nothing, so a `drawGroup` on it can never matter and is refused, while - one on a rendering format can matter to a caller that bare-references it. -- **A record's column set is fixed before the first draw.** Only a category-level - template is a record: a path descending into a field, or naming a folder or a - choice, errors. A tail may pass through a choice whose variants carry different - fields, so the columns — and with them the CSV header written once ahead of every - row — would vary per draw. A fixed column set is what the CSV and `INSERT` - contracts rest on, so the restriction holds even where a particular choice would - happen to agree. -- **Null is a `null` item, not a rate.** A null is one more outcome of a column's - draw, so a choice's weights skew it like any other; a null-rate option would be a - second way to state odds. -- **A typed column holds one value, not composed text.** Its bounds come from a - literal or a call's arguments, so a load error names a real value, a range check is - one comparison, and `1{digits(2)}` is a second spelling of `{int(100,199)}`. -- **A column of one reference alone is the column it reads.** `{/src.score}` renders - exactly what `src.score` draws, so it takes that column's datatype and null rather - than restating them, and a `datatype` restating the one it takes is a second - spelling. Any other `datatype` still types the values — the one way to type a column - someone else wrote. -- **A typed column's calc is refused unless proven.** Operand bounds must keep each - divisor from zero and the result finite; what they cannot show is refused rather - than trusted, since a bare `NaN` breaks the JSON and SQL it lands in. -- **`Column` carries text, not a Go value.** `Value` is the rendered string beside - `DataType` and `Null`, which each serializer writes as the load check proved it; a - `Value any` would hand every caller a type switch. -- **The package stays flat.** Go ties a package to one directory, so folders would - split the API into packages. -- **The performance gate asserts allocations, not wall-clock time.** `AllocsPerRun` - is deterministic across machines, so a ±10% ceiling does not flake under CI load, - while time varies with the machine and its neighbours. A rendering slowdown - almost always costs an allocation too (a lost pre-size, a per-item map, an extra - copy). The benchmark suite (see Development) reports time for a human, not as a - pass/fail gate. -- **Rows live in a TSV, the shape in JSON.** `New` allocates once per node, so a - register of thirty thousand rows written as JSON objects would cost it a second; - a TSV is one allocation whose cells are substrings, and the JSON says only how a - row is composed. The TSV sits beside its category file, named by `rows`, so a - data directory stays a directory of categories, and one nothing names is a - load error rather than a file silently ignored. -- **A selector is bracketed, and a dot inside it is literal.** `municipality[0180]` - reads as selection to anyone who has indexed an array, and `[St. Louis]` keeps a - name whole where a colon or a dot-separated spelling could not; zsh needs the - brackets quoted, which the README's examples show. A name that spells another - row's key is refused at load rather than shadowed: a key names one row by - contract, so the name could never select its own, and the check is one lookup per - row against the key index that already exists. -- **`parent` names the link column and the table alike.** One word says both, so a - child table sits in its parent's folder and links on a column of the parent's - name; the geo plan wants exactly that, and a table needing another folder or - another column name would be asking for a second spelling. -- **After a row, a path names a column or a linked table.** `locality[Lund].address`, - a template beside the family under a selected row, is refused today; admitting it - later is additive, since a refused spelling gains a meaning and no accepted one - changes, so the door stays open for the address records the plan describes. -- **A table read into is pinned; a table rendered whole draws afresh.** A path into a - table pins its row for the render and group, as a reference path pins its level, - and a bare `{/city}` draws each time, as a bare reference does; so a bare table - beside a path into its family is refused like a bare reference beside a path into - it. A bare table reference still counts as a read for a `drawGroup`, since the group - is what draws it apart from the family's pins. -- **Every reference path into one family selects the same rows, per render and - group.** A selector pins rows for the render, and a read that draws freely before it - could pin a row the selector contradicts, so accepting both would make the result - depend on which token rendered first. Requiring one selection per family per group - is checkable at load with no data lookup beyond the selectors themselves, and the - error names the rewrite. Two selectors naming one row by key and by name compare - equal, since the load check resolves them. -- **A parent row with no child row is a load error.** A descendant is drawn inside the - nearest pinned ancestor, so every ancestor row must lead to a row at every level - below it, or a render could find nothing to draw. The import script drops or fills - such rows; the alternative, falling back to a free draw, would break the consistency - the link exists for without saying so. -- **The choice-of-rows fence guards a data file's root, and requires string fields.** - A table is a category with a TSV beside its file, so only a root choice has the - spelling the fence names; a nested choice of same-shaped templates and an inline - one keep loading. Fields must all be strings because a cell is a string node: a - choice whose items carry a nested choice is not one table but two linked ones. -- **A table is a record of string columns.** Its columns are the CSV header and the - `INSERT` column list, fixed by the TSV header, so a table is a record by - construction; every column is a string until a typed column option earns its place. -- **The key index is built at load, the rest on first draw.** A link is proved - against the parent's keys and a key's uniqueness is a data mistake, so both are - load-time; the name index and the per-parent child lists serve only a draw or a - selection, so they wait for the first one, keeping `New` linear in the bytes read. -- **Two categories may name one TSV.** Each is a view of the file with its own - format and options, at the cost of holding the rows twice, which is what a - category over a register with two natural formats asks for; a TSV nothing names - stays a load error, since that one is a file forgotten rather than shared. -- **The rows of a table are alternatives.** Only one row renders, so a cell in one - row and a cell in another never meet, and each may select its own row of another - table; the cells of one row, and whatever they reach, do meet, and so does the - format beside them. The rule the fence applies is that two reads meet unless - they sit in two rows of one table, or in a row outside a selected ancestor's. A - choice's items get no such treatment yet: two items selecting different rows - are still refused. -- **A table never reaches its own family, by any route.** A `repeat` iteration and a - `drawGroup` each draw apart on purpose, but a row that lists three localities from - other regions is the output the family exists to prevent, so the own-family fence - walks through both rather than stopping where the draw fences do. -- **Tables carrying token cells stay small.** The family fence compares the reads of - every pair of rows that can render together, so a table whose every row's cell - selects a row of another table loads in time quadratic in its rows: about a - second at four thousand rows. No shipped table carries such cells, and a register - is a column set rather than a set of references, so the fence is left as it is - until a real data set needs the indexed form. -- **A path is walked once without drawing before it is walked for real.** A path - that fails below its first level then moves no seeded stream, at the cost of one - draw-free walk per call, which allocates nothing. -- **A country's postal codes and streets are siblings under its locality.** No open - source pairs a Swedish street with its postnummer, and pairing the US through its - ZIPs would shape the two trees differently, so both draw inside the pinned - locality and an address agrees at that level. A street's own code is the exact - pairing to add when a source carries it. -- **A locale's `address` reads its country's `geo` tree, so the shipped set loads - whole.** `data/sv_SE` alone no longer loads: a test loads `data` and prefixes - the locale, and `--no-shipped-data -d` takes the whole `data` folder or a set of - one's own. -- **The default embed holds every Swedish postort the import can place and give a - street-delivery code and a street, and the US places of 25,000 or more.** Sweden - fits whole in 700 KB; every US place of 10,000 would pass a - megabyte and fetch 1,200 counties of TIGER files, so the threshold sits where the - two countries match in size, and `--min-population` and - `--streets-per-locality` on the import scripts build a fuller set. The two trees - add about 20 ms to `New`, which loads the shipped set in about 45 ms. -- **A locale's `address` restates its country record's format.** A record cannot - read another whole and keep its columns, so `sv_SE.address` names the same four - columns as `geo.SE.address`, each a reference into it, and the format appears - twice; a column is spelled the same in both, `street-number`, so the two never - disagree on a name. -- **A postort's kommun comes from its name, its tätort or its codes, never from - distance.** GeoNames leaves a fifth of Sweden's codes without a kommun and - carries stale spellings; the nearest code across a border named the wrong kommun - half the time it was tried, so a postort none of the three rules place is - dropped, as is one not cased like a place name. -- **A highway designation is not a street, and a US postal code belongs to the place - holding most of its land inside places.** `I- 55 Bus` and `US Hwy 1` carry the - most address ranges in many places and would head every address, so the import - drops names spelled as a route. A ZCTA goes to the place its largest in-place part - lies in, census-designated places left out since they never ship, and ships only - when that place does, so a few dozen places whose every code lies mostly in a - bigger neighbour ship no address; counting the land outside every place too would - drop a quarter of the places, whose codes straddle unincorporated land, for a - postal city the USPS mostly names the same way. -- **`--list` stays a plain list of paths.** It is what a script reads, so every line - has to be a path that `Fake` takes; a marker for the tables a `[selector]` follows, - or a legend above them, would make the output something to parse before use. - `--help` names the selector spelling instead, and the Table section teaches it. -- **A layout is always quoted.** A layout may carry the comma that separates - arguments, `'January 2, 2006'`, and one spelling for every layout beats a rule - about which ones need the quotes, so the bare spelling is refused naming the - quoted one. The layout is Go's reference time because the library renders with - it and a Go caller already knows it; its names are English, and a locale's own - month and weekday names are data. -- **A title is a table under `sex`.** A prefix drawn apart would put `Mr` on a record - whose `sex` column says `female`, which is the disagreement the record exists to - prevent; the tables this set already has are what a title needs, so `en_US.title` - links to `sex` as `first-name` does. Swedish has no everyday sexed honorific, so - `sv_SE.title` is a table as well but carries no `parent`. Its weight column is - `share`, not the `count` a name table carries, because the values are a curated - proportion rather than bearers anyone counted. -- **A table owns the spelling of a selector on it.** A reference reaches a table by a - path that carries no selector — `sv_SE.person.first` reads `first-name` through - `sex` — so the walk that resolved a name cannot say where a reader would type one. - The table's own location can, which is why it keeps its path, and why an ambiguity - error names `sv_SE.sex[f].first-name[Kim]` rather than the table's own name. -- **No builtin reads the clock, so a date is bounded by days, never by an age.** - An `age(min,max)` would make a seeded fixture change with the day it runs on, - which is what a seed exists to prevent; a birthdate for someone 20 to 60 is - `date(1966-01-01,2006-12-31,…)`, re-pinned as any fixture is. -- **A name column without a key resolves inside its parent.** A given name both - sexes carry is a row under each, so `name` cannot be the key; the parent's row - tells the two apart, `sex[f].first-name[Kim]`, the ambiguity error spells each - row inside its parent, and a name repeating inside one parent row is refused at - load, since nothing could then select it. -- **`misc` is what every locale shares.** A category whose facts differ by country - belongs in that country's locale, read from the register that country's own - records use; `misc` takes only sources that are international. NHTSA vPIC and - Mobility Sweden's registrations are national, so they build `en_US.car` and - `sv_SE.car`; `misc.car` waits for an international source rather than take one - of theirs. -- **A register's canonical spelling loses to the one its domain writes.** Where a - source offers several spellings of one fact, the shipped one is what records in - that domain carry. `misc.timezone` reads `zone.tab` and not the `zone1970.tab` - that supersedes it, because the latter keeps one zone per set of countries that - have agreed since 1970: it spells Sweden `Europe/Berlin`, and no Swedish system - writes that. For the same reason `misc.language` takes the ISO 639-2 register's - first synonym over CLDR, which says "Chinese, Mandarin" for `zh`. -- **`misc.territory` is the spine, and a `misc` table naming a territory links to - it.** Where every territory row has a child the column is a `parent`, and the - import drops the child rows whose territory the set does not ship — 17 of - `misc.timezone`'s, Antarctica's ten among them. Agreement across a record is - worth more than the last rows of a table. Where no such link can hold the fact - stays a column. Layer your own `misc.territory` over the shipped one and you - must layer `misc.timezone` too, or the link fails at load naming the row. -- **A table whose register publishes no frequency draws evenly.** `misc.httpmethod`, - `misc.port`, `misc.httpstatus`, `misc.mimetype` and `misc.tld` weigh every row alike, - so GET is a ninth of the methods drawn. Goal 10 keeps an authored fact out of a - sourced table, and no register publishes how often a method, a port or a TLD is used, - so a weight here would be invented. Where one exists it is read, as `misc.timezone` - reads GeoNames populations and `sv_SE.first-name` SCB bearers. -- **`misc.port` selects by number, and carries no `name`.** 29 of its services sit on - more than one port, `http-alt` on three, so `misc.port[http-alt]` could name no one - row. The number is what a port field holds anyway, and `misc.port[443].service` - reads the other way. -- **`misc.car` is one flat table, not a make linked to its models.** A row is a make - and a model drawn together, so no render pairs a Volvo with a RAV4. Two linked - tables would reach the same pairs and add a selector nothing asks for; a make alone - is `misc.car.make`. -- **`misc.territory` names its sovereign in a column, and there is no `misc.country` - table.** ISO 3166-1 codes territories, so `territory` is the honest name, and - `is_independent` in the register gives each one's state. A second table of the 195 - sovereigns would hold a `DK` row beside the territory `DK` row, both carrying - Denmark's capital, currency, flag and TLD — two owners for one fact, drifting at - the next import. Splitting the columns to avoid that is worse: put `capital` on - the territory alone and a country row can no longer name Copenhagen. The column - cannot be a `parent`, since a table never reaches its own family; a test proves - every value names a row instead. A territory the register records no state for - stands alone, which is `EH` alone, and naming one for it would be a claim - fejkdata has no business making. -- **`misc.loglevel` is a table keyed by the code, rendering POSIX's keyword.** A flat - list of names carries neither the code a PRI encodes nor a selector reaching it, and - goal 3 draws the two as one fact. The canonical spelling losing to the one its domain - writes, above, settles the rest: a configuration writes `info`, so RFC 5424's - `Informational` stays the `severity` column, and the code and the keyword hold the - two selector slots. -- **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** - The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a - bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a - host with no separator. `misc.tld[se]` misses for it, which `todo.md` carries. -- **`misc.tld` is a table of its own, and `misc.territory.tld` stays a column.** A - `parent` demands a child for every parent row, so linking them would drop every root - zone row naming no territory, which is most of them, and goal 12 holds a sourced - table whole. The loader refuses the link outright anyway: `tld` is a column of - `misc.territory`, and a table may not be named like a column of its ancestor. -- **`misc.territory` carries a currency code, it does not link to `misc.currency`.** - A `parent` demands a child for every parent row, and ISO 4217 registers codes no - country's row can name: the funds codes (Mvdol, WIR Euro, US Dollar (Next day)), - and VED beside VES, both Venezuela's, of which a country row names one. Linking - would trade the register for the link, and `misc.territory.currency` already pairs a - country with its currency in one draw. -- **An extension may name two media types.** `.xml`, `.rtf`, `.sub`, `.mpp` and `.ac` - each name two rows of `misc.mimetype`. Separating them would mean dropping a - registered type, or naming one by an extension that is not its own — `.mpt` is - Project's template, not its document. Selecting such a name is an error listing - both keys; select by type instead. -- **The Swedish ids draw Skatteverket's test series.** A Luhn-valid personnummer - over a random birth number may be a living person's; 238 and 239 after any date - are blocked from assignment, so the shipped `personnummer` and - `samordningsnummer` use those. They sit in a `birth-number` table under `sex` - rather than as a column of it: the render's shared draw of the family is what - makes the number and the name agree on sex, and `sex` stays one shape across - locales instead of collecting every sex-keyed id fact. A samordningsnummer's - day, the birthday plus 60, is drawn from 61 to 88, valid in every month, rather - than computed from the date drawn. -- **The US given names come from a mirror of the SSA file.** ssa.gov refuses a - client outside the US, so `names-us.py` reads a GitHub copy that ends at 2020, - which a count over the births since 1930 barely feels; `--names` takes the - official zip. The SSA's placeholder rows are top-1000 entries that name nobody, so - the import drops them by name rather than by a rank a regeneration would move. -- **`List` advertises direct descents only.** `region.municipality.locality` is - listed, and `region.locality` resolves too but is not: the set of every descent - through a chain of five tables is every subsequence of it, and the direct chain is - the one a reader can predict from the tables' parents. +1. **The best fake-data tool there is, and the one developers reach for.** + 1. You can install it the way you install your other tools, and people hear about + it where developers read. + 2. **Great documentation.** +2. **Enough data ships built in to fill a real test database: the CLI carries all of it, a + library imports what it needs, and nothing is fetched at run time.** + 1. Each locale covers the common categories: names, addresses, phone numbers, ids, + companies and dates. + 2. The shipped data comes in Go packages, and a program links only the packages it + imports. +3. **The code is easy to understand.** + 1. You can go from a bug to the code behind it without asking anyone. + 2. You can change one part without keeping the rest in your head. + 3. Every name says what the thing really is. + 4. Each file makes sense on its own, without another document open beside it. +4. **Every value is valid.** It passes the same check the real system would apply, + such as the checksum in a Swedish personal number. + 1. Values that belong together come from the same random pick. You choose which + ones by giving that pick a name. +5. **Templates are easy to write and read.** A template is the text you write to + describe a value, such as `{first} {last}`. + 1. Everything outside `{…}` prints exactly as written. + 2. Each `{…}` makes a new random pick. + 3. A name keeps one pick. `{/person as p}` picks a person and prints nothing. + `{p}` then prints that person, and `{p.last}` their last name. + 4. A path works like a folder path: `.` goes down a level, and `..` goes up one. + 5. `[SE]` selects one row of a table, and means that row wherever you write it. + 6. If two people could read a template differently, loading it fails, and the + error shows how to write each meaning. + 7. Whatever is well-formed and means one thing loads, even where part of it does + nothing. fejkdata does not guess at mistakes. + 8. When a clearer name turns up, it replaces the old one, even if that needs a new + major version. + 9. Everything fejkdata renders is written as a template: on the command line, in a + struct tag and in the library. +6. **Errors surface when the data loads, never mid-render, and say plainly what is + wrong.** + 1. A template that could render a value its own rules forbid, such as `NaN` or `Inf`, + fails to load instead. + 2. Data that reads something nobody provided fails to load, and the error names the + module that provides it by default, or says that none does. + 3. A module from fejkdata is proven whole before it ships; a module from anyone else + may load on first reach, and its author proves it. +7. **One command line gets you a value.** + 1. Flags work like in other command-line tools, and can go anywhere on the line. + 2. Your first template of your own needs no escaping and no options. + 3. The CLI fills in what you leave out with a sane default: all the shipped data, the + system's randomness and today's date. +8. **Human readable and editable data formats** — source data is saved in JSON or + TSV formats when possible. +9. **Flexible and extensible.** + 1. Data and functions come in modules, a Go package or a data folder, from fejkdata + or anyone else, and fejkdata's load as anyone else's do. + 2. A module names the modules it reads by default, and anything providing the same + paths can stand in for one. + 3. Loading two things under one name fails, in the library and the CLI alike, unless + you say the second replaces the first, and then every read of that name reads + the second. + 4. The builtins ship with fejkdata itself, so a template means the same in every + program. + 5. A function loaded from a module can ask for the input it needs, and gets the + randomness and the date the generator uses. + 6. A CLI carrying your own modules is a few lines of Go, and works just like + `fejkdata`. +10. **The same seed, date, data and versions always give the same output.** + 1. In the library, the caller supplies everything that can change the output: data, + seed and date. +11. **No unnecessary runtime dependencies** — use standard library by default. +12. **It is fast enough that you never notice it with the shipped data.** + 1. A render's cost grows only with the data it reaches. +13. **Strongly prioritize data sourced from good online sources** +14. **The most common data comes first,** such as names, addresses, phone numbers and + dates. +15. **Values look real by default. You can also ask for values that can never reach + anything real,** such as a domain nobody can register. ## Layout ``` -fejkdata.go Generator, New, options, the embedded data set, List +doc.go the package doc, and the vocabulary the package is written in +fejkdata.go Generator, New, options, List +index.go a source's index: its categories left unloaded, and loading one on the first call reaching it node.go the node model and JSON -> node compilation -table.go tables: the rows TSV, its options and links, row selection and draws -path.go the dotted-path walk with its selectors, and proving a path resolves -render.go Fake and the recursive renderer (choices, format strings, expansions) +table.go tables: their options, format and cells, the link to the parent beside them, and how a path passes and draws a table +name.go names: {x as n} bindings, the scopes they live in, the reads they answer, and each binding's target +namefence.go the load fences over names +pick.go a render's named picks: each level's pick key, the frame each name scope renders in, the memo a pick keeps, and the reads kept under it +env.go renderEnv: what a render reads names and rows through, the stack of frames it renders in, and the frames a read opens and finds +path.go the dotted-path walks: the paths List advertises, the sub-paths every variant of a choice carries, proving a path reaches a node, and drawing the path +render.go Fake, the recursive renderer (choices, format strings, expansions) and the reads its tokens make record.go records: Record, the JSON/CSV/SQL serializers, and their entry points -struct.go structs: FakeStruct, fake tags, and a field's Go type as its column's datatype -inline.go inline templates: Template, NewTemplate, FakeTemplate, IsTemplate, and their compile and link -template.go the {token} grammar: scanning, tokens, operands, validation, compiling a format -hold.go the hold: one draw per expansion for paths and operands, and its fences -draw.go one reference draw per render and group: draw sets, the group option, and its fence -family.go a family of linked tables: the rows a render pins, and the fence over paths into one family -reference.go reference sigils, and binding references across the tree -graph.go the render graph: edges, cycles, the repeat bound, tree walks -builtins.go the {name()} function registry and its implementations -layout.go date and time layouts: the instants one is proved against, and the two samples -checksum.go the check characters a derivation appends, and the IBAN they sit inside -transform.go the builtins that rewrite an operand's value, and the ASCII folding -calc.go the {calc()} arithmetic evaluator: parser, eval, validation -datatype.go column datatypes: DataType, where datatype and null may sit, a column's datatype -value.go the value proof: what a typed column or calc operand holds, checked at load -data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge -cmd/fejkdata/ the fejkdata CLI -data/ shipped data (JSON, and a TSV per table), embedded at build: locale folders, geo, misc -data-import/ the scripts that rebuild each sourced table (see DATA-LICENSES.md) -release-tooling/ the release CI publishes from the changelog heading +struct.go structs: FakeStruct, fake tags as one record's columns, and a field's Go type as its column's datatype +inline.go inline templates: Template, NewTemplate, FakeTemplate, IsTemplate, and their compile and resolve +format.go a format's tokens checked against their fields, and compiled into ops +arm.go the compiled read of one {a|b} alternative or operand: its head, the path to its leaf, and the op holding it +reference.go one reference's resolution: its sigil, its category, its tail +resolve.go resolving templates against the assembled tree in steps, each over every template before the next +graph.go the node graph: tree walks, containment and render edges, and the cycle and repeat fences +sorted.go sortedNames: a map's keys, sorted +calc.go the {calc()} operand checks: never a number, and a constant zero divisor +datatype.go column datatypes: DataType, a template's declared datatype, a column's datatype, and a format that only reads a column +valueproof.go the value proof: what a typed column, calc operand or struct field holds, checked at load +data.go the load path: the categories internal/datafiles hands over, compiled into one merged tree, and the load pipeline every entry point runs +cli/ the fejkdata command, as the library call Main +cmd/fejkdata/ the fejkdata binary: Main over every shipped module +internal/builtinfunc/ the {name()} functions: their checks and draws, checksums, date and time layouts, transforms and calc arithmetic +internal/datafiles/ the walk of a data tree: its folders, its category files and the rows files beside them, and its manifest +internal/datatype/ the datatype a record column holds; the root's DataType is an alias of it +internal/drawstate/ the seeded randomness and the {seq()} counters a generator draws through +internal/grammar/ how the template language is written: format tokens, paths and selectors, reference sigils, identifiers, calc syntax, and whether a string is a template or a path +internal/invariant/ the one phrase every package panics with when an invariant breaks +internal/jsonvalue/ a decoded JSON value, its numbers kept as written, the one input a template compiles from, and the kind an error names it by +internal/proven/ what a proof knows of a value, and the bounds a calc takes from its operands +internal/rows/ a table's rows: the TSV, the options proved over it, the links between tables, row selection and draws, and the pin set one path or one named pick fixes +data/ the shipped modules, each a Go package embedding its categories (JSON, and a TSV per table) and its manifest; and Modules, which lists them +data-import/ the scripts that rebuild each sourced table (see DATA-LICENSES.md), and the modules they share +docs/ the decision log, the register research behind the shipped data, the survey of other libraries, panels' scores and rulings on how readable the code is, and what each chunk cost an agent to read +release-tooling/ the release CI publishes from the changelog heading, and the binaries, packages, image and package repositories built for it testdata/ the pinned shipped shape (see Versioning) +vulnerability-filing/ the script CI runs to file each known vulnerability govulncheck reports as a `todo.md` item ``` ## Development @@ -1342,8 +1190,9 @@ docker compose run --rm dev # interactive shell Commands that rewrite source keep your file ownership when run with `--user`: ```sh -docker compose run --rm --user "$(id -u):$(id -g)" fmt # gofmt -w . -docker compose run --rm --user "$(id -u):$(id -g)" tidy # go mod tidy +docker compose run --rm --user "$(id -u):$(id -g)" fmt # gofmt -w . +docker compose run --rm --user "$(id -u):$(id -g)" generate # go generate: each data/ module's .fejkdata.json +docker compose run --rm --user "$(id -u):$(id -g)" tidy # go mod tidy ``` A `--user` command answering `permission denied` on `/cache` has met root-owned @@ -1356,20 +1205,53 @@ docker compose run --rm --user root --entrypoint chown test -R "$(id -u):$(id -g ``` Every pull request runs `docker build .` against both the latest and the lowest -supported Go, and must pass before it can be merged — unless it changes none of -the files the build and its tests read, nor the workflow itself, in which case -it's skipped (see [Decisions](#decisions)). That build is the whole gate but the -changelog check, which CI runs against the PR base — vet, complexity, format check -and tests — so run it locally before pushing: +supported Go, and must pass before it can be merged — unless it changes none of the +files the build and its tests read, nor the workflow itself, in which case it's skipped. +That build runs vet, complexity, the format check, the Go tests, the release tooling's +tests and the vulnerability filer's tests, which file an item into `todo.md` itself, so +a change to the plan's shape can fail it; the rest of the gate is the changelog check, +which CI runs against the PR base, and the package-repository checks below. Run `docker +build .` locally before pushing. The lowest Go builds the `portable` stage, which skips +the format check and both Python test suites: ```sh -docker build . # latest -docker build --build-arg GO_VERSION=1.22.12 . # lowest supported -GO_VERSION=1.22.12 docker compose run --rm test # the same tests, without the image build +docker build . # latest +docker build --build-arg GO_VERSION=1.22.12 --target portable . # lowest supported +GO_VERSION=1.22.12 docker compose run --rm test # the same tests, without the image build ``` -A change to the shipped data re-pins [`testdata/shipped_shape.txt`](testdata/shipped_shape.txt) -in its own commit: +A known vulnerability in the Go version the gate builds with, or in a module fejkdata +requires, holds the release and no merge. CI checks on every pull request, every push to +`main`, and daily. On `main` and in the daily run, CI files each vulnerability as a +`security` item atop the next release in `todo.md`, through a pull request from the branch +`vulnerabilities` unless one is open; each item names the version to raise to. When the +step "Fetch govulncheck's report from vuln.go.dev" fails, vuln.go.dev did not answer: +rerun it. When GitHub pauses the daily run after 60 days with no activity, re-enable it +under Actions. The same check: + +```sh +docker compose run --rm vulncheck +``` + +A change to Go code, `release-tooling/`, the `Dockerfile`, `.dockerignore`, the licence +files or either workflow also runs the package-repository checks. They build the release +binaries for every platform and the apt, dnf and apk repositories, sign the repositories +with throwaway keys, and install fejkdata from each repository: + +```sh +docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output /tmp/fejkdata-keys . +PACKAGES_GPG_KEY="$(cat /tmp/fejkdata-keys/gpg.asc)" PACKAGES_RSA_KEY="$(cat /tmp/fejkdata-keys/rsa.pem)" \ + docker build -f release-tooling/package-repos.Dockerfile --target checks \ + --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY . +``` + +Goal 3 is scored by a panel of simulated readers, and [`AGENTS.md`](AGENTS.md) says +what the score asks of a pull request. Goal 5 is checked against simulated template +writers from the Audience. + +A change to the shipped data runs `generate` above first and commits each module's +`.fejkdata.json` with the change, then re-pins +[`testdata/shipped_shape.txt`](testdata/shipped_shape.txt) in its own commit: ```sh REPIN=1 docker compose run --rm --user "$(id -u):$(id -g)" test @@ -1381,7 +1263,7 @@ A shipped table built from a source is rebuilt by its script under `data-import/cache/`, which is ignored by version control and grows past a gigabyte, so pass `--cache DIR` to reuse a copy you already have and delete it to fetch afresh; `geo-us.py` fetches two -TIGER/Line files per county it ships, a few hundred megabytes, `geo-se.py` needs +TIGER/Line files per county it ships, about a gigabyte, `geo-se.py` needs a Trafikverket API key, free at [data.trafikverket.se](https://data.trafikverket.se/), in `TRAFIKVERKET_API_KEY` or a `--key-file`, and the Census host behind `geo-us.py` and `names-us.py` rejects a client for a while after a burst, so `--surnames` takes @@ -1408,8 +1290,40 @@ docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/usera To release, head `CHANGELOG.md` with the version's section in place of `Unreleased` and merge: once `main` passes the gate, CI tags that commit `vX.Y.Z` and publishes -the Gitea release with the section as its body. A top heading of `[Unreleased]` -publishes nothing. +the GitHub release with the section as its body. A top heading of `[Unreleased]` +publishes nothing. CI then runs the Packages workflow on the new tag, which attaches the +binaries and packages, pushes the image to GHCR and copies it to Docker Hub, updates the +Homebrew tap and the Scoop bucket, and deploys the apt, dnf and apk repositories to GitHub +Pages. The workflow reads these repository secrets: + +- `DOCKERHUB_TOKEN`: where `larvit` is a Docker Hub organization, an organization token + that can push only to `larvit/fejkdata`; otherwise a personal token with Read & Write and + no Delete, which reaches every repository the account can write. The repository variable + `DOCKERHUB_USERNAME` names its account. +- `PACKAGES_GPG_KEY`, `PACKAGES_RSA_KEY`: the private keys that sign the repositories. +- `TAP_GITHUB_TOKEN`: a token that can push to `larvit/homebrew-tap` and + `larvit/scoop-bucket`. + +The image's Docker Hub repository and GHCR package must be public, and Pages must deploy +from GitHub Actions. To package a release again, run the workflow by hand with its tag; it +replaces what it uploaded before. For a tag older than the latest release it leaves the +tap, the bucket, the image's `latest` tags and the repositories alone, since each holds one +version. + +### Documentation + +How the README serves goal 1.2: + +- The top shows the CLI and the library side by side, each a short quickstart rendering + the same value. +- A table of the shipped data follows: every path, an example value, and what makes it + valid or where it comes from. +- Recipes show common tasks end to end. +- Every feature has its own heading, and the docs are a map of the template language. +- Every example runs as a test and shows its output. +- Each rule is stated in one place. +- Edge cases and a category's caveats sit in the reference sections, below what a first + reader needs. ## License diff --git a/arm.go b/arm.go new file mode 100644 index 0000000..9775e48 --- /dev/null +++ b/arm.go @@ -0,0 +1,117 @@ +package fejkdata + +import ( + "fmt" + + "github.com/larvit/fejkdata/internal/builtinfunc" + "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/invariant" +) + +// arm is one alternative of a {a|b} token or one operand, split into the head, +// whose node `template.startOf` finds, and the tail of a dotted path into it. +type arm struct { + spelling string // as written, for messages + head string + tail []string + levels []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it + steps []pathStep + leaves []node // every node the path may land on, one per variant it passes + kind armKind + named *nameBinding // namedRead: the binding of the name it reads through +} + +// armKind is how expand reads an arm, fixed at compile. +type armKind uint8 + +const ( + freshRead armKind = iota // drawn afresh at every read + namedRead // read through a name, kept in its pick +) + +// splitArm splits one name into head and tail. refs maps a reference to what +// resolveRefs resolved it to; before that, a reference is whole. +func splitArm(name string, refs map[string]resolvedRef) arm { + if grammar.IsRef(name) { + if r, resolved := refs[name]; resolved { + return pathArm(name, r.head, r.tail) + } + return pathArm(name, name, nil) + } + segs, err := grammar.SplitPath(name) + if err != nil || len(segs) == 1 { + return pathArm(name, name, nil) + } + return pathArm(name, segs[0], segs[1:]) +} + +func pathArm(name, head string, segs []string) arm { + return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)} +} + +// key is the key of the leaf a lands on, the one key every way of writing this read shares. +func (a arm) key() pickKey { return a.levels[len(a.levels)-1] } + +func checkSegments(a arm) error { + if len(a.tail) == 0 { + return nil + } + return grammar.CheckSegments(append([]string{a.head}, a.tail...)) +} + +// op is one compiled unit of a format string: a literal run, a name read, +// or a builtin already prepared with its args. +type op struct { + grammar.Token + arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once + call builtinfunc.Call + // operands are the fields the builtin reads, in the order its operands func + // fixed; expand reads them before the call. nil for a builtin that reads none. + operands []arm +} + +func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) { + if len(names) == 0 { + return nil, nil + } + arms := make([]arm, len(names)) + for i, name := range names { + a, err := t.compileArm(name, targets) + if err != nil { + return nil, err + } + arms[i] = a + } + return arms, nil +} + +// compileArm compiles one read into a path: from the head it names, or, for a read through a +// name, from what that name binds. +func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) { + a := splitArm(name, t.refs.byName) + if !t.isName(a.head) { + start := t.startOf(a.head) + if start == nil { + panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling)) + } + w := compilePath(start, a.tail) + a.steps, a.leaves = w.steps, w.leaves + return a, nil + } + b := t.nameScope.lookup(a.head) + if grammar.HasSelector(a.tail) { + if b.bindsField() { + return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head) + } + return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...))) + } + target := targets[b] + full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...) + if err := provePath(target.start, full, a.head, len(target.tail)); err != nil { + return a, err + } + w := compilePath(target.start, full) + a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves + a.levels = levelKeys(full, 0) + return a, nil +} diff --git a/bench_test.go b/bench_test.go index 3df242a..91d94d0 100644 --- a/bench_test.go +++ b/bench_test.go @@ -6,17 +6,17 @@ import ( "testing" ) -func benchGenerator(b *testing.B, dir string) *Generator { +func benchGenerator(b *testing.B, data Option) *Generator { b.Helper() - f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1)) + f, err := New(data, WithSeed(1)) if err != nil { b.Fatal(err) } return f } -func benchPath(b *testing.B, dir, path string) { - f := benchGenerator(b, dir) +func benchPath(b *testing.B, data Option, path string) { + f := benchGenerator(b, data) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) { } } -func BenchmarkPerson(b *testing.B) { benchPath(b, "data", "sv_SE.person") } -func BenchmarkAddress(b *testing.B) { benchPath(b, "data", "sv_SE.address") } -func BenchmarkWord(b *testing.B) { benchPath(b, "data", "sv_SE.word") } -func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data", "misc.creditcard") } -func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") } -func BenchmarkUUIDv7(b *testing.B) { benchPath(b, "data", "misc.uuid") } +func BenchmarkPerson(b *testing.B) { benchPath(b, withShipped(), "sv_SE.person") } +func BenchmarkAddress(b *testing.B) { benchPath(b, withShipped(), "sv_SE.address") } +func BenchmarkWord(b *testing.B) { benchPath(b, withShipped(), "sv_SE.word") } +func BenchmarkCreditcard(b *testing.B) { benchPath(b, withShipped(), "misc.creditcard") } +func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") } +func BenchmarkUUIDv7(b *testing.B) { benchPath(b, withShipped(), "misc.uuid") } func tmpData(b *testing.B, name, body string) string { b.Helper() @@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string { func BenchmarkCalc(b *testing.B) { dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`) - benchPath(b, dir, "inv") + benchPath(b, WithDataPath(dir), "inv") } func BenchmarkLongLiteral(b *testing.B) { dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) - benchPath(b, dir, "sql") + benchPath(b, WithDataPath(dir), "sql") } func BenchmarkRepeat(b *testing.B) { dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`) - benchPath(b, dir, "many") + benchPath(b, WithDataPath(dir), "many") } -// BenchmarkBound measures a format that binds a head and reads two paths from it — -// what a correlated pair costs against BenchmarkUnbound, the same output drawn from -// two independent fields. -func BenchmarkBound(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`) - benchPath(b, dir, "addr") +// BenchmarkNamed measures a format that binds a field to a name and reads two paths through +// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two +// independent fields. +func BenchmarkNamed(b *testing.B) { + dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`) + benchPath(b, WithDataPath(dir), "addr") } -func BenchmarkUnbound(b *testing.B) { +func BenchmarkUnnamed(b *testing.B) { dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`) - benchPath(b, dir, "addr") + benchPath(b, WithDataPath(dir), "addr") } -// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the -// depth question is about. -func BenchmarkBoundDeep(b *testing.B) { - dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`) - benchPath(b, dir, "addr") +// BenchmarkNamedDeep reads two paths through a name and an intermediate level. +func BenchmarkNamedDeep(b *testing.B) { + dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`) + benchPath(b, WithDataPath(dir), "addr") } -// BenchmarkBoundWide reads ten paths from one row: the point where the maps an -// expansion holds outgrow a single bucket. -func BenchmarkBoundWide(b *testing.B) { - dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[ +// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps +// outgrow a single bucket. +func BenchmarkNamedWide(b *testing.B) { + dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[ {"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"}, {"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`) - benchPath(b, dir, "row") + benchPath(b, WithDataPath(dir), "row") +} + +func BenchmarkFirstFake(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + f, err := New(withShipped(), WithSeed(1)) + if err != nil { + b.Fatal(err) + } + if _, err := f.Fake("sv_SE.address"); err != nil { + b.Fatal(err) + } + } } func BenchmarkNew(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - if _, err := New(WithSeed(1)); err != nil { + if _, err := New(withShipped(), WithSeed(1)); err != nil { b.Fatal(err) } } diff --git a/builtins_test.go b/builtins_test.go index adcf039..045d8cd 100644 --- a/builtins_test.go +++ b/builtins_test.go @@ -2,11 +2,14 @@ package fejkdata import ( "encoding/base64" + "fmt" "regexp" "strconv" "strings" "testing" "time" + + "github.com/larvit/fejkdata/internal/invariant" ) func TestBuiltinIDGenerators(t *testing.T) { @@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) { } } -// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on +// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on // each call, named counters independent, and the whole thing scoped to one -// session so a fresh Generator restarts at 1. -func TestBuiltinSeqPerSession(t *testing.T) { +// draw state so a fresh Generator restarts at 1. +func TestBuiltinSeqPerGenerator(t *testing.T) { f := engine(1) for i := 1; i <= 5; i++ { if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) { @@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) { if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" { t.Fatalf("repeat seq = %q, want 1,2,3", got) } - // ...and a fresh session restarts from 1. + // ...and a fresh generator restarts from 1. if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" { - t.Fatalf("new session seq = %q, want 1", got) + t.Fatalf("new generator seq = %q, want 1", got) } } @@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) { `{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join } for _, tmpl := range bad { - if _, err := compile(parse(t, tmpl)); err == nil { + if _, err := resolved(t, tmpl); err == nil { t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl) } } @@ -187,51 +190,26 @@ func ibanValid(s string) bool { return rem == 1 } -// TestRegistryShapes pins the builtin contract compileOps relies on: every entry -// supplies prep, and args parsed at compile only behind a check. -func TestRegistryShapes(t *testing.T) { - for name, b := range builtins { - if b.prep == nil { - t.Errorf("builtin %q has no prep: compileOps would call a nil func", name) - } - if b.arity != 0 && b.check == nil { - t.Errorf("builtin %q parses args in prep with no check", name) - } - } -} - -// TestArgGuardsPanic pins the guards that report a builtin arg its check should -// have rejected. No data reaches them — checkFunc runs a builtin's check before -// compileOps ever calls prep — so they are exercised directly. -func TestArgGuardsPanic(t *testing.T) { - for name, call := range map[string]func(){ - "atoi on an unvalidated arg": func() { atoi("nope") }, - "atof on an unvalidated arg": func() { atof("nope") }, - } { - mustPanic(t, name, call) - } -} - -// mustPanic fails unless call panics, which is what separates a reported -// invariant break from a silently wrong value. +// mustPanic fails unless call panics with invariant.Broken's phrase, which is what +// separates a reported invariant break from a silently wrong value. func mustPanic(t *testing.T, name string, call func()) { t.Helper() defer func() { - if recover() == nil { - t.Errorf("%s: no panic, want the invariant reported", name) + if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) { + t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r) } }() call() } func TestClassBuiltinArgs(t *testing.T) { - for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} { - if _, err := compile(parse(t, bad)); err == nil { + for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} { + if _, err := resolved(t, bad); err == nil { t.Errorf("compile(%s) = nil error, want the arg rejected", bad) } } for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} { - if _, err := compile(parse(t, ok)); err != nil { + if _, err := resolved(t, ok); err != nil { t.Errorf("compile(%s) = %v", ok, err) } } @@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) { } } -// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, -// the layout is quoted, names a field, and for time names no date field. +// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the +// layout is quoted. func TestBuiltinDateArgs(t *testing.T) { for tmpl, want := range map[string]string{ `"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01", + `"{time('2006-01-02 15:04')}"`: "write date(from,to,layout)", `"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after", - `"{date(1990-01-01,1990-01-01,'2006-01-02')}"`: "write it as text", `"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'", `"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'", `"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'", - `"{date(1990-01-01,1990-12-31,'x')}"`: "text", - `"{date(1990-01-01,1990-12-31,'')}"`: "text", `"{date(1990-01-01,1990-12-31)}"`: "3 arguments", `"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'", `"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'", `"{time(15:04)}"`: "'15:04'", - `"{time('2006-01-02 15:04')}"`: "date(", - `"{time('x')}"`: "text", - `"{date(1990-01-01,1990-12-31,'15:04')}"`: "time('15:04')", } { - _, err := compile(parse(t, tmpl)) + _, err := resolved(t, tmpl) if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want) } @@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) { } // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it -// draws every second of it, and without one it could only emit one value. +// draws every second of it, and without one it prints that day. func TestBuiltinDateSpansOneDay(t *testing.T) { f := engine(1) seen := map[string]bool{} @@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) { if len(seen) < 400 { t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen)) } - _, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`)) - if err == nil || !strings.Contains(err.Error(), "write it as text") { - t.Fatalf("one day in a date-only layout = %v, want it named a constant", err) + if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" { + t.Fatalf("one day in a date-only layout = %q, want that day", got) + } +} + +func TestClassBuiltins(t *testing.T) { + cases := map[string]*regexp.Regexp{ + `"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`), + `"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`), + `"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`), + `"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`), + `"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`), + `"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`), + } + f := engine(7) + for tmpl, re := range cases { + for i := 0; i < 100; i++ { + if got := mustRender(t, f, tmpl); !re.MatchString(got) { + t.Fatalf("%s produced %q, want %s", tmpl, got, re) + } + } + } +} + +func TestFunctionTokenLuhn(t *testing.T) { + // {luhn()} appends a Luhn check digit over the digits emitted so far in the + // current expansion (non-digits skipped but kept). It reads the output + // buffer, so a value is never re-rendered. Bodies are escaped to fix input. + f := engine(1) + cases := map[string]string{ + `"811218987{luhn()}"`: "8112189876", // personnummer body + `"7992739871{luhn()}"`: "79927398713", // classic Luhn vector + `"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept + `{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token + } + for tmpl, want := range cases { + if got := mustRender(t, f, tmpl); got != want { + t.Fatalf("%s = %q, want %q", tmpl, got, want) + } + } +} + +func TestBuiltinCompileErrors(t *testing.T) { + for _, bad := range []string{ + `"{upper(x)}"`, // count must be an integer + `"{int(a,b)}"`, // non-integer args + `"{int(5,1)}"`, // min > max + `"{nanoid(-1)}"`, // negative count + `"{float(1,2,-1)}"`, // negative decimals + `"{float(NaN,NaN,2)}"`, // bounds must be finite + `"{float(Inf,Inf,2)}"`, // same-sign infinities + `"{float(1,NaN,2)}"`, // one NaN bound + `"{float(-Inf,1,2)}"`, // one infinite bound + `"{iban(US)}"`, // unsupported country + `"{seq(a,b)}"`, // seq takes at most one name + } { + if _, err := resolved(t, bad); err == nil { + t.Errorf("compile(%s) = nil error, want error", bad) + } + } +} + +func TestArgErrorsNameTheSpelling(t *testing.T) { + if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") { + t.Errorf("compile(hex past the range) = %v, want the maximum named", err) + } +} + +func TestASignOrLeadingZeroIsTheInteger(t *testing.T) { + for long, short := range map[string]string{ + `"{float(1,2,02)}"`: `"{float(1,2,2)}"`, + `{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`, + `"{digits(+5)}"`: `"{digits(5)}"`, + `"{digits(05)}"`: `"{digits(5)}"`, + `"{int(007,9)}"`: `"{int(7,9)}"`, + `"{int(+1,5)}"`: `"{int(1,5)}"`, + } { + sameRenders(t, long, short) } } diff --git a/calc.go b/calc.go index eb0c2c4..8339e7d 100644 --- a/calc.go +++ b/calc.go @@ -2,392 +2,149 @@ package fejkdata import ( "fmt" - "math" + "slices" "strconv" "strings" - "unicode" + + "github.com/larvit/fejkdata/internal/builtinfunc" + "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/invariant" ) -// calcNode is a parsed expression node. It evaluates over the operand values expand -// read, so the evaluator touches neither the rng nor the node tree. -type calcNode interface { - eval(operands []string) float64 -} - -type calcNum float64 // a number literal -type calcVar string // a sibling-field name, before indexVars places it -type calcIdx int // an operand, by its position in the values expand read -type calcNeg struct{ x calcNode } -type calcBin struct { // a + - * / b - op byte - l, r calcNode -} - -func (n calcNum) eval([]string) float64 { return float64(n) } - -func (n calcIdx) eval(operands []string) float64 { - v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64) - if err != nil { - return math.NaN() // a non-numeric operand stays visible, never an error - } - return v -} - -// eval on an unplaced name cannot happen: calcPrep runs indexVars over every -// expression it compiles, so only a calcIdx reaches a render. It panics rather -// than returning NaN, so a node kind indexVars forgets is a stack trace and not a -// silently wrong number. -func (n calcVar) eval([]string) float64 { - panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n))) -} - -func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) } - -func (n calcBin) eval(operands []string) float64 { - l, r := n.l.eval(operands), n.r.eval(operands) - switch n.op { - case '+': - return l + r - case '-': - return l - r - case '*': - return l * r - default: // '/' - return l / r - } -} - -// checkCalc validates a calc token at compile time: a parseable expression whose -// operands all name existing fields, and an optional non-negative integer dp. It -// is a builtin check (fields first), so calc dispatches through the registry like -// every other {name(args)} function. -func checkCalc(fields map[string]node, args []string) error { - if len(args) < 1 || len(args) > 2 { - return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args)) - } - expr, err := parseCalc(args[0]) - if err != nil { - return fmt.Errorf("calc(%q): %w", args[0], err) - } - for _, name := range calcVars(expr) { - operand, ok := fields[name] - if !ok { - return fmt.Errorf("calc(%q): no field %q", args[0], name) +// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no +// field is a name, which checkCalcNames checks once names resolve. +func checkCalcFields(args []string, fields map[string]node) error { + return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node { + if n, ok := fields[name]; ok { + return []node{n} } - if text, never := neverNumeric(operand); never { - return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text) + return nil + }) +} + +// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field. +func checkCalcNames(label string, t *template) error { + for _, o := range t.compiled.ops { + if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) { + continue + } + if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil { + return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err) } } - if divisor, zero := constantZeroDivisor(expr, fields); zero { - return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor) + return nil +} + +// operandNodes lists every node each operand of o may render, once its template and names resolve. +func operandNodes(o op) func(name string) []node { + return func(name string) []node { + a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })] + if len(a.leaves) == 0 { + panic(invariant.Broken("calc operand %q was read before it was compiled", name)) + } + return a.leaves } - if len(args) == 2 { - dp, err := plainInt(args[1]) - if err != nil { - return fmt.Errorf("calc decimals %w", err) - } - if dp < 0 || dp > maxDecimals { - return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals) +} + +// checkOperands refuses an operand that is never a number and a division by a constant zero. +// operand lists every node an operand may render, nil while that is unknown. +func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error { + for _, name := range c.Operands { + if rendered, never := allNeverNumeric(operand(name)); never { + return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered) } } + if divisor, zero := constantZeroDivisor(c.Expr, operand); zero { + return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor) + } return nil } // constantZeroDivisor finds a division whose right side is a constant zero: number // literals and fixed operands folded, anything that varies left unknown. -func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) { +func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) { switch n := n.(type) { - case calcNeg: - return constantZeroDivisor(n.x, fields) - case calcBin: - if n.op == '/' { - if v, known := constantValue(n.r, fields); known && v == 0 { - return calcText(n.r), true + case grammar.CalcNeg: + return constantZeroDivisor(n.X, operand) + case grammar.CalcBin: + if n.Operator == '/' { + if v, known := constantValue(n.R, operand); known && v == 0 { + return grammar.CalcText(n.R), true } } - if d, zero := constantZeroDivisor(n.l, fields); zero { + if d, zero := constantZeroDivisor(n.L, operand); zero { return d, true } - return constantZeroDivisor(n.r, fields) + return constantZeroDivisor(n.R, operand) } return "", false } // constantValue evaluates an expression whose every operand is fixed. -func constantValue(n calcNode, fields map[string]node) (float64, bool) { +func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) { switch n := n.(type) { - case calcNum: + case grammar.CalcNum: return float64(n), true - case calcVar: - t, ok := fields[string(n)].(*template) - if !ok || !t.fixed || t.repeat > 1 { + case grammar.CalcVar: + nodes := operand(n.Name) + if len(nodes) != 1 { return 0, false } - v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64) + t, ok := nodes[0].(*template) + if !ok || t.repeat != 1 { + return 0, false + } + lit, fixed := t.fixedText() + if !fixed { + return 0, false + } + v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64) return v, err == nil - case calcNeg: - v, ok := constantValue(n.x, fields) + case grammar.CalcNeg: + v, ok := constantValue(n.X, operand) return -v, ok - case calcBin: - l, lok := constantValue(n.l, fields) - r, rok := constantValue(n.r, fields) + case grammar.CalcBin: + l, lok := constantValue(n.L, operand) + r, rok := constantValue(n.R, operand) if !lok || !rok { return 0, false } - return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true + return builtinfunc.Arith(n.Operator, l, r), true } return 0, false } -// calcText spells an expression node the way an author would read it. -func calcText(n calcNode) string { - switch n := n.(type) { - case calcNum: - return strconv.FormatFloat(float64(n), 'f', -1, 64) - case calcVar: - return string(n) - case calcNeg: - return "-" + calcText(n.x) - case calcBin: - return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")" - } - return "?" -} - -// neverNumeric reports a node no render of which is a number: a null, fixed text that -// does not parse, or a choice of only such items. text is one such render. +// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed +// text that does not parse, or a choice of only such items. text is one such render. func neverNumeric(n node) (text string, never bool) { switch n := n.(type) { - case *null: + case *nullItem: return "", true case *template: - if !n.fixed || n.repeat > 1 { + lit, fixed := n.fixedText() + if n.repeat == 0 { + return "", true + } + if !fixed || n.repeat != 1 { return "", false } - if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil { - return n.lit, true + if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil { + return lit, true } case *choice: - for _, it := range n.items { - t, itemNever := neverNumeric(it) - if !itemNever { - return "", false - } - text = t - } - return text, true + return allNeverNumeric(n.items) } return "", false } -// calcPrep parses the expression and decimals once, at compile time, and places each -// operand name at the position expand will read it into. checkCalc proved both args -// valid, so no step here can fail; dp -1 prints the minimal form. -func calcPrep(args []string) callFn { - expr, err := parseCalc(args[0]) - if err != nil { // a nil AST would be a nil dereference per render, with no message - panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err)) - } - at := make(map[string]int) - for i, name := range calcVars(expr) { - at[name] = i - } - placed := indexVars(expr, at) - dp := calcDecimals(args) - return func(_ *session, _ string, operands []string) string { - return formatFloat(placed.eval(operands), dp) - } -} - -// calcDecimals is a calc's decimals count, or -1 for the shortest form. -func calcDecimals(args []string) int { - if len(args) == 2 { - return atoi(args[1]) - } - return -1 -} - -// indexVars replaces each operand name with its position in the values expand reads. -// Both sides take that order from calcVars, so they cannot drift. -func indexVars(n calcNode, at map[string]int) calcNode { - switch n := n.(type) { - case calcVar: - i, placed := at[string(n)] - if !placed { // calcVars named every operand, so a miss means the two disagree - panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n))) +// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes. +func allNeverNumeric(nodes []node) (text string, never bool) { + for _, n := range nodes { + t, nodeNever := neverNumeric(n) + if !nodeNever { + return "", false } - return calcIdx(i) - case calcNeg: - return calcNeg{indexVars(n.x, at)} - case calcBin: - return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)} + text = t } - return n -} - -// calcOperands lists the sibling-field names a calc's args read. checkCalc reports -// an expression that does not parse, so one that does not simply names nothing. -func calcOperands(args []string) []string { - if len(args) == 0 { - return nil - } - expr, err := parseCalc(args[0]) - if err != nil { - return nil - } - return calcVars(expr) -} - -// calcVars lists the distinct field names an expression reads, in the order it first -// names each. That order is the contract between expand, which reads the operands -// into a slice, and indexVars, which places each name at its position in it. -func calcVars(n calcNode) []string { - var out []string - seen := map[string]bool{} - var walk func(calcNode) - walk = func(n calcNode) { - switch n := n.(type) { - case calcVar: - if name := string(n); !seen[name] { - seen[name] = true - out = append(out, name) - } - case calcNeg: - walk(n.x) - case calcBin: - walk(n.l) - walk(n.r) - } - } - walk(n) - return out -} - -// calcParser is a recursive-descent parser over the expression runes, threading -// expr -> term -> factor for the standard * / before + - precedence. -type calcParser struct { - rs []rune - pos int -} - -// parseCalc parses a whole expression, requiring it to consume all input. -func parseCalc(expr string) (calcNode, error) { - p := &calcParser{rs: []rune(expr)} - if p.space(); p.pos >= len(p.rs) { - return nil, fmt.Errorf("empty expression") - } - n, err := p.expr() - if err != nil { - return nil, err - } - if p.space(); p.pos != len(p.rs) { - return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:])) - } - return n, nil -} - -func (p *calcParser) space() { - for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) { - p.pos++ - } -} - -func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') } -func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') } - -// binary parses a left-associative run of next() operands joined by the given -// operators, the one shape expr and term share. -func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) { - n, err := next() - if err != nil { - return nil, err - } - for { - p.space() - if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) { - return n, nil - } - op := byte(p.rs[p.pos]) - p.pos++ - r, err := next() - if err != nil { - return nil, err - } - n = calcBin{op, n, r} - } -} - -// factor is a table-shaped scanner, one case per token kind, kept whole on purpose. -func (p *calcParser) factor() (calcNode, error) { - p.space() - if p.pos >= len(p.rs) { - return nil, fmt.Errorf("unexpected end of expression") - } - switch c := p.rs[p.pos]; { - case c == '-': - p.pos++ - x, err := p.factor() - if err != nil { - return nil, err - } - return calcNeg{x}, nil - case c == '(': - p.pos++ - n, err := p.expr() - if err != nil { - return nil, err - } - if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' { - return nil, fmt.Errorf("missing ')'") - } - p.pos++ - return n, nil - case c == '.' || c >= '0' && c <= '9': - return p.number() - case c == '_' || unicode.IsLetter(c): - return p.ident() - default: - return nil, fmt.Errorf("unexpected %q", string(c)) - } -} - -func (p *calcParser) number() (calcNode, error) { - start, dot := p.pos, false - for p.pos < len(p.rs) { - if c := p.rs[p.pos]; c >= '0' && c <= '9' { - p.pos++ - } else if c == '.' && !dot { - dot, p.pos = true, p.pos+1 - } else { - break - } - } - v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64) - if err != nil { - return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos])) - } - return calcNum(v), nil -} - -// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-' -// is always the minus operator, so a hyphenated field name can't be an operand. -func (p *calcParser) ident() (calcNode, error) { - start := p.pos - for p.pos < len(p.rs) { - if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) { - p.pos++ - } else { - break - } - } - return calcVar(string(p.rs[start:p.pos])), nil -} - -func contains(bs []byte, b byte) bool { - for _, x := range bs { - if x == b { - return true - } - } - return false + return text, len(nodes) > 0 } diff --git a/calc_test.go b/calc_test.go index 7a8b7bd..0de014a 100644 --- a/calc_test.go +++ b/calc_test.go @@ -3,8 +3,11 @@ package fejkdata import ( "fmt" "math" + "strconv" "strings" "testing" + + "github.com/larvit/fejkdata/internal/grammar" ) // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus @@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) { if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" { t.Fatalf("calc(a - b) = %q, want 7", got) } + if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" { + t.Fatalf("calc(b * (a - b)) = %q, want 21", got) + } } // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does @@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) { } } -// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that -// is not a calc, and for one whose expression does not parse: nothing, either way. -// checkCalc is what reports a bad expression, so the callers that run after it -// never meet one — but they must not have to depend on that order to be safe. -func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) { - for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} { - if got := tokenOperands(body); got != nil { - t.Errorf("tokenOperands(%q) = %v, want none", body, got) +// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a +// call that is not a calc, and for one whose expression does not parse: none, either way. +func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) { + for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} { + if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil { + t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err) } } - if got := tokenOperands("calc(net * qty)"); len(got) != 2 { - t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got) - } -} - -// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed -// the expression before prep sees it, and indexVars places every name calcVars -// read, so a break in either is reported rather than computed around. -func TestCalcGuardsPanic(t *testing.T) { - for name, call := range map[string]func(){ - "prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) }, - "an operand name never placed": func() { calcVar("n").eval(nil) }, - "indexVars on a name it did not read": func() { indexVars(calcVar("n"), map[string]int{}) }, - } { - mustPanic(t, name, call) + if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 { + t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err) } } // --- one draw, one value: a calc operand reads the expansion's draw --- -// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand. -// A field the format renders and a calc reads is drawn once per expansion, so the -// operand shown is the operand computed — the correlation the dotted-path rule -// already gives a level (see bound_test.go), applied to a plain sibling. -func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) { +// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format +// shows: the format and the calc read one name. +func TestANamedCalcOperandIsTheOneShown(t *testing.T) { dir := writeData(t, map[string]string{ - "inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, + "inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, }) f := newGenerator(t, dir, WithSeed(3)) for i := 0; i < 300; i++ { @@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) { } } -// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the -// expansion, not to the calc, so the bare token rendering the same name reads it too. -func TestCalcOperandSharesOneDraw(t *testing.T) { - dir := writeData(t, map[string]string{ - "same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`, - }) +// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does. +func TestACalcOperandDrawsAfresh(t *testing.T) { + dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`}) f := newGenerator(t, dir, WithSeed(5)) for i := 0; i < 200; i++ { - got := fake(t, f, "same") - if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] { - t.Fatalf("same = %q, want one value twice", got) - } - } -} - -// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is -// held, so an ordinary {w} {w} still draws twice. -func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) { - dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`}) - f := newGenerator(t, dir, WithSeed(5)) - for i := 0; i < 200; i++ { - if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] { + if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] { return } } - t.Fatal("{w} {w} never differed in 200 draws, want two independent draws") + t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws") } -// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is -// its own expansion, so it draws again while staying self-consistent. -func TestCalcHoldIsPerExpansion(t *testing.T) { +// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each +// iteration, staying one value within it. +func TestANamedCalcOperandPicksEachIteration(t *testing.T) { dir := writeData(t, map[string]string{ - "rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`, + "rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`, }) f := newGenerator(t, dir, WithSeed(11)) varied := false @@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) { } } -// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner -// {v} and its calc agree with each other, not with the outer pair. -func TestCalcHoldIsPerTemplate(t *testing.T) { +// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner +// pair agrees with itself, not with the outer pair. +func TestTwoNamesPickApart(t *testing.T) { dir := writeData(t, map[string]string{ - "nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"], - "inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`, + "nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"], + "inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`, }) f := newGenerator(t, dir, WithSeed(13)) differed := false @@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { `{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`, `{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`, } { - _, err := compile(parse(t, src)) + _, err := resolved(t, src) if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") { t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want) } @@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { `{"format":"{calc(x * 2)}","x":["1","abc"]}`, `{"format":"{calc(x * 2)}","x":"{digits(2)}"}`, } { - if _, err := compile(parse(t, ok)); err != nil { + if _, err := resolved(t, ok); err != nil { t.Errorf("compile(%s) = %v, want it accepted", ok, err) } } @@ -250,14 +223,14 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) { func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { for src, want := range map[string]string{ - `"{calc(1/0)}"`: "divides by 0", - `"{calc(2/(1-1))}"`: "divides by (1 - 1)", + `"{calc(1/0)}"`: "divides by 0", + `"{calc(2/(1-1))}"`: "divides by (1 - 1)", `{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y", `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)", `{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0", `{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0", } { - if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { + if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) { t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want) } } @@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { } t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws") } + +func TestCalcCompileErrors(t *testing.T) { + for _, bad := range []string{ + `"{calc(1/0)}"`, // a constant zero divisor + `{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor + `"{calc()}"`, // calc needs an expression + `"{calc(1 +)}"`, // dangling operator + `"{calc((1 + 2)}"`, // unbalanced parenthesis + `"{calc(1 2)}"`, // two operands, no operator + `"{calc(price)}"`, // operand names no field + `"{calc(1, 2, 3)}"`, // too many args + `"{calc(1, x)}"`, // decimals arg not an integer + `"{calc(1, -1)}"`, // decimals negative + } { + if _, err := resolved(t, bad); err == nil { + t.Errorf("compile(%s) = nil error, want error", bad) + } + } +} + +func TestCalcReadsAName(t *testing.T) { + dir := writeData(t, map[string]string{ + "n": `["2","3","5","7"]`, + "card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 50; i++ { + var k, double int + got := fake(t, f, "card") + if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 { + t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got) + } + } +} + +func TestCalcReadsANameItReadsNowhereElse(t *testing.T) { + dir := writeData(t, map[string]string{ + "n": `["2","3"]`, + "card": `"{/n as k}{calc(k * 2)}"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + if got := fake(t, f, "card"); got != "4" && got != "6" { + t.Fatalf("card = %q, want 4 or 6", got) + } +} + +func TestCalcRefusesABadOperand(t *testing.T) { + for card, want := range map[string]string{ + `"{/word as w}{w}{calc(w * 2)}"`: `operand "w" is never a number: it renders "def"`, + `"{/zero as z}{z}{calc(1 / z)}"`: `divides by z, which is always zero`, + `"{calc(nope * 2)}"`: `no field "nope"`, + `"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`, + } { + dir := writeData(t, map[string]string{ + "word": `["abc","def"]`, + "zero": `"0"`, + "card": card, + }) + if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("New with card %s = %v, want an error containing %q", card, err, want) + } + } +} + +func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) { + _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{ + "row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`, + }))) + if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") { + t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err) + } +} + +func TestATypedColumnProvesACalcOverAName(t *testing.T) { + dir := writeData(t, map[string]string{ + "n": `"{int(1,9)}"`, + "order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.FakeRecord("order") + if err != nil { + t.Fatal(err) + } + if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 { + t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value) + } + if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) { + t.Fatalf("JSON = %s, want double written as a number", r.JSON()) + } + bad := writeData(t, map[string]string{ + "word": `["1","x"]`, + "order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`, + }) + if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) { + t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err) + } +} + +func TestARepeatOfZeroIsNeverANumber(t *testing.T) { + if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) { + t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err) + } +} diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 0000000..e6a96f7 --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,706 @@ +// Package cli is the fejkdata command: it renders the template on its stdin with fake +// values from the modules it is given and any directories layered over them. A program +// passing its own modules to Main is a fejkdata CLI carrying them, for example +// os.Exit(cli.Main(append(data.Modules(), mydata)...)). +package cli + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "math/rand/v2" + "os" + "path/filepath" + "reflect" + "runtime/debug" + "sort" + "strconv" + "strings" + + "github.com/larvit/fejkdata" +) + +const usage = `Usage: