Test filing each known vulnerability as a security item atop the next release

.github/workflows/packages.yml

@@ -0,0 +1,162 @@
+name: Packages
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        required: true
+    secrets:
+      TAP_GITHUB_TOKEN:
+        required: true
+      PACKAGES_GPG_KEY:
+        required: true
+      PACKAGES_RSA_KEY:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The released tag to package, such as v0.1.0
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: packages
+  cancel-in-progress: false
+
+jobs:
+  packages:
+    name: Binaries, packages and image for ${{ inputs.tag }}
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+      packages: write
+    outputs:
+      latest: ${{ steps.latest.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.tag }}
+          path: src
+          fetch-depth: 0
+          persist-credentials: false
+      # Check out the packaging config at the workflow's commit, so a tag cut before the
+      # config existed can still be packaged.
+      - uses: actions/checkout@v7
+        with:
+          path: tooling
+          persist-credentials: false
+      - name: The Go version the gate tests
+        id: go
+        run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: false
+      - name: Channels holding one version
+        id: latest
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
+          if [ "$latest" = "$TAG" ]; then
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
+            echo "OLDER_TAG=1" >> "$GITHUB_ENV"
+          fi
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: v2.18.3
+          workdir: src
+          args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      - name: The built binary and the published image print the tag
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
+          test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  dockerhub:
+    name: Docker Hub image for ${{ inputs.tag }}
+    needs: packages
+    runs-on: ubuntu-26.04
+    permissions:
+      packages: read
+    steps:
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Copy the GHCR image to Docker Hub, and check it prints the tag
+        env:
+          LATEST: ${{ needs.packages.outputs.latest }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
+          if [ "$LATEST" = true ]; then
+            tags+=(-t docker.io/larvit/fejkdata:latest)
+          fi
+          docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
+          test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  repos:
+    name: apt, dnf and apk repositories for ${{ inputs.tag }}
+    needs: packages
+    if: needs.packages.outputs.latest == 'true'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: The release's Linux packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
+      - uses: docker/setup-buildx-action@v4
+      - name: Install fejkdata from each repository, then export them
+        env:
+          PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+          PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          build() {
+            docker buildx build -f release-tooling/package-repos.Dockerfile \
+              --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
+              --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
+          }
+          build --target checks --output type=local,dest=checks
+          build --target site --output type=local,dest=site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - id: deploy
+        uses: actions/deploy-pages@v5

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * *'
 
 permissions:
   contents: read
@@ -11,14 +14,15 @@ permissions:
 jobs:
   test:
     name: vet + fmt + tests
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
           fetch-depth: 0
       - name: Relevant changes
         id: changes
+        if: github.event_name != 'schedule'
         run: |
           if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
             git fetch --no-tags origin "$GITHUB_BASE_REF"
@@ -26,13 +30,19 @@ jobs:
           else
             base='${{ github.event.before }}'
           fi
-          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata README.md; then
             echo "code=false" >> "$GITHUB_OUTPUT"
           else
             echo "code=true" >> "$GITHUB_OUTPUT"
           fi
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
+            echo "packaging=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "packaging=true" >> "$GITHUB_OUTPUT"
+          fi
           echo "base=$base" >> "$GITHUB_OUTPUT"
       - name: CHANGELOG entry for data changes
+        if: github.event_name != 'schedule'
         run: |
           base='${{ steps.changes.outputs.base }}'
           if ! git diff --quiet "$base"...HEAD -- data testdata/shipped_shape.txt && git diff --quiet "$base"...HEAD -- CHANGELOG.md; then
@@ -48,22 +58,54 @@ jobs:
       # failure is version-specific.
       - name: Lowest supported Go
         if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
-        run: docker build --build-arg GO_VERSION=1.22.12 .
+        run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
+      # Runs on every event, and daily, since a vulnerability is published without a change
+      # here; `--no-cache-filter` keeps a cached pass from hiding one.
+      - name: Known vulnerabilities in this Go release and the module
+        if: ${{ !cancelled() }}
+        run: docker build --no-cache-filter vulnerabilities --target vulnerabilities .
+      - name: Package repositories, signed with throwaway keys
+        if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
+        run: |
+          docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
+          PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
+            docker build -f release-tooling/package-repos.Dockerfile --target checks \
+            --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
 
   release:
-    name: Gitea release from CHANGELOG.md
+    name: Release from CHANGELOG.md
     needs: test
     if: github.event_name == 'push'
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     permissions:
       contents: write
+    outputs:
+      tag: ${{ steps.publish.outputs.tag }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - run: python3 release-tooling/publish_release.py
+      - id: publish
+        run: python3 release-tooling/publish_release.py
         env:
-          GITEA_API_URL: ${{ github.api_url }}
+          FORGE_API_URL: ${{ github.api_url }}
-          GITEA_REPOSITORY: ${{ github.repository }}
+          FORGE_REPOSITORY: ${{ github.repository }}
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SHA: ${{ github.sha }}
+
+  packages:
+    needs: release
+    if: needs.release.outputs.tag != ''
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/packages.yml
+    with:
+      tag: ${{ needs.release.outputs.tag }}
+    secrets:
+      TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+      PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

AGENTS.md

@@ -1,27 +1,136 @@
 # Rules
 
-- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check.
+- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate>`; the merge gate is `docker build .`, CI's changelog check, its known-vulnerabilities check and, for a change the README's Development section lists, its package-repository checks.
 - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
 - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
 - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
-- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it
+- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
-  offers fails with "is it still open?", as does a merge whose required checks are
+  only, `gh pr merge --rebase` where the branch is already on top of main.
-  still pending, so read the checks before believing the style is the problem. Use
+- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
-  `tea api /repos/{owner}/{repo}/pulls/<n>/merge -f Do=fast-forward-only`.
+  its status once when a PR is marked ready, answer a review if one is there, and never
-- Gitea queues Actions runs that `/repos/{owner}/{repo}/actions/tasks` does not
+  wait for one.
-  list, so an empty task list says nothing about whether CI ran; read
+- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
-  `/repos/{owner}/{repo}/commits/<sha>/status`.
+  nothing else merges — no feature, no category, no data — bar a security fix, a
-- README goal 2 is gated at 7.0 on the `comprehension-panel` skill, and below it
+  dependency bump, a repair of behaviour the README documents, the infrastructure
-  nothing else merges — no feature, no category, no data — bar a security fix and a
+  the round itself runs on, and a chunk the comprehension-floor decision lists as
-  dependency bump. `todo.md` carries the round: the nine-seat findings run at depth 1,
+  shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
-  a PR per item it names, then the run again, until the score passes. At or above 7.0
+  findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
+  again, until the score passes. At or above 7.0
   every pull request scores with the four-seat run and answers it in one run. No merge
-  lowers the last score.
+  lowers the last score its own panel recorded: the nine-seat round and the four-seat
+  run are two ratchets, never compared with each other. A lower score blocks the merge
+  even where it may be panel noise: fix what the seats name until the score rises. A PR
+  that only records a run's scores and files what its seats name merges even when a
+  score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
 - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
-- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
-- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
 - Prose naming a source states what that source states: read the register's own field
   before paraphrasing it, and name the register you read before claiming none
   publishes a spelling.
 - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
-- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed.
+- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
+
+# Decisions
+
+In [docs/decisions.md](docs/decisions.md):
+
+- GitHub is canonical, and the module path names it
+- The Gitea copy stays, as a pull mirror of GitHub
+- The vocabulary sits below `doc.go`'s package clause, not in the package doc
+- Options and fields share one namespace
+- `{a|b}` stays beside nested choices
+- Flags follow getopt_long
+- A struct tag is a template by its shape
+- The CLI renders exactly the template on its stdin, and adds nothing
+- The CLI reads only the library's public API
+- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+- `FakeTemplate` and `NewTemplate` both stay
+- The shipped data is embedded, not discovered
+- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+- Every `{…}` draws afresh, and only a name keeps a pick
+- Reference sigils follow the filesystem
+- A change to what exists is a major; a minor only adds
+- An error is a contract by what it names, not its bytes
+- A load reports every mistake at once
+- Formats are checked after all data is loaded, so their errors name the category
+- Raising the lowest supported Go is a major
+- The format check runs on the latest Go only
+- The changelog heading is the one spelling of a release; CI cuts the tag
+- A `--data-path` override rebinds every reference to the category it replaces
+- The repeat cap bounds renders, not bytes
+- 64-bit targets only
+- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+- Samples say what they emit, transforms what they do
+- A record is a template seen as columns; a Go struct is the one second schema
+- A struct's records follow Go's field access, and compile on first use
+- A category may reference itself, and a record's fences run at load
+- A record's column set is fixed before the first draw
+- Null is a `null` item, not a rate
+- A typed column holds one value, not composed text
+- A column that only reads one reference or name is the column it reads
+- A typed column's calc is refused unless proven
+- `Column` carries text, not a Go value
+- A unit takes the stem of what it is, and a file the stem of the units it holds
+- One name, one meaning
+- The performance gate asserts allocations, not wall-clock time
+- Rows live in a TSV, the shape in JSON
+- A selector is bracketed, and a dot inside it is literal
+- `parent` names the link column and the table alike
+- After a row, a path names a column or a linked table
+- A selected row is fixed, not drawn
+- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+- A table selects by one key column, a code, never a free-form name
+- A parent row with no child row is a load error
+- A table is a record of string columns
+- The key map is built at load, the rest on first draw
+- Two categories may name one TSV
+- A cell may read a reference, and each row its own
+- A path is walked once without drawing before it is walked for real
+- A country's postal codes and streets are siblings under its locality
+- A locale package imports its country's geo package, and its `Modules()` returns both
+- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+- A locale's `address` binds its country's record once and prints it whole
+- A postort's kommun comes from its name, its tätort or its codes, never from distance
+- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+- `--list` stays a plain list of paths
+- A layout is always quoted
+- A title is a table under `sex`
+- A table owns the spelling of a selector on it
+- No builtin reads the clock, so a date is bounded by days, never by an age
+- `misc` is what every locale shares
+- A register's canonical spelling loses to the one its domain writes
+- `misc.territory` is the spine, and a `misc` table naming a territory links to it
+- A table whose register publishes no frequency draws evenly
+- `misc.timezone` weighs a zone by the people living in it
+- `misc.car` is one flat table, not a make linked to its models
+- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+- `misc.tld` keys carry the leading dot, where other tables key on a bare code
+- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+- `misc.territory` carries a currency code, it does not link to `misc.currency`
+- An extension may name two media types
+- The Swedish ids draw Skatteverket's test series
+- The US given names come from a mirror of the SSA file
+- `List` advertises direct descents only
+- A path draws through its compiled steps
+- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+- A country with no open address register covering it whole ships its places and postal codes without streets
+- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+
+# Scoring run
+
+The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
+
+- `{language}`: Go
+- `{kind}`: a Go library and CLI that renders fake data from JSON templates
+- `{domain}`: fake test-data generation and template languages
+- `{domain docs}`: the documentation of any fake-data library, such as Faker
+- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another

CHANGELOG.md

@@ -8,85 +8,99 @@ replacement, and each removed path, column or flag.
 
 ### Added
 
-- First release: the CLI, the library and the shipped data set.
+- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
-- Table categories: a category JSON naming a `rows` TSV beside it, with the options
+  dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
-  `key`, `name`, `weight` and `parent`; a path selects a row by key or name,
+  the CLI installs without Go. 0.1.0 ships them too; the README's
-  `misc.territory[SE]`, and descends to a linked table by name; linked tables draw
+  [Install](https://github.com/larvit/fejkdata#install) lists each command.
-  consistently within one render and draw group. `rows` is an option, so no
+
-  template may carry a field of that name. A `name` without a `key` resolves
+## [0.1.0] - 2026-10-13
-  inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
+
-  `parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any
+The first release: a Go library and a CLI that render locale-aware fake data from JSON
-  table above it, a table whose format or cell references its own family, and,
+templates, and the data they ship with. The
-  within one render and draw group, a path drawing a table another path selects a
+[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
-  row of, or two paths pinning different rows of one table.
+Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
-- `New` refuses a root choice of templates sharing one format and one set of string
+library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
-  fields, naming the rows TSV to write instead.
+and any release may change what a seed draws, so pin one exact version
-- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and
+([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
-  `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has
+
-  a capital, a currency and a TLD, with the columns `calling-code`, `capital`,
+### Added
-  `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country`
+
-  holding the alpha-2 code of the sovereign state it belongs to and its own where it
+- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
-  is one; `misc.currency` the
+  `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
-  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
+  repeats a run's output while the version and data stay the same, and without it each
-  added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a
+  run takes a new seed; `-n` renders the template that many times, and `--list`
-  639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each
+  prints the paths the data offers.
-  code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each
+  `--data-path` layers a directory of your own data over the shipped set, and
-  type mime-db records as IANA-registered and gives a filename extension.
+  `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
-  `DATA-LICENSES.md` lists each source.
+  a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
-- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone`
+  error such as data that fails to load, and `2` on misuse.
-  is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the
+- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
-  zone's standard UTC offset and `territory` linking to `misc.territory`, so
+  `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
-  `misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory
+  from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
-  `misc.territory` does not ship, Antarctica's among them, and the constant `UTC`
+  without data, it fails and names `WithDataFS`. The same seed repeats the draws
-  are gone; spell that one as the text `"UTC"`. A `weight` column carries the
+  while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
-  population GeoNames records in each zone, so a draw favours the zones people live
+  and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
-  in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`,
+  `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
-  `device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a
+  `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
-  `model` column.
+  `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
-- `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries as tables.
+  and `IsTemplate` tells an inline template from a path. Data that fails to load gives
-  `misc.httpmethod` is the eight methods RFC 9110 defines and PATCH, carrying
+  an error matching `ErrLoad`, and a record with no fields gives one matching
-  `safe` and `idempotent` as `true` or `false`; `misc.protocol` the keyword, its
+  `ErrNoColumns`.
-  `number` and its `name`, selectable by either; `misc.port` renders a TCP port number,
+- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
-  with the IANA `service` name beside it, keeping the first service the registry
+  `cli.Main(modules...)` runs it over the modules given, so
-  describes for a port so a number selects one row.
+  `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
-- `misc.tld` is the IANA root zone as a table: every delegated TLD keyed with its
+  That CLI names itself by the file it runs from, and its `--version` prints its own
-  leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode`
+  module's version before fejkdata's.
-  column holding the form the register displays, so `misc.tld[.рф]` selects the row
+- The template language
-  `.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship.
+  ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
-- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes,
+  Every value in a data file is a string, a choice, a template, a table, a number or a
-  `err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by
+  boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
-  either, with the `severity` RFC 5424 spells beside it.
+  root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
-- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+  or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
-  `locality`, `postal-code` and `street`, weighted by population and address counts
+  pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
-  and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
+  digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
-  `address` record over one consistent draw of them. `sv_SE.address` and
+  `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
-  `en_US.address` read those records, so `en_US.address.street` no longer carries
+  transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
-  `name` and `suffix`, and a locale folder loads only beside `geo`.
+  grammar, could mean two things or cannot render a valid value.
-- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or
+- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
-  within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`.
+  a template seen as columns. A column is a string unless it declares a `datatype`,
-  `sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so
+  `integer`, `number` or `boolean`, and a `null` item draws it as null.
-  `date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`,
+- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
-  `time.sec` and `time.ampm` are no longer paths — a part of a date is now its own
+  category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
-  `{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant.
+  `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
-- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by
+  linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
-  bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a
+  `{/misc.timezone[Europe/Oslo]..territory.name}`.
-  name both sexes carry is a row under each. `person` reads them, so its columns are
+- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
-  `first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are
+  a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
-  no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead.
+  manifest at a source's root may carry a `namespace`, the dot path the source mounts
-  `en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms`
+  under, and an `index`, with which each category loads on the first call reaching it.
-  without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the
+- The shipped data, in Go packages that each export `FS` and `Modules()`:
-  same shape.
+  `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
-- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series
+  `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
-  from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose
+  `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
-  `ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the
+  [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
-  ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so
+  names each source.
-  `--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new
+  - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
-  and carries the same three. `person.prefix` is null where a person has no title,
+    `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
-  where it used to be an empty string, so `--format sql` writes `NULL` and a `string`
+    the Census Bureau. A person's first name and sex agree, and so does an `en_US`
-  struct field reading it becomes `*string`.
+    person's title; `person.prefix` is null where a person has no title.
-- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can
+    `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
-  answer from the rest.
+    `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
-- An error names a spelling that runs: a layout is named single-quoted and free of
+    columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
-  its own quotes, a row of a table with no key is named as the path that selects it,
+    assign.
-  `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record
+  - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
-  that gives it one.
+    HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
+    symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
+    zones weighted by the people living in each, user agents and cars, all as tables;
+    and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
+    ObjectIds and UUIDs.
+  - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+    `locality`, `postal-code` and `street`, weighted by population and address counts
+    and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
+    and an `address` record that draws a street and a postal code inside one
+    locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
+    (postal town) the import can place ships with all its streets. A US place ships
+    only if it is incorporated and holds most of some postal code's land. Of those
+    places, every state capital but Honolulu ships, as does every place of 100,000
+    people or more and a tenth of all of them drawn by population. Each US place
+    carries a quarter of its streets, at least ten.

DATA-LICENSES.md

@@ -9,12 +9,12 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
 | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
 | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
 | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
-| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
+| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
 | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
 | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
 | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
 | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
-| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 12) | — | — | — |
+| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 14) | — | — | — |
 | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
 | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
 | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |

Dockerfile

@@ -1,19 +1,33 @@
-# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
+ARG GO_VERSION=1.27.2
-# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
+FROM golang:${GO_VERSION} AS portable
-# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
-# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
-ARG GO_VERSION=1.26.4
-FROM golang:${GO_VERSION}
 
 WORKDIR /app
 
-# Module layer cached separately from source.
 COPY go.mod ./
 RUN go mod download
 
 COPY . .
 RUN go vet ./... && \
 	go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
-	{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
-	  { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
 	go test -race ./...
+
+FROM golang:${GO_VERSION} AS vulnerabilities
+WORKDIR /app
+COPY . .
+RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
+
+FROM python:3.14.8-slim AS release-tooling
+COPY release-tooling/ /release-tooling/
+RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
+
+FROM goreleaser/goreleaser:v2.18.3 AS packaging
+WORKDIR /check
+COPY release-tooling/goreleaser.yaml ./
+RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
+
+FROM portable
+# BuildKit skips a stage nothing copies from.
+COPY --from=release-tooling /tested /tmp/release-tooling-tested
+COPY --from=packaging /tested /tmp/packaging-tested
+RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
+	{ echo "unformatted files:"; echo "$unformatted"; exit 1; }

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2026 Timewave
+Copyright (c) 2026 Larvit
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

arm.go

@@ -0,0 +1,117 @@
+package fejkdata
+
+import (
+	"fmt"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// arm is one alternative of a {a|b} token or one operand, split into the head,
+// whose node `template.startOf` finds, and the tail of a dotted path into it.
+type arm struct {
+	spelling string // as written, for messages
+	head     string
+	tail     []string
+	levels   []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
+	steps    []pathStep
+	leaves   []node // every node the path may land on, one per variant it passes
+	kind     armKind
+	named    *nameBinding // namedRead: the binding of the name it reads through
+}
+
+// armKind is how expand reads an arm, fixed at compile.
+type armKind uint8
+
+const (
+	freshRead armKind = iota // drawn afresh at every read
+	namedRead                // read through a name, kept in its pick
+)
+
+// splitArm splits one name into head and tail. refs maps a reference to what
+// resolveRefs resolved it to; before that, a reference is whole.
+func splitArm(name string, refs map[string]resolvedRef) arm {
+	if grammar.IsRef(name) {
+		if r, resolved := refs[name]; resolved {
+			return pathArm(name, r.head, r.tail)
+		}
+		return pathArm(name, name, nil)
+	}
+	segs, err := grammar.SplitPath(name)
+	if err != nil || len(segs) == 1 {
+		return pathArm(name, name, nil)
+	}
+	return pathArm(name, segs[0], segs[1:])
+}
+
+func pathArm(name, head string, segs []string) arm {
+	return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
+}
+
+// key is the key of the leaf a lands on, the one key every way of writing this read shares.
+func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
+
+func checkSegments(a arm) error {
+	if len(a.tail) == 0 {
+		return nil
+	}
+	return grammar.CheckSegments(append([]string{a.head}, a.tail...))
+}
+
+// op is one compiled unit of a format string: a literal run, a name read,
+// or a builtin already prepared with its args.
+type op struct {
+	grammar.Token
+	arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
+	call builtinfunc.Call
+	// operands are the fields the builtin reads, in the order its operands func
+	// fixed; expand reads them before the call. nil for a builtin that reads none.
+	operands []arm
+}
+
+func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+	arms := make([]arm, len(names))
+	for i, name := range names {
+		a, err := t.compileArm(name, targets)
+		if err != nil {
+			return nil, err
+		}
+		arms[i] = a
+	}
+	return arms, nil
+}
+
+// compileArm compiles one read into a path: from the head it names, or, for a read through a
+// name, from what that name binds.
+func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
+	a := splitArm(name, t.refs.byName)
+	if !t.isName(a.head) {
+		start := t.startOf(a.head)
+		if start == nil {
+			panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
+		}
+		w := compilePath(start, a.tail)
+		a.steps, a.leaves = w.steps, w.leaves
+		return a, nil
+	}
+	b := t.nameScope.lookup(a.head)
+	if grammar.HasSelector(a.tail) {
+		if b.bindsField() {
+			return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
+		}
+		return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
+	}
+	target := targets[b]
+	full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
+	if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
+		return a, err
+	}
+	w := compilePath(target.start, full)
+	a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
+	a.levels = levelKeys(full, 0)
+	return a, nil
+}

bench_test.go

@@ -6,17 +6,17 @@ import (
 	"testing"
 )
 
-func benchGenerator(b *testing.B, dir string) *Generator {
+func benchGenerator(b *testing.B, data Option) *Generator {
 	b.Helper()
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
+	f, err := New(data, WithSeed(1))
 	if err != nil {
 		b.Fatal(err)
 	}
 	return f
 }
 
-func benchPath(b *testing.B, dir, path string) {
+func benchPath(b *testing.B, data Option, path string) {
-	f := benchGenerator(b, dir)
+	f := benchGenerator(b, data)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
 	}
 }
 
-func BenchmarkPerson(b *testing.B)       { benchPath(b, "data", "sv_SE.person") }
+func BenchmarkPerson(b *testing.B)       { benchPath(b, withShipped(), "sv_SE.person") }
-func BenchmarkAddress(b *testing.B)      { benchPath(b, "data", "sv_SE.address") }
+func BenchmarkAddress(b *testing.B)      { benchPath(b, withShipped(), "sv_SE.address") }
-func BenchmarkWord(b *testing.B)         { benchPath(b, "data", "sv_SE.word") }
+func BenchmarkWord(b *testing.B)         { benchPath(b, withShipped(), "sv_SE.word") }
-func BenchmarkCreditcard(b *testing.B)   { benchPath(b, "data", "misc.creditcard") }
+func BenchmarkCreditcard(b *testing.B)   { benchPath(b, withShipped(), "misc.creditcard") }
-func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") }
+func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
-func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, "data", "misc.uuid") }
+func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, withShipped(), "misc.uuid") }
 
 func tmpData(b *testing.B, name, body string) string {
 	b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
 
 func BenchmarkCalc(b *testing.B) {
 	dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
-	benchPath(b, dir, "inv")
+	benchPath(b, WithDataPath(dir), "inv")
 }
 
 func BenchmarkLongLiteral(b *testing.B) {
 	dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "sql")
+	benchPath(b, WithDataPath(dir), "sql")
 }
 
 func BenchmarkRepeat(b *testing.B) {
 	dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "many")
+	benchPath(b, WithDataPath(dir), "many")
 }
 
-// BenchmarkBound measures a format that binds a head and reads two paths from it —
+// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
-// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
+// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
-// two independent fields.
+// independent fields.
-func BenchmarkBound(b *testing.B) {
+func BenchmarkNamed(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
+	dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-func BenchmarkUnbound(b *testing.B) {
+func BenchmarkUnnamed(b *testing.B) {
 	dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
+// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
-// depth question is about.
+func BenchmarkNamedDeep(b *testing.B) {
-func BenchmarkBoundDeep(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
-	dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
+	benchPath(b, WithDataPath(dir), "addr")
-	benchPath(b, dir, "addr")
 }
 
-// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
+// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
-// expansion holds outgrow a single bucket.
+// outgrow a single bucket.
-func BenchmarkBoundWide(b *testing.B) {
+func BenchmarkNamedWide(b *testing.B) {
-	dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
+	dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
 		{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
 		{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
-	benchPath(b, dir, "row")
+	benchPath(b, WithDataPath(dir), "row")
+}
+
+func BenchmarkFirstFake(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		f, err := New(withShipped(), WithSeed(1))
+		if err != nil {
+			b.Fatal(err)
+		}
+		if _, err := f.Fake("sv_SE.address"); err != nil {
+			b.Fatal(err)
+		}
+	}
 }
 
 func BenchmarkNew(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if _, err := New(WithSeed(1)); err != nil {
+		if _, err := New(withShipped(), WithSeed(1)); err != nil {
 			b.Fatal(err)
 		}
 	}

builtins_test.go

@@ -2,11 +2,14 @@ package fejkdata
 
 import (
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 func TestBuiltinIDGenerators(t *testing.T) {
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
 	}
 }
 
-// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
+// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
 // each call, named counters independent, and the whole thing scoped to one
-// session so a fresh Generator restarts at 1.
+// draw state so a fresh Generator restarts at 1.
-func TestBuiltinSeqPerSession(t *testing.T) {
+func TestBuiltinSeqPerGenerator(t *testing.T) {
 	f := engine(1)
 	for i := 1; i <= 5; i++ {
 		if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
 	if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
 		t.Fatalf("repeat seq = %q, want 1,2,3", got)
 	}
-	// ...and a fresh session restarts from 1.
+	// ...and a fresh generator restarts from 1.
 	if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
-		t.Fatalf("new session seq = %q, want 1", got)
+		t.Fatalf("new generator seq = %q, want 1", got)
 	}
 }
 
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
 		`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
 	}
 	for _, tmpl := range bad {
-		if _, err := compile(parse(t, tmpl)); err == nil {
+		if _, err := resolved(t, tmpl); err == nil {
 			t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
 		}
 	}
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
 	return rem == 1
 }
 
-// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
-// supplies prep, and args parsed at compile only behind a check.
+// separates a reported invariant break from a silently wrong value.
-func TestRegistryShapes(t *testing.T) {
-	for name, b := range builtins {
-		if b.prep == nil {
-			t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
-		}
-		if b.arity != 0 && b.check == nil {
-			t.Errorf("builtin %q parses args in prep with no check", name)
-		}
-	}
-}
-
-// TestArgGuardsPanic pins the guards that report a builtin arg its check should
-// have rejected. No data reaches them — checkFunc runs a builtin's check before
-// compileOps ever calls prep — so they are exercised directly.
-func TestArgGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"atoi on an unvalidated arg": func() { atoi("nope") },
-		"atof on an unvalidated arg": func() { atof("nope") },
-	} {
-		mustPanic(t, name, call)
-	}
-}
-
-// mustPanic fails unless call panics, which is what separates a reported
-// invariant break from a silently wrong value.
 func mustPanic(t *testing.T, name string, call func()) {
 	t.Helper()
 	defer func() {
-		if recover() == nil {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
-			t.Errorf("%s: no panic, want the invariant reported", name)
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
 		}
 	}()
 	call()
 }
 
 func TestClassBuiltinArgs(t *testing.T) {
-	for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
+	for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
-		if _, err := compile(parse(t, bad)); err == nil {
+		if _, err := resolved(t, bad); err == nil {
 			t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
 		}
 	}
 	for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v", ok, err)
 		}
 	}
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
 	}
 }
 
-// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order,
+// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
-// the layout is quoted, names a field, and for time names no date field.
+// layout is quoted.
 func TestBuiltinDateArgs(t *testing.T) {
 	for tmpl, want := range map[string]string{
 		`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`:        "1990-13-01",
+		`"{time('2006-01-02 15:04')}"`:                        "write date(from,to,layout)",
 		`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`:        "is after",
-		`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`:        "write it as text",
 		`"{date(1990-01-01,1990-12-31,2006-01-02)}"`:          "'2006-01-02'",
 		`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`:    "'",
 		`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
-		`"{date(1990-01-01,1990-12-31,'x')}"`:                 "text",
-		`"{date(1990-01-01,1990-12-31,'')}"`:                  "text",
 		`"{date(1990-01-01,1990-12-31)}"`:                     "3 arguments",
 		`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`:     "'January 2, 2006'",
 		`"{time(3:04 PM, Mon)}"`:                              "'3:04 PM, Mon'",
 		`"{time(15:04)}"`:                                     "'15:04'",
-		`"{time('2006-01-02 15:04')}"`:                        "date(",
-		`"{time('x')}"`:                                       "text",
-		`"{date(1990-01-01,1990-12-31,'15:04')}"`:             "time('15:04')",
 	} {
-		_, err := compile(parse(t, tmpl))
+		_, err := resolved(t, tmpl)
 		if err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
 		}
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
 }
 
 // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
-// draws every second of it, and without one it could only emit one value.
+// draws every second of it, and without one it prints that day.
 func TestBuiltinDateSpansOneDay(t *testing.T) {
 	f := engine(1)
 	seen := map[string]bool{}
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
 	if len(seen) < 400 {
 		t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
 	}
-	_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`))
+	if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
-	if err == nil || !strings.Contains(err.Error(), "write it as text") {
+		t.Fatalf("one day in a date-only layout = %q, want that day", got)
-		t.Fatalf("one day in a date-only layout = %v, want it named a constant", err)
+	}
+}
+
+func TestClassBuiltins(t *testing.T) {
+	cases := map[string]*regexp.Regexp{
+		`"{digits(1)}"`:          regexp.MustCompile(`^[0-9]$`),
+		`"{digits(3)}"`:          regexp.MustCompile(`^[0-9]{3}$`),
+		`"{int(1,9)}"`:           regexp.MustCompile(`^[1-9]$`),
+		`"{upper(1)}"`:           regexp.MustCompile(`^[A-Z]$`),
+		`"{lower(1)}"`:           regexp.MustCompile(`^[a-z]$`),
+		`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
+	}
+	f := engine(7)
+	for tmpl, re := range cases {
+		for i := 0; i < 100; i++ {
+			if got := mustRender(t, f, tmpl); !re.MatchString(got) {
+				t.Fatalf("%s produced %q, want %s", tmpl, got, re)
+			}
+		}
+	}
+}
+
+func TestFunctionTokenLuhn(t *testing.T) {
+	// {luhn()} appends a Luhn check digit over the digits emitted so far in the
+	// current expansion (non-digits skipped but kept). It reads the output
+	// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
+	f := engine(1)
+	cases := map[string]string{
+		`"811218987{luhn()}"`:                      "8112189876",  // personnummer body
+		`"7992739871{luhn()}"`:                     "79927398713", // classic Luhn vector
+		`"811218-987{luhn()}"`:                     "811218-9876", // '-' skipped, kept
+		`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876",  // over a rendered token
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Fatalf("%s = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestBuiltinCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{upper(x)}"`,         // count must be an integer
+		`"{int(a,b)}"`,         // non-integer args
+		`"{int(5,1)}"`,         // min > max
+		`"{nanoid(-1)}"`,       // negative count
+		`"{float(1,2,-1)}"`,    // negative decimals
+		`"{float(NaN,NaN,2)}"`, // bounds must be finite
+		`"{float(Inf,Inf,2)}"`, // same-sign infinities
+		`"{float(1,NaN,2)}"`,   // one NaN bound
+		`"{float(-Inf,1,2)}"`,  // one infinite bound
+		`"{iban(US)}"`,         // unsupported country
+		`"{seq(a,b)}"`,         // seq takes at most one name
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestArgErrorsNameTheSpelling(t *testing.T) {
+	if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
+		t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
+	}
+}
+
+func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
+	for long, short := range map[string]string{
+		`"{float(1,2,02)}"`:                 `"{float(1,2,2)}"`,
+		`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
+		`"{digits(+5)}"`:                    `"{digits(5)}"`,
+		`"{digits(05)}"`:                    `"{digits(5)}"`,
+		`"{int(007,9)}"`:                    `"{int(7,9)}"`,
+		`"{int(+1,5)}"`:                     `"{int(1,5)}"`,
+	} {
+		sameRenders(t, long, short)
 	}
 }

calc.go

@@ -2,392 +2,149 @@ package fejkdata
 
 import (
 	"fmt"
-	"math"
+	"slices"
 	"strconv"
 	"strings"
-	"unicode"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// calcNode is a parsed expression node. It evaluates over the operand values expand
+// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
-// read, so the evaluator touches neither the rng nor the node tree.
+// field is a name, which checkCalcNames checks once names resolve.
-type calcNode interface {
+func checkCalcFields(args []string, fields map[string]node) error {
-	eval(operands []string) float64
+	return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
-}
+		if n, ok := fields[name]; ok {
-
+			return []node{n}
-type calcNum float64 // a number literal
-type calcVar string  // a sibling-field name, before indexVars places it
-type calcIdx int     // an operand, by its position in the values expand read
-type calcNeg struct{ x calcNode }
-type calcBin struct { // a + - * / b
-	op   byte
-	l, r calcNode
-}
-
-func (n calcNum) eval([]string) float64 { return float64(n) }
-
-func (n calcIdx) eval(operands []string) float64 {
-	v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
-	if err != nil {
-		return math.NaN() // a non-numeric operand stays visible, never an error
-	}
-	return v
-}
-
-// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
-// expression it compiles, so only a calcIdx reaches a render. It panics rather
-// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
-// silently wrong number.
-func (n calcVar) eval([]string) float64 {
-	panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
-}
-
-func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
-
-func (n calcBin) eval(operands []string) float64 {
-	l, r := n.l.eval(operands), n.r.eval(operands)
-	switch n.op {
-	case '+':
-		return l + r
-	case '-':
-		return l - r
-	case '*':
-		return l * r
-	default: // '/'
-		return l / r
-	}
-}
-
-// checkCalc validates a calc token at compile time: a parseable expression whose
-// operands all name existing fields, and an optional non-negative integer dp. It
-// is a builtin check (fields first), so calc dispatches through the registry like
-// every other {name(args)} function.
-func checkCalc(fields map[string]node, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return fmt.Errorf("calc(%q): %w", args[0], err)
-	}
-	for _, name := range calcVars(expr) {
-		operand, ok := fields[name]
-		if !ok {
-			return fmt.Errorf("calc(%q): no field %q", args[0], name)
 		}
-		if text, never := neverNumeric(operand); never {
+		return nil
-			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
+	})
+}
+
+// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
+func checkCalcNames(label string, t *template) error {
+	for _, o := range t.compiled.ops {
+		if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
+			continue
+		}
+		if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
+			return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
 		}
 	}
-	if divisor, zero := constantZeroDivisor(expr, fields); zero {
+	return nil
-		return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
+}
+
+// operandNodes lists every node each operand of o may render, once its template and names resolve.
+func operandNodes(o op) func(name string) []node {
+	return func(name string) []node {
+		a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
+		if len(a.leaves) == 0 {
+			panic(invariant.Broken("calc operand %q was read before it was compiled", name))
+		}
+		return a.leaves
 	}
-	if len(args) == 2 {
+}
-		dp, err := plainInt(args[1])
+
-		if err != nil {
+// checkOperands refuses an operand that is never a number and a division by a constant zero.
-			return fmt.Errorf("calc decimals %w", err)
+// operand lists every node an operand may render, nil while that is unknown.
-		}
+func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
-		if dp < 0 || dp > maxDecimals {
+	for _, name := range c.Operands {
-			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
+		if rendered, never := allNeverNumeric(operand(name)); never {
+			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
 		}
 	}
+	if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
+		return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
+	}
 	return nil
 }
 
 // constantZeroDivisor finds a division whose right side is a constant zero: number
 // literals and fixed operands folded, anything that varies left unknown.
-func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
+func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
 	switch n := n.(type) {
-	case calcNeg:
+	case grammar.CalcNeg:
-		return constantZeroDivisor(n.x, fields)
+		return constantZeroDivisor(n.X, operand)
-	case calcBin:
+	case grammar.CalcBin:
-		if n.op == '/' {
+		if n.Operator == '/' {
-			if v, known := constantValue(n.r, fields); known && v == 0 {
+			if v, known := constantValue(n.R, operand); known && v == 0 {
-				return calcText(n.r), true
+				return grammar.CalcText(n.R), true
 			}
 		}
-		if d, zero := constantZeroDivisor(n.l, fields); zero {
+		if d, zero := constantZeroDivisor(n.L, operand); zero {
 			return d, true
 		}
-		return constantZeroDivisor(n.r, fields)
+		return constantZeroDivisor(n.R, operand)
 	}
 	return "", false
 }
 
 // constantValue evaluates an expression whose every operand is fixed.
-func constantValue(n calcNode, fields map[string]node) (float64, bool) {
+func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
 	switch n := n.(type) {
-	case calcNum:
+	case grammar.CalcNum:
 		return float64(n), true
-	case calcVar:
+	case grammar.CalcVar:
-		t, ok := fields[string(n)].(*template)
+		nodes := operand(n.Name)
-		if !ok || !t.fixed || t.repeat > 1 {
+		if len(nodes) != 1 {
 			return 0, false
 		}
-		v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
+		t, ok := nodes[0].(*template)
+		if !ok || t.repeat != 1 {
+			return 0, false
+		}
+		lit, fixed := t.fixedText()
+		if !fixed {
+			return 0, false
+		}
+		v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
 		return v, err == nil
-	case calcNeg:
+	case grammar.CalcNeg:
-		v, ok := constantValue(n.x, fields)
+		v, ok := constantValue(n.X, operand)
 		return -v, ok
-	case calcBin:
+	case grammar.CalcBin:
-		l, lok := constantValue(n.l, fields)
+		l, lok := constantValue(n.L, operand)
-		r, rok := constantValue(n.r, fields)
+		r, rok := constantValue(n.R, operand)
 		if !lok || !rok {
 			return 0, false
 		}
-		return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
+		return builtinfunc.Arith(n.Operator, l, r), true
 	}
 	return 0, false
 }
 
-// calcText spells an expression node the way an author would read it.
+// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
-func calcText(n calcNode) string {
+// text that does not parse, or a choice of only such items. text is one such render.
-	switch n := n.(type) {
-	case calcNum:
-		return strconv.FormatFloat(float64(n), 'f', -1, 64)
-	case calcVar:
-		return string(n)
-	case calcNeg:
-		return "-" + calcText(n.x)
-	case calcBin:
-		return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
-	}
-	return "?"
-}
-
-// neverNumeric reports a node no render of which is a number: a null, fixed text that
-// does not parse, or a choice of only such items. text is one such render.
 func neverNumeric(n node) (text string, never bool) {
 	switch n := n.(type) {
-	case *null:
+	case *nullItem:
 		return "", true
 	case *template:
-		if !n.fixed || n.repeat > 1 {
+		lit, fixed := n.fixedText()
+		if n.repeat == 0 {
+			return "", true
+		}
+		if !fixed || n.repeat != 1 {
 			return "", false
 		}
-		if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
+		if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
-			return n.lit, true
+			return lit, true
 		}
 	case *choice:
-		for _, it := range n.items {
+		return allNeverNumeric(n.items)
-			t, itemNever := neverNumeric(it)
-			if !itemNever {
-				return "", false
-			}
-			text = t
-		}
-		return text, true
 	}
 	return "", false
 }
 
-// calcPrep parses the expression and decimals once, at compile time, and places each
+// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
-// operand name at the position expand will read it into. checkCalc proved both args
+func allNeverNumeric(nodes []node) (text string, never bool) {
-// valid, so no step here can fail; dp -1 prints the minimal form.
+	for _, n := range nodes {
-func calcPrep(args []string) callFn {
+		t, nodeNever := neverNumeric(n)
-	expr, err := parseCalc(args[0])
+		if !nodeNever {
-	if err != nil { // a nil AST would be a nil dereference per render, with no message
+			return "", false
-		panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
-	}
-	at := make(map[string]int)
-	for i, name := range calcVars(expr) {
-		at[name] = i
-	}
-	placed := indexVars(expr, at)
-	dp := calcDecimals(args)
-	return func(_ *session, _ string, operands []string) string {
-		return formatFloat(placed.eval(operands), dp)
-	}
-}
-
-// calcDecimals is a calc's decimals count, or -1 for the shortest form.
-func calcDecimals(args []string) int {
-	if len(args) == 2 {
-		return atoi(args[1])
-	}
-	return -1
-}
-
-// indexVars replaces each operand name with its position in the values expand reads.
-// Both sides take that order from calcVars, so they cannot drift.
-func indexVars(n calcNode, at map[string]int) calcNode {
-	switch n := n.(type) {
-	case calcVar:
-		i, placed := at[string(n)]
-		if !placed { // calcVars named every operand, so a miss means the two disagree
-			panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
 		}
-		return calcIdx(i)
+		text = t
-	case calcNeg:
-		return calcNeg{indexVars(n.x, at)}
-	case calcBin:
-		return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
 	}
-	return n
+	return text, len(nodes) > 0
-}
-
-// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
-// an expression that does not parse, so one that does not simply names nothing.
-func calcOperands(args []string) []string {
-	if len(args) == 0 {
-		return nil
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return nil
-	}
-	return calcVars(expr)
-}
-
-// calcVars lists the distinct field names an expression reads, in the order it first
-// names each. That order is the contract between expand, which reads the operands
-// into a slice, and indexVars, which places each name at its position in it.
-func calcVars(n calcNode) []string {
-	var out []string
-	seen := map[string]bool{}
-	var walk func(calcNode)
-	walk = func(n calcNode) {
-		switch n := n.(type) {
-		case calcVar:
-			if name := string(n); !seen[name] {
-				seen[name] = true
-				out = append(out, name)
-			}
-		case calcNeg:
-			walk(n.x)
-		case calcBin:
-			walk(n.l)
-			walk(n.r)
-		}
-	}
-	walk(n)
-	return out
-}
-
-// calcParser is a recursive-descent parser over the expression runes, threading
-// expr -> term -> factor for the standard * / before + - precedence.
-type calcParser struct {
-	rs  []rune
-	pos int
-}
-
-// parseCalc parses a whole expression, requiring it to consume all input.
-func parseCalc(expr string) (calcNode, error) {
-	p := &calcParser{rs: []rune(expr)}
-	if p.space(); p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("empty expression")
-	}
-	n, err := p.expr()
-	if err != nil {
-		return nil, err
-	}
-	if p.space(); p.pos != len(p.rs) {
-		return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
-	}
-	return n, nil
-}
-
-func (p *calcParser) space() {
-	for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
-		p.pos++
-	}
-}
-
-func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
-func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
-
-// binary parses a left-associative run of next() operands joined by the given
-// operators, the one shape expr and term share.
-func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
-	n, err := next()
-	if err != nil {
-		return nil, err
-	}
-	for {
-		p.space()
-		if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
-			return n, nil
-		}
-		op := byte(p.rs[p.pos])
-		p.pos++
-		r, err := next()
-		if err != nil {
-			return nil, err
-		}
-		n = calcBin{op, n, r}
-	}
-}
-
-// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
-func (p *calcParser) factor() (calcNode, error) {
-	p.space()
-	if p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("unexpected end of expression")
-	}
-	switch c := p.rs[p.pos]; {
-	case c == '-':
-		p.pos++
-		x, err := p.factor()
-		if err != nil {
-			return nil, err
-		}
-		return calcNeg{x}, nil
-	case c == '(':
-		p.pos++
-		n, err := p.expr()
-		if err != nil {
-			return nil, err
-		}
-		if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
-			return nil, fmt.Errorf("missing ')'")
-		}
-		p.pos++
-		return n, nil
-	case c == '.' || c >= '0' && c <= '9':
-		return p.number()
-	case c == '_' || unicode.IsLetter(c):
-		return p.ident()
-	default:
-		return nil, fmt.Errorf("unexpected %q", string(c))
-	}
-}
-
-func (p *calcParser) number() (calcNode, error) {
-	start, dot := p.pos, false
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c >= '0' && c <= '9' {
-			p.pos++
-		} else if c == '.' && !dot {
-			dot, p.pos = true, p.pos+1
-		} else {
-			break
-		}
-	}
-	v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
-	if err != nil {
-		return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
-	}
-	return calcNum(v), nil
-}
-
-// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
-// is always the minus operator, so a hyphenated field name can't be an operand.
-func (p *calcParser) ident() (calcNode, error) {
-	start := p.pos
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
-			p.pos++
-		} else {
-			break
-		}
-	}
-	return calcVar(string(p.rs[start:p.pos])), nil
-}
-
-func contains(bs []byte, b byte) bool {
-	for _, x := range bs {
-		if x == b {
-			return true
-		}
-	}
-	return false
 }

calc_test.go

@@ -3,8 +3,11 @@ package fejkdata
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
 )
 
 // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
 	if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
 		t.Fatalf("calc(a - b) = %q, want 7", got)
 	}
+	if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
+		t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
+	}
 }
 
 // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
 	}
 }
 
-// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
+// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
-// is not a calc, and for one whose expression does not parse: nothing, either way.
+// call that is not a calc, and for one whose expression does not parse: none, either way.
-// checkCalc is what reports a bad expression, so the callers that run after it
+func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
-// never meet one — but they must not have to depend on that order to be safe.
+	for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
-func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
+		if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
-	for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
+			t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
-		if got := tokenOperands(body); got != nil {
-			t.Errorf("tokenOperands(%q) = %v, want none", body, got)
 		}
 	}
-	if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
+	if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
-		t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
+		t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
-	}
-}
-
-// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
-// the expression before prep sees it, and indexVars places every name calcVars
-// read, so a break in either is reported rather than computed around.
-func TestCalcGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
-		"an operand name never placed":              func() { calcVar("n").eval(nil) },
-		"indexVars on a name it did not read":       func() { indexVars(calcVar("n"), map[string]int{}) },
-	} {
-		mustPanic(t, name, call)
 	}
 }
 
 // --- one draw, one value: a calc operand reads the expansion's draw ---
 
-// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
+// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
-// A field the format renders and a calc reads is drawn once per expansion, so the
+// shows: the format and the calc read one name.
-// operand shown is the operand computed — the correlation the dotted-path rule
+func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
-// already gives a level (see bound_test.go), applied to a plain sibling.
-func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(3))
 	for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	}
 }
 
-// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
+// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
-// expansion, not to the calc, so the bare token rendering the same name reads it too.
+func TestACalcOperandDrawsAfresh(t *testing.T) {
-func TestCalcOperandSharesOneDraw(t *testing.T) {
+	dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
-	dir := writeData(t, map[string]string{
-		"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
-	})
 	f := newGenerator(t, dir, WithSeed(5))
 	for i := 0; i < 200; i++ {
-		got := fake(t, f, "same")
+		if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
-		if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
-			t.Fatalf("same = %q, want one value twice", got)
-		}
-	}
-}
-
-// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
-// held, so an ordinary {w} {w} still draws twice.
-func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
-	dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
-	f := newGenerator(t, dir, WithSeed(5))
-	for i := 0; i < 200; i++ {
-		if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
 			return
 		}
 	}
-	t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
+	t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
 }
 
-// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
+// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
-// its own expansion, so it draws again while staying self-consistent.
+// iteration, staying one value within it.
-func TestCalcHoldIsPerExpansion(t *testing.T) {
+func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
+		"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(11))
 	varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
 	}
 }
 
-// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
+// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
-// {v} and its calc agree with each other, not with the outer pair.
+// pair agrees with itself, not with the outer pair.
-func TestCalcHoldIsPerTemplate(t *testing.T) {
+func TestTwoNamesPickApart(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
+		"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
-			"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
+			"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
 	})
 	f := newGenerator(t, dir, WithSeed(13))
 	differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["a","b"]}`:       `"x"`,
 		`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
 	} {
-		_, err := compile(parse(t, src))
+		_, err := resolved(t, src)
 		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
 			t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
 		}
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
 		`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
 	} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
 		}
 	}
@@ -250,14 +223,14 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 
 func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	for src, want := range map[string]string{
-		`"{calc(1/0)}"`:                                  "divides by 0",
+		`"{calc(1/0)}"`:     "divides by 0",
-		`"{calc(2/(1-1))}"`:                              "divides by (1 - 1)",
+		`"{calc(2/(1-1))}"`: "divides by (1 - 1)",
 		`{"format":"{calc(x/y)}","x":"1","y":"0"}`:       "divides by y",
 		`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
 		`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`:   "divides by 0",
 		`{"format":"{calc(a/-0)}","a":"1"}`:              "divides by -0",
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
 		}
 	}
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	}
 	t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
 }
+
+func TestCalcCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{calc(1/0)}"`, // a constant zero divisor
+		`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
+		`"{calc()}"`,        // calc needs an expression
+		`"{calc(1 +)}"`,     // dangling operator
+		`"{calc((1 + 2)}"`,  // unbalanced parenthesis
+		`"{calc(1 2)}"`,     // two operands, no operator
+		`"{calc(price)}"`,   // operand names no field
+		`"{calc(1, 2, 3)}"`, // too many args
+		`"{calc(1, x)}"`,    // decimals arg not an integer
+		`"{calc(1, -1)}"`,   // decimals negative
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestCalcReadsAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3","5","7"]`,
+		"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	for i := 0; i < 50; i++ {
+		var k, double int
+		got := fake(t, f, "card")
+		if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
+			t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
+		}
+	}
+}
+
+func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3"]`,
+		"card": `"{/n as k}{calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	if got := fake(t, f, "card"); got != "4" && got != "6" {
+		t.Fatalf("card = %q, want 4 or 6", got)
+	}
+}
+
+func TestCalcRefusesABadOperand(t *testing.T) {
+	for card, want := range map[string]string{
+		`"{/word as w}{w}{calc(w * 2)}"`:    `operand "w" is never a number: it renders "def"`,
+		`"{/zero as z}{z}{calc(1 / z)}"`:    `divides by z, which is always zero`,
+		`"{calc(nope * 2)}"`:                `no field "nope"`,
+		`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
+	} {
+		dir := writeData(t, map[string]string{
+			"word": `["abc","def"]`,
+			"zero": `"0"`,
+			"card": card,
+		})
+		if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
+		}
+	}
+}
+
+func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
+		"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
+	})))
+	if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
+		t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
+	}
+}
+
+func TestATypedColumnProvesACalcOverAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":     `"{int(1,9)}"`,
+		"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	r, err := f.FakeRecord("order")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
+		t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
+	}
+	if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
+		t.Fatalf("JSON = %s, want double written as a number", r.JSON())
+	}
+	bad := writeData(t, map[string]string{
+		"word":  `["1","x"]`,
+		"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
+	})
+	if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
+		t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
+	}
+}
+
+func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
+	if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
+		t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
+	}
+}

cli/cli.go

@@ -0,0 +1,706 @@
+// Package cli is the fejkdata command: it renders the template on its stdin with fake
+// values from the modules it is given and any directories layered over them. A program
+// passing its own modules to Main is a fejkdata CLI carrying them, for example
+// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
+package cli
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata"
+)
+
+const usage = `Usage: <template> | COMMAND [flags]
+       COMMAND --list | --help | --version
+
+COMMAND reads one template from stdin and renders it:
+
+  echo 'id: {int(1,999)}' | COMMAND
+  echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
+  COMMAND < template.txt
+
+COMMAND <<'EOF'
+born {date(1990-01-01,2010-12-31,'2006-01-02')}
+EOF
+
+A template is a format string, or a JSON object, array, string or null. Its {…} tokens
+reach the data by reference from the root: {/path}, for any path --list prints, shipped
+or layered with --data-path. A reference names a category or a dotted path into one, a
+table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
+links to.
+
+Under --format text, the default, what you send is what renders, and COMMAND adds
+nothing: a format string keeps every byte, so echo's newline ends each render, and
+printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
+object, array or string drops the whitespace around it, and null alone renders nothing;
+42 or true alone is a format string, printed as written. An empty or blank stdin is a
+format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
+a backslash or a quote as written.
+
+With --format json, ndjson, csv or sql the template must be a record — a JSON
+template whose fields are its columns, or a lone reference such as {/path}:
+one reference and nothing else, with one newline after it allowed — and the rows are
+written as one JSON array, one JSON object per line, one CSV row (after a header), or
+one INSERT.
+
+  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
+      --format F         output form: text (default), json, ndjson, csv or sql
+  -h, --help             print this help, then exit
+      --list             list the paths the data offers, each written after {/ in a template, then exit;
+                         --repeat, --separator, --format and --table do nothing here
+      --no-shipped-data  load only the --data-path directories
+  -n, --repeat N         render the template N times, 0..1048576 (default 1); 0 prints nothing,
+                         [] under --format json, or the header under --format csv
+  -s, --seed N           same seed, version and data: identical output (default: a random seed)
+      --separator S      string between repeated renders (default empty); refused with a record
+                         --format, which has its own separators
+      --table T          the INSERT target for --format sql, ignored under any other
+                         (default: a lone reference's last segment, else records)
+      --version          print the version, then exit
+
+A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
+-- ends the flags.
+
+Exit codes: 0 success, 1 runtime error, 2 misuse.
+`
+
+type invocation struct {
+	args         []string
+	dirs         []string
+	format       string
+	help         bool
+	list         bool
+	noShipped    bool
+	repeat       int
+	seed         uint64
+	seeded       bool
+	separator    string
+	separatorSet bool
+	table        string
+	tableSet     bool
+	version      bool
+}
+
+type flagDef struct {
+	long  string
+	short string
+	value bool
+	set   func(*invocation, string) error
+}
+
+var flagDefs = []flagDef{
+	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
+	{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
+	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
+	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
+	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
+	{"repeat", "n", true, func(in *invocation, v string) error {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n > fejkdata.MaxRepeat {
+			return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
+		}
+		in.repeat = n
+		return nil
+	}},
+	{"seed", "s", true, func(in *invocation, v string) error {
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
+		}
+		in.seed, in.seeded = n, true
+		return nil
+	}},
+	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
+	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
+	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
+}
+
+func flagByLong(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].long == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+func flagByShort(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].short == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+// flagArg is one flag as written: its definition and the value attached to it,
+// if any.
+type flagArg struct {
+	def    *flagDef
+	value  string
+	inline bool
+}
+
+// splitFlags reads one argv element as flags: --name or --name=value, or -abc
+// where each letter is a short flag and the first that takes a value takes the
+// rest of the element.
+func splitFlags(arg string) ([]flagArg, error) {
+	if strings.HasPrefix(arg, "--") {
+		name, value, inline := strings.Cut(arg[2:], "=")
+		def := flagByLong(name)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag --%s", name)
+		}
+		return []flagArg{{def, value, inline}}, nil
+	}
+	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
+		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
+	}
+	var flags []flagArg
+	letters := arg[1:]
+	for i, r := range letters {
+		letter := string(r)
+		def := flagByShort(letter)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag -%s", letter)
+		}
+		if !def.value {
+			flags = append(flags, flagArg{def: def})
+			continue
+		}
+		rest := letters[i+len(letter):]
+		if strings.HasPrefix(rest, "=") {
+			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
+				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
+		}
+		return append(flags, flagArg{def, rest, rest != ""}), nil
+	}
+	return flags, nil
+}
+
+func parseArgs(argv []string) (invocation, error) {
+	in := invocation{repeat: 1, format: "text"}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		if arg == "--" {
+			in.args = append(in.args, argv[i+1:]...)
+			return in, nil
+		}
+		if len(arg) < 2 || arg[0] != '-' {
+			in.args = append(in.args, arg)
+			continue
+		}
+		flags, err := splitFlags(arg)
+		if err != nil {
+			return in, err
+		}
+		for _, fl := range flags {
+			if !fl.def.value {
+				if fl.inline {
+					return in, fmt.Errorf("--%s takes no value", fl.def.long)
+				}
+				_ = fl.def.set(&in, "")
+				continue
+			}
+			value := fl.value
+			if !fl.inline {
+				if i++; i >= len(argv) {
+					return in, fmt.Errorf("--%s needs a value", fl.def.long)
+				}
+				value = argv[i]
+			}
+			if err := fl.def.set(&in, value); err != nil {
+				return in, err
+			}
+		}
+	}
+	return in, nil
+}
+
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
+type recordFormat struct {
+	header func(*fejkdata.Record) string
+	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
+	writes string // what it writes between values, which --separator would seem to set
+}
+
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson writes one object per line.
+var recordFormats = map[string]recordFormat{
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
+	"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
+}
+
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
+func (in invocation) writesRecords() bool {
+	return in.format != "text"
+}
+
+// formatNames lists the --format values, text included, for the misuse error.
+func formatNames() string {
+	names := []string{"text"}
+	for name := range recordFormats {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
+}
+
+// checkFlags rejects a flag value or combination that cannot run.
+func (in invocation) checkFlags() error {
+	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
+		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
+	}
+	if in.tableSet && in.table == "" && in.format == "sql" {
+		return errors.New("--table names the INSERT target, so it cannot be empty")
+	}
+	if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
+		return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
+	}
+	return nil
+}
+
+func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
+
+// operandError refuses an argument that is no flag, naming the spelling that reads what
+// it most likely means: dropping it for -, a file for a file's name, else a template
+// piped in.
+func operandError(arg, name string) error {
+	unexpected := "unexpected argument '" + arg + "'"
+	if arg == "-" {
+		return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
+	}
+	if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
+		return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
+	}
+	ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
+	_, argLone := loneReference(arg)
+	inline, err := fejkdata.IsTemplate(arg)
+	_, refLone := loneReference(ref)
+	example := pipeHint(name)
+	switch {
+	case argLone || inline:
+		example = piped(arg, name)
+	case err == nil && refLone:
+		example = piped(ref, name)
+	}
+	return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
+}
+
+// check rejects a flag combination or an operand that cannot run, before any data is
+// read.
+func (in invocation) check(name string) error {
+	if err := in.checkFlags(); err != nil {
+		return err
+	}
+	if len(in.args) > 0 {
+		return operandError(in.args[0], name)
+	}
+	return nil
+}
+
+func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
+	if in.noShipped {
+		modules = nil
+	}
+	opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
+	for _, dir := range in.dirs {
+		opts = append(opts, fejkdata.WithDataPath(dir))
+	}
+	seed := in.seed
+	if !in.seeded {
+		seed = systemSeed
+	}
+	return append(opts, fejkdata.WithSeed(seed))
+}
+
+// write streams the input's renders to w, repeat of them joined by the separator; a write
+// failure surfaces from Flush, bufio keeping the first one.
+func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
+	if in.writesRecords() {
+		return in.writeRecords(f, src, w)
+	}
+	t, err := f.NewTemplate(src.template)
+	if err != nil {
+		return templateError(err)
+	}
+	out := bufio.NewWriter(w)
+	for i := 0; i < in.repeat; i++ {
+		if i > 0 {
+			out.WriteString(in.separator)
+		}
+		out.WriteString(t.Fake())
+	}
+	return out.Flush()
+}
+
+// recordStream builds the record drawer for the input, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
+	record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
+	if src.record == "" {
+		t, err := f.NewRecordTemplate(src.template)
+		if err != nil {
+			return nil, "", templateError(spacedReference(src.template, err))
+		}
+		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
+	}
+	table := in.table
+	if table == "" {
+		table = defaultTable(src.record)
+	}
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
+	record, table, err := in.recordStream(f, src)
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	if in.repeat == 0 {
+		return writeNoRecord(record, format, w)
+	}
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
+		r, err := record()
+		if err != nil {
+			return err
+		}
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
+		}
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
+}
+
+// writeNoRecord writes zero records: the header where the format writes one, an empty document
+// where it frames one, else nothing. It draws one record, which names the header's columns, so a
+// lone reference to nothing fails as it would with records.
+func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
+	r, err := record()
+	switch {
+	case err != nil:
+		return err
+	case format.header != nil:
+		_, err = io.WriteString(w, format.header(r)+"\n")
+	case format.open != "":
+		_, err = io.WriteString(w, format.open+format.close+"\n")
+	}
+	return err
+}
+
+// defaultTable names the INSERT target when --table is absent: the path's last
+// segment, or "records" when stdin is not a lone reference.
+func defaultTable(path string) string {
+	if path == "" {
+		return "records"
+	}
+	segments := strings.Split(withoutSelectors(path), ".")
+	return segments[len(segments)-1]
+}
+
+// withoutSelectors is path with its [selectors] cut out.
+func withoutSelectors(path string) string {
+	var names strings.Builder
+	for depth, i := 0, 0; i < len(path); i++ {
+		switch {
+		case path[i] == '[':
+			depth++
+		case path[i] == ']':
+			depth--
+		case depth == 0:
+			names.WriteByte(path[i])
+		}
+	}
+	return names.String()
+}
+
+// inputError marks a failure that is the input's own fault — nothing piped in, a template
+// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
+// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
+// error (exit 1).
+type inputError struct{ error }
+
+func (e inputError) Unwrap() error { return e.error }
+
+// jsonSpace is the whitespace JSON allows around a value.
+const jsonSpace = " \t\r\n"
+
+// input is what stdin holds: the template, and the path a lone reference names as a
+// record.
+type input struct {
+	template string
+	record   string
+}
+
+// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
+// reference once JSON's surrounding whitespace, or one final newline, is cut.
+func parseInput(raw string) input {
+	trimmed := strings.Trim(raw, jsonSpace)
+	text := raw
+	if json.Valid([]byte(trimmed)) {
+		text = trimmed
+	}
+	text, found := strings.CutSuffix(text, "\n")
+	if found {
+		text = strings.TrimSuffix(text, "\r")
+	}
+	record, _ := loneReference(text)
+	return input{template: raw, record: record}
+}
+
+// loneReference returns the path that text reads when it is one reference from the root
+// and nothing else, bare or as a JSON string.
+func loneReference(text string) (string, bool) {
+	var unquoted string
+	if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
+		text = unquoted
+	}
+	path, opens := strings.CutPrefix(text, "{/")
+	path, closes := strings.CutSuffix(path, "}")
+	if !opens || !closes || !isPath(path) {
+		return "", false
+	}
+	return path, true
+}
+
+// isPath reports whether a reference's body after its / is a path alone: no second /, no
+// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
+// It copies the grammar, since the CLI reads only the library's public API.
+func isPath(body string) bool {
+	if body == "" || strings.ContainsAny(body[:1], "/.") {
+		return false
+	}
+	names := withoutSelectors(body)
+	if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
+		return false
+	}
+	inline, err := fejkdata.IsTemplate(body)
+	return !inline && err == nil
+}
+
+// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
+// step up, and only a step up may end the path.
+func hasEmptySegment(path string) bool {
+	segments := strings.Split(path, ".")
+	stepUp := false
+	for i, seg := range segments[1:] {
+		last := i == len(segments)-2
+		switch {
+		case seg != "":
+			stepUp = false
+		case stepUp != last:
+			return true
+		default:
+			stepUp = !stepUp
+		}
+	}
+	return false
+}
+
+// spacedReference adds to a record refusal that whitespace around a lone reference made
+// the template text.
+func spacedReference(template string, err error) error {
+	if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
+		if path, lone := loneReference(trimmed); lone {
+			return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
+		}
+	}
+	return err
+}
+
+// piped is the command piping template into the command called name; some shells' echo
+// reads a backslash as an escape, so a template holding one goes through printf.
+func piped(template, name string) string {
+	if strings.ContainsRune(template, '\\') {
+		return "printf '%s' " + shellQuoted(template) + " | " + name
+	}
+	return "echo " + shellQuoted(template) + " | " + name
+}
+
+// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
+func shellQuoted(s string) string {
+	if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
+		return s
+	}
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// Main runs the command on the process's arguments and standard streams over modules, each
+// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
+// 2 misuse. --no-shipped-data leaves modules out.
+func Main(modules ...fs.FS) int {
+	var stdin io.Reader = os.Stdin
+	if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+		stdin = nil
+	}
+	return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
+}
+
+// commandName is the name the command runs under: the file arg0 names, less a Windows
+// .exe, or fejkdata where arg0 names none.
+func commandName(arg0 string) string {
+	name := filepath.Base(arg0)
+	if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
+		name = strings.TrimSuffix(name, ext)
+	}
+	if name == "." || name == "" {
+		return "fejkdata"
+	}
+	return name
+}
+
+// run is Main over the given streams, run as the command name, seeded by systemSeed where
+// no --seed is given. A nil stdin is a terminal, which holds no template.
+func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
+	in, err := parseArgs(args)
+	if err != nil {
+		return misuse(stderr, name, err)
+	}
+	if in.help {
+		io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
+		return 0
+	}
+	if in.version {
+		info, _ := debug.ReadBuildInfo()
+		fmt.Fprintln(stdout, versionLine(name, info))
+		return 0
+	}
+	if err := in.check(name); err != nil {
+		return misuse(stderr, name, err)
+	}
+	var src input
+	if !in.list {
+		if src, err = readStdin(stdin, name); err != nil {
+			return fail(stderr, name, err)
+		}
+	}
+	f, err := fejkdata.New(in.options(modules, systemSeed)...)
+	if err != nil {
+		return fail(stderr, name, err)
+	}
+	if in.list {
+		for _, p := range f.List() {
+			fmt.Fprintln(stdout, p)
+		}
+		return 0
+	}
+	if err := in.write(f, src, stdout); err != nil {
+		return fail(stderr, name, err)
+	}
+	return 0
+}
+
+func readStdin(stdin io.Reader, name string) (input, error) {
+	if stdin == nil {
+		return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
+	}
+	return parseInput(string(raw)), nil
+}
+
+// templateError is misuse unless the data the template reads failed to load.
+func templateError(err error) error {
+	if errors.Is(err, fejkdata.ErrLoad) {
+		return err
+	}
+	return inputError{err}
+}
+
+func fail(stderr io.Writer, name string, err error) int {
+	var te inputError
+	if errors.As(err, &te) {
+		return misuse(stderr, name, te.error)
+	}
+	report(stderr, name, err)
+	if errors.Is(err, fejkdata.ErrNoColumns) {
+		fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
+	}
+	return 1
+}
+
+func misuse(stderr io.Writer, name string, err error) int {
+	report(stderr, name, err)
+	fmt.Fprintf(stderr, "try '%s --help'\n", name)
+	return 2
+}
+
+// report writes err under the command name, in place of the library's own on each line.
+func report(stderr io.Writer, name string, err error) {
+	msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
+	fmt.Fprintf(stderr, "%s: %s\n", name, msg)
+}
+
+// fejkdataModule is the path of the module holding the library; the root package's path is
+// the module's.
+var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
+
+// versionLine names the fejkdata version the binary is built with. Where another module's
+// main package carries the CLI, the main module's version comes first, under the command name.
+func versionLine(name string, info *debug.BuildInfo) string {
+	if info == nil {
+		return "fejkdata devel"
+	}
+	if info.Main.Path == fejkdataModule {
+		return "fejkdata " + moduleVersion(info.Main)
+	}
+	for _, m := range info.Deps {
+		if m.Path == fejkdataModule {
+			return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
+		}
+	}
+	return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
+}
+
+// moduleVersion is the version m is built at: the replacement's where a replace directive
+// names one, and "devel" where it carries none, as a local directory does.
+func moduleVersion(m debug.Module) string {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Version == "" || m.Version == "(devel)" {
+		return "devel"
+	}
+	return m.Version
+}

cli/cli_test.go

@@ -0,0 +1,961 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// shipped holds the shipped modules as read from the checkout, each folder under ../data
+// holding a manifest, so these tests link no data package.
+var shipped = func() []fs.FS {
+	var modules []fs.FS
+	err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
+		if err == nil && d.Name() == ".fejkdata.json" {
+			modules = append(modules, os.DirFS(filepath.Dir(path)))
+		}
+		return err
+	})
+	if err != nil || len(modules) == 0 {
+		panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
+	}
+	return modules
+}()
+
+// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
+// layers its categories unprefixed with -d.
+var svSE, enUS, misc string
+
+func TestMain(m *testing.M) {
+	tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
+	if err != nil {
+		panic(err)
+	}
+	svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
+	code := m.Run()
+	os.RemoveAll(tmp)
+	os.Exit(code)
+}
+
+// bareCopy copies the category and rows files of the shipped module in data/<module> into a
+// folder under tmp, and returns that folder.
+func bareCopy(tmp, module string) string {
+	src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
+	if err := os.Mkdir(dst, 0o755); err != nil {
+		panic(err)
+	}
+	for _, pattern := range []string{"*.json", "*.tsv"} {
+		files, err := filepath.Glob(filepath.Join(src, pattern))
+		if err != nil {
+			panic(err)
+		}
+		for _, f := range files {
+			if strings.HasPrefix(filepath.Base(f), ".") {
+				continue
+			}
+			b, err := os.ReadFile(f)
+			if err == nil {
+				err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
+			}
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	return dst
+}
+
+func runOut(stdin string, args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+// runTerminal runs as from a shell with nothing piped in.
+func runTerminal(args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
+	module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
+		t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
+	}
+	out.Reset()
+	if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
+		t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
+	}
+}
+
+func TestRunOutputsValue(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if out == "" || strings.Contains(out, "\n") {
+		t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
+	}
+}
+
+func TestRunDotPath(t *testing.T) {
+	code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person run = %d", code)
+	}
+	code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(last) == "" {
+		t.Fatal("empty person.last output")
+	}
+	if last == full {
+		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
+}
+
+func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
+	var drawn, seeded, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
+		t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
+		t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if drawn.String() != seeded.String() {
+		t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
+	}
+}
+
+func TestRunSeedSpellings(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"--seed=42", "--data-path", svSE},
+		{"-s", "42", "-d", svSE},
+		{"--data-path", svSE, "--seed", "42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+}
+
+func TestRunShortFlagValues(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"-s42", "-d", svSE},
+		{"-s", "42", "-d" + svSE},
+		{"-d", svSE, "-s42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+	_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
+	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
+		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-s=42", "-d", svSE}, "-s42"},
+		{[]string{"-s=42", "-d", svSE}, "--seed=42"},
+		{[]string{"-d=" + svSE}, "--data-path="},
+		{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
+		{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
+	} {
+		code, out, errb := runOut("{/word}", c.args...)
+		if code != 2 || out != "" {
+			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+	code, out, _ := runOut("", "-hd", svSE)
+	if code != 0 || !strings.Contains(out, "Usage") {
+		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
+	}
+}
+
+func TestRunDoubleDashEndsFlags(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
+	if code != 2 || !strings.Contains(errb, "stdin") {
+		t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSingleDashLongIsRejected(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
+		{[]string{"-seed=42", "-d", svSE}, "use --seed"},
+		{[]string{"-data-path", svSE}, "use --data-path"},
+		{[]string{"-list", "-d", svSE}, "use --list"},
+		{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
+		{[]string{"-x", "-d", svSE}, "unknown flag -x"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunFlagValues(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
+		{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
+		{[]string{"--seed", "x", "-d", svSE}, "--seed"},
+		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
+		{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
+		{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunRepeat(t *testing.T) {
+	code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
+	}
+	_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
+	if short != out {
+		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
+	}
+}
+
+func TestRunSeparator(t *testing.T) {
+	code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if n := strings.Count(out, "\n"); n != 0 {
+		t.Errorf("want no newline, got %d: %q", n, out)
+	}
+	if !strings.Contains(out, ",") {
+		t.Errorf("values should be comma-joined: %q", out)
+	}
+}
+
+func TestRunRepeatAdvancesRNG(t *testing.T) {
+	code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	uniq := map[string]bool{}
+	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+		uniq[l] = true
+	}
+	if len(uniq) < 2 {
+		t.Errorf("repeat should vary output, all identical: %q", out)
+	}
+}
+
+func TestRunMisuse(t *testing.T) {
+	for _, c := range []struct {
+		stdin string
+		args  []string
+	}{
+		{"{/person}", []string{"-d", svSE, "word"}},
+		{"", []string{"-d", svSE, "--list", "person"}},
+		{"{/sv_SE.person}", []string{"--no-shipped-data"}},
+	} {
+		code, out, errb := runOut(c.stdin, c.args...)
+		if code != 2 {
+			t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
+		}
+		if !strings.Contains(errb, "try 'fejkdata --help'") {
+			t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
+		}
+		if out != "" {
+			t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
+		}
+	}
+}
+
+func TestRunTakesNoOperand(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "users.tmpl")
+	if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
+		{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
+		{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
+		{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
+		{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
+		{[]string{"-"}, "drop it"},
+		{[]string{file}, "fejkdata < " + file},
+	} {
+		code, out, errb := runOut("", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+type failingReader struct{}
+
+func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
+
+func TestRunUnreadableStdinIsMisuse(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
+		t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
+	}
+}
+
+func TestRunTerminalStdinIsMisuse(t *testing.T) {
+	code, out, errb := runTerminal("--seed", "1")
+	if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
+		t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
+	}
+	for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
+		if code, out, errb := runTerminal(args...); code != 0 || out == "" {
+			t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
+		}
+	}
+}
+
+func TestRunRendersExactlyWhatIsSent(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"hihi":                           "hihi",
+		"hihi\n":                         "hihi\n",
+		"hihi\r\n":                       "hihi\r\n",
+		"hihi\n\n":                       "hihi\n\n",
+		"{{x}}\n":                        "{x}\n",
+		"\"a\"\n":                        "a",
+		" \t\"a\"\r\n\n":                 "a",
+		"{/misc.territory[SE].alpha2}":   "SE",
+		"{/misc.territory[SE].alpha2}\n": "SE\n",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
+		}
+	}
+	if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
+		t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
+	}
+	if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
+		t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
+	}
+}
+
+func TestRunRendersBlankStdin(t *testing.T) {
+	for _, stdin := range []string{"", "\n", " \r\n\t"} {
+		if code, out, errb := runOut(stdin); code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunTextPrintsAsWritten(t *testing.T) {
+	for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
+		code, out, errb := runOut(stdin)
+		if code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunMultipleDataPaths(t *testing.T) {
+	code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(out) == "" {
+		t.Fatal("empty output for multi-dir run")
+	}
+}
+
+func TestRunUnknownCategoryFails(t *testing.T) {
+	code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if !strings.Contains(errb, "nope") {
+		t.Errorf("stderr %q should name the unknown category", errb)
+	}
+}
+
+func TestRunList(t *testing.T) {
+	code, out, errb := runOut("", "--data-path", svSE, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	for _, want := range []string{"person", "person.last", "address"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("list output missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRunHelpOnStdout(t *testing.T) {
+	for _, h := range []string{"-h", "--help"} {
+		code, out, errb := runOut("", h)
+		if code != 0 {
+			t.Errorf("%s = %d, want 0", h, code)
+		}
+		if !strings.Contains(out, "Usage") || errb != "" {
+			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
+		}
+	}
+}
+
+func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
+	fejkdata := func(version string, replace *debug.Module) *debug.Module {
+		return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
+	}
+	mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
+	for _, c := range []struct {
+		info *debug.BuildInfo
+		want string
+	}{
+		{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
+		{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
+		{nil, "fejkdata devel"},
+	} {
+		if got := versionLine("mycli", c.info); got != c.want {
+			t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
+		}
+	}
+}
+
+func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
+	for arg0, want := range map[string]string{
+		"/usr/local/bin/mycli": "mycli",
+		"mycli.EXE":            "mycli",
+		"mycli.exe":            "mycli",
+		"my.cli":               "my.cli",
+		"":                     "fejkdata",
+	} {
+		if got := commandName(arg0); got != want {
+			t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
+		}
+	}
+}
+
+func TestRunNamesItselfByTheCommandName(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
+		t.Errorf("mycli --nope = %d, %q", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
+		t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
+	}
+	if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
+		t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	code, out, errb := runOut("", "--version")
+	if code != 0 {
+		t.Fatalf("--version = %d, stderr=%q", code, errb)
+	}
+	version, ok := strings.CutPrefix(out, "fejkdata ")
+	if !ok || strings.TrimSpace(version) == "" {
+		t.Errorf("--version = %q, want the command name and a version on stdout", out)
+	}
+}
+
+func TestRunMissingDirFails(t *testing.T) {
+	code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if errb == "" {
+		t.Error("want an error message on stderr")
+	}
+}
+
+func TestRunShippedDataByDefault(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, list, _ := runOut("", "--list")
+	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
+		t.Errorf("--list without --data-path = %d, %q", code, list)
+	}
+	code, _, errb = runOut("{/person}")
+	if code != 2 || !strings.Contains(errb, "person") {
+		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSelectsATableRow(t *testing.T) {
+	dir := t.TempDir()
+	for name, content := range map[string]string{
+		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
+		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
+		t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
+	}
+	if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
+		t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
+	}
+	if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
+		t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
+	}
+	if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
+		t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
+	}
+	if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
+		t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
+	}
+}
+
+func TestUsageExamplesRenderWithNoData(t *testing.T) {
+	examples := regexp.MustCompile(`(?m)^  echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n  "+pipeHint("COMMAND")+"\n", -1)
+	if len(examples) < 3 {
+		t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
+	}
+	for _, e := range examples {
+		if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
+			t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
+		}
+	}
+	for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
+		if token != "{/path}" && token != "{/table[KEY]}" {
+			t.Errorf("usage names %s, a path a custom CLI may not carry", token)
+		}
+	}
+	if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
+		t.Errorf("usage does not list the exit codes")
+	}
+}
+
+func TestRunOperandMisuseBeforeLoad(t *testing.T) {
+	code, _, errb := runOut("", "--no-shipped-data", "x")
+	if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
+		t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
+	}
+}
+
+func TestRunInlineTemplate(t *testing.T) {
+	code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
+	if code != 0 || (out != "name: bosse" && out != "name: lina") {
+		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
+	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
+		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
+	if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
+		t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
+	}
+}
+
+func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
+	for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
+		if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
+			t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
+		}
+	}
+	if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
+		t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
+	}
+}
+
+func TestRunTemplateMisuse(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"{bad":                  "unterminated",
+		"name: {/no.such.path}": "no entry",
+		"a } b":                 "}}",
+		"{//sv_SE.person}":      "write {/sv_SE.person}",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
+			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
+		}
+		if strings.Contains(errb, "fejkdata: fejkdata:") {
+			t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
+		}
+	}
+}
+
+func TestRunNoShippedData(t *testing.T) {
+	code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
+		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
+	}
+	code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Errorf("run = %d, out=%q", code, out)
+	}
+	if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
+		t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
+	}
+	if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
+		t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
+	}
+}
+
+func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{nil, ""},
+		{[]string{"--format", "json"}, "[]\n"},
+		{[]string{"--format", "ndjson"}, ""},
+		{[]string{"--format", "csv"}, "first,last\n"},
+		{[]string{"--format", "sql"}, ""},
+	} {
+		if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
+			t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
+		}
+	}
+	if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
+		t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
+	}
+}
+
+func TestRunRepeatIsBounded(t *testing.T) {
+	for _, args := range [][]string{
+		{"--repeat", "9223372036854775807"},
+		{"--repeat", "1048577"},
+		{"-n", "99999999999"},
+	} {
+		code, out, errb := runOut("{/sv_SE.word}", args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
+			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
+		}
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
+	if code != 0 || len(out) != 1048576 {
+		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
+	}
+}
+
+func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		stdin         string
+		base, ignored []string
+	}{
+		{"", []string{"--list"}, []string{"-n", "3"}},
+		{"", []string{"--list"}, []string{"--separator", ","}},
+		{"", []string{"--list"}, []string{"--format", "json"}},
+		{"", []string{"--list"}, []string{"--table", "t"}},
+		{"{/users}", nil, []string{"--table", "t"}},
+		{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
+	} {
+		args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
+		_, want, _ := runOut(c.stdin, args...)
+		code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
+		if code != 0 || want == "" || got != want {
+			t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
+		}
+	}
+}
+
+func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
+	if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
+		t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
+	}
+}
+
+func TestRunRendersAnyJSONValue(t *testing.T) {
+	for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
+		if code, out, errb := runOut(stdin); code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
+		}
+	}
+}
+
+func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "-ä")
+	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
+		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
+	}
+}
+
+func TestRunEmptyDataPathIsNamed(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
+	if code != 1 || !strings.Contains(errb, "empty") {
+		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
+	}
+}
+
+func recordDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
+	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestRunRecordJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
+		}
+	}
+}
+
+func TestRunRecordCSVRoundTrips(t *testing.T) {
+	code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 4 {
+		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
+	}
+	if lines[0] != "first,last" {
+		t.Fatalf("csv header = %q, want first,last", lines[0])
+	}
+	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
+	if err != nil {
+		t.Fatalf("csv output is not valid CSV: %v", err)
+	}
+	if len(r) != 4 || len(r[0]) != 2 {
+		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
+	}
+}
+
+func TestRunRecordSQL(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
+		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
+	}
+}
+
+func TestRunRecordDefaultTable(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
+	}
+}
+
+func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
+	code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
+	}
+	code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
+	if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
+		t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
+	}
+}
+
+func TestRunRecordInlineTemplate(t *testing.T) {
+	code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
+	if code != 0 {
+		t.Fatalf("inline record = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
+	}
+}
+
+func TestRunRecordMisuse(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
+		{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
+		{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
+		{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
+	} {
+		code, out, errb := runOut("{/users}", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
+	if code != 1 {
+		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
+	}
+}
+
+func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
+	dir := t.TempDir()
+	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
+	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
+	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
+	}
+	code, _, errb = runOut(content, "--format", "json")
+	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
+	}
+	if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
+		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
+	}
+}
+
+func TestRunNumbersFollowTheShell(t *testing.T) {
+	_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
+	code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
+	if code != 0 || got != want {
+		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	}
+}
+
+// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
+// for columns: the record to write, and the quoting a shell needs to take it. The
+// template is well-formed and only the data cannot serve it, so the code is 1.
+func TestRunRecordOfAFieldlessPath(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
+	}
+	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
+		if !strings.Contains(errb, want) {
+			t.Errorf("stderr = %q, want it to name %s", errb, want)
+		}
+	}
+	if out != "" {
+		t.Errorf("stdout = %q, want nothing", out)
+	}
+}
+
+func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
+	dir := t.TempDir()
+	for name, data := range map[string]string{
+		".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
+		"bad.json":       `"{/nope}"`,
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
+		if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
+			t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
+		}
+	}
+}

cli/lone_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// grammarLoneReference is the path a format reads when the grammar parses it as one path
+// read of one reference from the root, and nothing else.
+func grammarLoneReference(text string) (string, bool) {
+	toks, err := grammar.ParseFormat(text)
+	if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
+		return "", false
+	}
+	sigil, path, err := grammar.RefShape(toks[0].Arms[0])
+	if err != nil || sigil != "/" {
+		return "", false
+	}
+	return path, true
+}
+
+func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
+	for _, text := range []string{
+		" {/x}",
+		"x",
+		"{.x}",
+		"{/.x}",
+		"{//x}",
+		"{/a...b}",
+		"{/a..b}",
+		"{/a..}",
+		"{/a.[x]}",
+		"{/a.}",
+		"{/a|/b}",
+		"{/a}{/b}",
+		"{/misc.currency[US Dollar (Next day)]}",
+		"{/sv_SE.person}",
+		"{/t[a as b].c}",
+		"{/t[f(x)]}",
+		"{/x as p}",
+		"{/x(1)}",
+		"{/x}}",
+		"{/}",
+	} {
+		path, lone := loneReference(text)
+		want, wantLone := grammarLoneReference(text)
+		if lone != wantLone || path != want {
+			t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
+		}
+	}
+}

cmd/fejkdata/main.go

@@ -1,512 +1,12 @@
-// Command fejkdata prints fake values from the shipped data and any directories
+// Command fejkdata renders the template on its stdin with fake values from every shipped
-// layered over it.
+// module; package cli documents it.
-//
-//	fejkdata sv_SE.person                        # a full person
-//	fejkdata sv_SE.person.last                   # just the surname
-//	fejkdata --data-path ./mydata sv_SE.person   # layer custom data; the last dir wins
-//	fejkdata --seed 42 sv_SE.address
 package main
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"runtime/debug"
-	"sort"
-	"strconv"
-	"strings"
 
-	"gitea.larvit.se/larvit/fejkdata"
+	"github.com/larvit/fejkdata/cli"
+	"github.com/larvit/fejkdata/data"
 )
 
-const usage = `Usage: fejkdata [flags] <path|template>
+func main() { os.Exit(cli.Main(data.Modules()...)) }
-
-  <path>                 a category, or a dotted path into one (person, person.last);
-                         a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'
-  <template>             a format string or JSON value to render inline, e.g.
-                         'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
-
-A layout inside a template is single-quoted, so quote the whole argument with " to
-keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
-
-An argument containing a { token, or a JSON object, array or string, is a
-template; any other argument is a path (a path never contains a brace or a quote,
-and a bracket only as a [selector] after a table's name). Templates reach the
-data by reference from the root — {/sv_SE.person.last} — whether the data is
-shipped or layered with --data-path. An argument carrying a closing brace or a
-quote but no valid JSON names neither.
-
-With --format json, ndjson, csv or sql the argument must name a record — a
-template whose fields are its columns — and the rows are written as one JSON
-array, one JSON object per line, one CSV row (after a header), or one INSERT.
-
-  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, ndjson, csv or sql
-  -h, --help             print this help, then exit
-      --list             list the paths the data offers, then exit
-      --no-shipped-data  load only the --data-path directories
-  -n, --repeat N         render the value N times, 1..1048576 (default 1)
-  -s, --seed N           seed for reproducible output
-      --separator S      string between repeated values (default newline)
-      --table T          the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
-      --version          print the version, then exit
-
-Flags may come before or after <path|template>; -- ends the flags. A short flag's
-value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
-`
-
-type invocation struct {
-	dirs         []string
-	format       string
-	formatSet    bool
-	help         bool
-	list         bool
-	noShipped    bool
-	paths        []string
-	repeat       int
-	repeatSet    bool
-	seed         uint64
-	seeded       bool
-	separator    string
-	separatorSet bool
-	table        string
-	tableSet     bool
-	version      bool
-}
-
-type flagDef struct {
-	long  string
-	short string
-	value bool
-	set   func(*invocation, string) error
-}
-
-var flagDefs = []flagDef{
-	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
-	{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
-	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
-	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
-	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
-	{"repeat", "n", true, func(in *invocation, v string) error {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > fejkdata.MaxRepeat {
-			return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
-		}
-		in.repeat, in.repeatSet = n, true
-		return nil
-	}},
-	{"seed", "s", true, func(in *invocation, v string) error {
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
-		}
-		in.seed, in.seeded = n, true
-		return nil
-	}},
-	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
-	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
-	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
-}
-
-func flagByLong(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].long == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-func flagByShort(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].short == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-// flagArg is one flag as written: its definition and the value attached to it,
-// if any.
-type flagArg struct {
-	def    *flagDef
-	value  string
-	inline bool
-}
-
-// splitFlags reads one argv element as flags: --name or --name=value, or -abc
-// where each letter is a short flag and the first that takes a value takes the
-// rest of the element.
-func splitFlags(arg string) ([]flagArg, error) {
-	if strings.HasPrefix(arg, "--") {
-		name, value, inline := strings.Cut(arg[2:], "=")
-		def := flagByLong(name)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag --%s", name)
-		}
-		return []flagArg{{def, value, inline}}, nil
-	}
-	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
-		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
-	}
-	var flags []flagArg
-	letters := arg[1:]
-	for i, r := range letters {
-		letter := string(r)
-		def := flagByShort(letter)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag -%s", letter)
-		}
-		if !def.value {
-			flags = append(flags, flagArg{def: def})
-			continue
-		}
-		rest := letters[i+len(letter):]
-		if strings.HasPrefix(rest, "=") {
-			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
-				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
-		}
-		return append(flags, flagArg{def, rest, rest != ""}), nil
-	}
-	return flags, nil
-}
-
-func parseArgs(argv []string) (invocation, error) {
-	in := invocation{repeat: 1, separator: "\n", format: "text"}
-	for i := 0; i < len(argv); i++ {
-		arg := argv[i]
-		if arg == "--" {
-			in.paths = append(in.paths, argv[i+1:]...)
-			return in, nil
-		}
-		if len(arg) < 2 || arg[0] != '-' {
-			in.paths = append(in.paths, arg)
-			continue
-		}
-		flags, err := splitFlags(arg)
-		if err != nil {
-			return in, err
-		}
-		for _, fl := range flags {
-			if !fl.def.value {
-				if fl.inline {
-					return in, fmt.Errorf("--%s takes no value", fl.def.long)
-				}
-				_ = fl.def.set(&in, "")
-				continue
-			}
-			value := fl.value
-			if !fl.inline {
-				if i++; i >= len(argv) {
-					return in, fmt.Errorf("--%s needs a value", fl.def.long)
-				}
-				value = argv[i]
-			}
-			if err := fl.def.set(&in, value); err != nil {
-				return in, err
-			}
-		}
-	}
-	return in, nil
-}
-
-// recordFormat is one way to write a record out: the line each record renders,
-// the header that precedes the first one, and the open/close frame plus the
-// between-record separator a document form needs.
-type recordFormat struct {
-	header func(*fejkdata.Record) string
-	line   func(r *fejkdata.Record, table string) string
-	open   string
-	close  string
-	sep    string
-}
-
-// recordFormats is every --format that writes records. json frames the records
-// as one array document; ndjson is the same column, one object per line.
-var recordFormats = map[string]recordFormat{
-	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
-	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
-	"ndjson": {line: jsonLine, sep: "\n"},
-	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
-}
-
-func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
-
-// writesRecords reports whether the format writes records rather than plain text.
-func (in invocation) writesRecords() bool {
-	return in.format != "text"
-}
-
-// formatNames lists the --format values, text included, for the misuse error.
-func formatNames() string {
-	names := []string{"text"}
-	for name := range recordFormats {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
-}
-
-// checkFlags rejects a flag value or combination that cannot run.
-func (in invocation) checkFlags() error {
-	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
-		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
-	}
-	if in.tableSet && in.table == "" {
-		return errors.New("--table names the INSERT target, so it cannot be empty")
-	}
-	if in.tableSet && in.format != "sql" {
-		return errors.New("--table names the INSERT target, so it needs --format sql")
-	}
-	if in.writesRecords() && in.separatorSet {
-		return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
-	}
-	return nil
-}
-
-// check rejects a flag combination or an argument that cannot run, and reports
-// what the argument names, so its shape is settled before any data is read.
-func (in invocation) check() (argKind, error) {
-	if err := in.checkFlags(); err != nil {
-		return argPath, err
-	}
-	if in.list && len(in.paths) > 0 {
-		return argPath, errors.New("--list takes no path")
-	}
-	if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
-		return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
-	}
-	if in.list {
-		return argPath, nil
-	}
-	if len(in.paths) != 1 {
-		return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
-	}
-	return classify(in.paths[0])
-}
-
-func (in invocation) options() []fejkdata.Option {
-	var opts []fejkdata.Option
-	if in.noShipped {
-		opts = append(opts, fejkdata.WithoutShippedData())
-	}
-	for _, dir := range in.dirs {
-		opts = append(opts, fejkdata.WithDataPath(dir))
-	}
-	if in.seeded {
-		opts = append(opts, fejkdata.WithSeed(in.seed))
-	}
-	return opts
-}
-
-// write streams the argument's renders to w, repeat of them joined by the
-// separator and ended by a newline. A value that renders once renders every time,
-// so a render failure comes before anything is written; a write failure surfaces
-// from Flush, bufio keeping the first one.
-func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	if in.writesRecords() {
-		return in.writeRecords(f, kind, w)
-	}
-	draw, err := in.textDraw(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	out := bufio.NewWriter(w)
-	for i := 0; i < in.repeat; i++ {
-		v, err := draw()
-		if err != nil {
-			return err
-		}
-		if i > 0 {
-			out.WriteString(in.separator)
-		}
-		out.WriteString(v)
-	}
-	out.WriteString("\n")
-	return out.Flush()
-}
-
-// textDraw builds what one text render yields: the value, from a path or an
-// inline template.
-func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if kind != argTemplate {
-		return func() (string, error) { return f.Fake(arg) }, nil
-	}
-	t, err := f.NewTemplate(arg)
-	if err != nil {
-		return nil, templateError{err}
-	}
-	return func() (string, error) { return t.Fake(), nil }, nil
-}
-
-// recordStream builds the record drawer for the argument, plus the INSERT table
-// a sql format names.
-func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
-	record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
-	if kind == argTemplate {
-		t, err := f.NewRecordTemplate(arg)
-		if err != nil {
-			return nil, "", templateError{err}
-		}
-		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
-	}
-	table := in.table
-	if table == "" {
-		table = defaultTable(arg, kind)
-	}
-	return record, table, nil
-}
-
-// writeRecords streams a record per line in the chosen format, framing a document
-// form with its open/close brackets and a header preceding the first record.
-func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	record, table, err := in.recordStream(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	format := recordFormats[in.format]
-	out := bufio.NewWriter(w)
-	if format.open != "" {
-		out.WriteString(format.open)
-		out.WriteByte('\n')
-	}
-	for i := 0; i < in.repeat; i++ {
-		r, err := record()
-		if err != nil {
-			return err
-		}
-		if i == 0 && format.header != nil {
-			out.WriteString(format.header(r))
-			out.WriteByte('\n')
-		}
-		if i > 0 {
-			out.WriteString(format.sep)
-		}
-		out.WriteString(format.line(r, table))
-	}
-	if format.close != "" {
-		out.WriteByte('\n')
-		out.WriteString(format.close)
-	}
-	out.WriteByte('\n')
-	return out.Flush()
-}
-
-// defaultTable names the INSERT target when --table is absent: the path's last
-// segment, or "records" for an inline template that sits in no folder.
-func defaultTable(arg string, kind argKind) string {
-	if kind == argTemplate {
-		return "records"
-	}
-	var names strings.Builder // the path with its [selectors] cut out
-	for depth, i := 0, 0; i < len(arg); i++ {
-		switch {
-		case arg[i] == '[':
-			depth++
-		case arg[i] == ']':
-			depth--
-		case depth == 0:
-			names.WriteByte(arg[i])
-		}
-	}
-	segments := strings.Split(names.String(), ".")
-	return segments[len(segments)-1]
-}
-
-// templateError marks a render failure that is the argument's own fault — an
-// inline template that does not compile. run reports it as misuse (exit 2, with a
-// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
-type templateError struct{ error }
-
-func (e templateError) Unwrap() error { return e.error }
-
-type argKind int
-
-const (
-	argPath argKind = iota
-	argTemplate
-)
-
-// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
-func classify(arg string) (argKind, error) {
-	inline, err := fejkdata.IsTemplate(arg)
-	if inline {
-		return argTemplate, nil
-	}
-	return argPath, err
-}
-
-func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
-
-// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
-func run(args []string, stdout, stderr io.Writer) int {
-	in, err := parseArgs(args)
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	if in.help {
-		fmt.Fprint(stdout, usage)
-		return 0
-	}
-	if in.version {
-		fmt.Fprintln(stdout, "fejkdata "+buildVersion())
-		return 0
-	}
-	kind, err := in.check()
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	f, err := fejkdata.New(in.options()...)
-	if errors.Is(err, fejkdata.ErrNoData) {
-		return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
-	}
-	if err != nil {
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	if in.list {
-		for _, p := range f.List() {
-			fmt.Fprintln(stdout, p)
-		}
-		return 0
-	}
-	if err := in.write(f, kind, stdout); err != nil {
-		var te templateError
-		if errors.As(err, &te) {
-			return misuse(stderr, te.error)
-		}
-		fmt.Fprintln(stderr, err)
-		if errors.Is(err, fejkdata.ErrNoColumns) {
-			fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
-		}
-		return 1
-	}
-	return 0
-}
-
-// templateSplit names the cure for a template a shell split on its spaces, which is
-// what several arguments carrying a token mean.
-func templateSplit(paths []string) string {
-	for _, p := range paths {
-		if strings.Contains(p, "{") {
-			return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
-		}
-	}
-	return ""
-}
-
-func misuse(stderr io.Writer, err error) int {
-	// A library error already names the program, so the prefix is not doubled.
-	fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
-	return 2
-}
-
-// buildVersion is the module version go install stamps into the binary, or "devel".
-func buildVersion() string {
-	if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
-		return info.Main.Version
-	}
-	return "devel"
-}

cmd/fejkdata/main_test.go

@@ -2,640 +2,46 @@ package main
 
 import (
 	"bytes"
-	"encoding/csv"
-	"encoding/json"
 	"os"
-	"path/filepath"
+	"os/exec"
-	"regexp"
 	"strings"
 	"testing"
 )
 
-const (
+func TestMain(m *testing.M) {
-	svSE = "../../data/sv_SE"
+	if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
-	enUS = "../../data/en_US"
+		main()
-	misc = "../../data/misc"
+	}
-)
+	os.Exit(m.Run())
-
-func runOut(args ...string) (int, string, string) {
-	var out, errb bytes.Buffer
-	code := run(args, &out, &errb)
-	return code, out.String(), errb.String()
 }
 
-func TestRunOutputsValue(t *testing.T) {
+func TestMainRendersTheShippedData(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "person")
+	cmd := exec.Command(os.Args[0], "--seed", "1")
-	if code != 0 {
+	cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-		t.Fatalf("run = %d, stderr=%q", code, errb)
+	cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
-	}
+	var stderr bytes.Buffer
-	if strings.TrimSpace(out) == "" {
+	cmd.Stderr = &stderr
-		t.Fatalf("empty output, stderr=%q", errb)
+	out, err := cmd.Output()
-	}
+	if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
-	if !strings.HasSuffix(out, "\n") {
+		t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
-		t.Errorf("output should end with newline, got %q", out)
 	}
 }
 
-func TestRunDotPath(t *testing.T) {
+func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
-	code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
+	var outs [2]string
-	if code != 0 {
+	for i := range outs {
-		t.Fatalf("person run = %d", code)
+		cmd := exec.Command(os.Args[0])
-	}
+		cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-	code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
+		cmd.Stdin = strings.NewReader("{uuid()}")
-	if code != 0 {
+		var stderr bytes.Buffer
-		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+		cmd.Stderr = &stderr
-	}
+		out, err := cmd.Output()
-	if strings.TrimSpace(last) == "" {
+		if err != nil || len(out) == 0 {
-		t.Fatal("empty person.last output")
+			t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
-	}
-	if last == full {
-		t.Errorf("person.last %q should differ from person %q", last, full)
-	}
-}
-
-func TestRunSeedSpellings(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"--seed=42", "--data-path", svSE, "address"},
-		{"-s", "42", "-d", svSE, "address"},
-		{"--data-path", svSE, "address", "--seed", "42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
 		}
-	}
+		outs[i] = string(out)
-}
+	}
-
+	if outs[0] == outs[1] {
-func TestRunShortFlagValues(t *testing.T) {
+		t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"-s42", "-d", svSE, "address"},
-		{"-s", "42", "-d" + svSE, "address"},
-		{"-d", svSE, "address", "-s42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-	_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
-	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
-		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
-	}
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
-		{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
-		{[]string{"-d=" + svSE, "address"}, "--data-path="},
-		{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
-		{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" {
-			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-	code, out, _ := runOut("-hd", svSE)
-	if code != 0 || !strings.Contains(out, "Usage") {
-		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
-	}
-}
-
-func TestRunDoubleDashEndsFlags(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--", "person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, _, errb = runOut("--data-path", svSE, "--", "--list")
-	if code != 1 || !strings.Contains(errb, "--list") {
-		t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestRunSingleDashLongIsRejected(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-data-path", svSE, "person"}, "use --data-path"},
-		{[]string{"-list", "-d", svSE}, "use --list"},
-		{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
-		{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunFlagValues(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
-		{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
-		{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
-		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
-		{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
-		{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunRepeat(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 3 {
-		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
-	}
-	_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
-	if short != out {
-		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
-	}
-}
-
-func TestRunSeparator(t *testing.T) {
-	code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if n := strings.Count(out, "\n"); n != 1 {
-		t.Errorf("want one trailing newline, got %d: %q", n, out)
-	}
-	if !strings.Contains(out, ",") {
-		t.Errorf("values should be comma-joined: %q", out)
-	}
-}
-
-func TestRunRepeatAdvancesRNG(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	uniq := map[string]bool{}
-	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
-		uniq[l] = true
-	}
-	if len(uniq) < 2 {
-		t.Errorf("repeat should vary output, all identical: %q", out)
-	}
-}
-
-func TestRunMisuse(t *testing.T) {
-	for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
-		code, out, errb := runOut(args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", args, code)
-		}
-		if !strings.Contains(errb, "try 'fejkdata --help'") {
-			t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
-		}
-		if out != "" {
-			t.Errorf("run(%v) stdout = %q, want nothing", args, out)
-		}
-	}
-	// A template a shell split names both quotings, since which one cures it turns
-	// on whether the argument carries a double-quoted JSON.
-	_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
-	for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
-		}
-	}
-}
-
-func TestRunMultipleDataPaths(t *testing.T) {
-	code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatal("empty output for multi-dir run")
-	}
-}
-
-func TestRunUnknownCategoryFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", svSE, "nope")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if !strings.Contains(errb, "nope") {
-		t.Errorf("stderr %q should name the unknown category", errb)
-	}
-}
-
-func TestRunList(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	for _, want := range []string{"person", "person.last", "address"} {
-		if !strings.Contains(out, want) {
-			t.Errorf("list output missing %q:\n%s", want, out)
-		}
-	}
-}
-
-func TestRunHelpOnStdout(t *testing.T) {
-	for _, h := range []string{"-h", "--help"} {
-		code, out, errb := runOut(h)
-		if code != 0 {
-			t.Errorf("%s = %d, want 0", h, code)
-		}
-		if !strings.Contains(out, "Usage") || errb != "" {
-			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
-		}
-	}
-}
-
-func TestRunVersion(t *testing.T) {
-	code, out, errb := runOut("--version")
-	if code != 0 {
-		t.Fatalf("--version = %d, stderr=%q", code, errb)
-	}
-	version, ok := strings.CutPrefix(out, "fejkdata ")
-	if !ok || strings.TrimSpace(version) == "" {
-		t.Errorf("--version = %q, want the command name and a version on stdout", out)
-	}
-}
-
-func TestRunMissingDirFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", "../../data/nope", "person")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if errb == "" {
-		t.Error("want an error message on stderr")
-	}
-}
-
-func TestRunShippedDataByDefault(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "sv_SE.person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, list, _ := runOut("--list")
-	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
-		t.Errorf("--list without --data-path = %d, %q", code, list)
-	}
-	code, _, errb = runOut("person")
-	if code != 1 || !strings.Contains(errb, "person") {
-		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestClassify(t *testing.T) {
-	for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
-		if got, err := classify(arg); err != nil || got != want {
-			t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
-		}
-	}
-	if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
-		t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
-	}
-	if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
-		t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
-	}
-}
-
-func TestRunSelectsATableRow(t *testing.T) {
-	dir := t.TempDir()
-	for name, content := range map[string]string{
-		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
-		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
-	} {
-		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
-		t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
-		t.Fatalf("region[Skåne län].code = %d, %q", code, out)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
-		t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
-		t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
-		t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
-		t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out)
-	}
-}
-
-func TestUsageReferencesResolve(t *testing.T) {
-	for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
-		path := token[2 : len(token)-1]
-		code, out, errb := runOut("--seed", "1", path)
-		if code != 0 || strings.TrimSpace(out) == "" {
-			t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
-		}
-	}
-}
-
-func TestRunShapeMisuseBeforeLoad(t *testing.T) {
-	code, _, errb := runOut("--no-shipped-data", "[abc]")
-	if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
-		t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
-	}
-}
-
-func TestRunInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
-	if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
-		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
-	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
-		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
-	}
-}
-
-func TestRunTemplateMisuse(t *testing.T) {
-	for arg, want := range map[string]string{
-		"{bad":                  "unterminated",
-		"[red,green]":           `starts with "["`,
-		`{"format":"x"}`:        "is a string",
-		"name: {/no.such.path}": "no entry",
-		"{/sv_SE.person}":       "write sv_SE.person",
-		"x[1]y":                 `"]"`,
-		` ["a","b"] `:           "may not be padded",
-	} {
-		code, out, errb := runOut("--seed", "1", arg)
-		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
-			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
-		}
-		if strings.Contains(errb, "fejkdata: fejkdata:") {
-			t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
-		}
-	}
-}
-
-func TestRunNoShippedData(t *testing.T) {
-	code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
-		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
-	}
-	code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Errorf("run = %d, out=%q", code, out)
-	}
-	code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
-	if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
-		t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
-	}
-}
-
-func TestRunRepeatIsBounded(t *testing.T) {
-	for _, args := range [][]string{
-		{"--repeat", "9223372036854775807", "sv_SE.word"},
-		{"--repeat", "1048577", "sv_SE.word"},
-		{"-n", "99999999999", "sv_SE.word"},
-	} {
-		code, out, errb := runOut(args...)
-		if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
-			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
-		}
-	}
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
-	if code != 0 || len(out) != 1048576+1 {
-		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
-	}
-}
-
-func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
-	for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
-		code, _, errb := runOut(args...)
-		if code != 2 || !strings.Contains(errb, "--list takes no") {
-			t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
-		}
-	}
-}
-
-func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
-	code, _, errb := runOut("-ä", "sv_SE.word")
-	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
-		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
-	}
-}
-
-func TestRunEmptyDataPathIsNamed(t *testing.T) {
-	code, _, errb := runOut("--data-path=", "sv_SE.word")
-	if code != 1 || !strings.Contains(errb, "empty") {
-		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
-	}
-}
-
-func recordDir(t *testing.T) string {
-	t.Helper()
-	dir := t.TempDir()
-	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
-	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
-func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil {
-		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
-	}
-	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
-		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
-	}
-}
-
-func TestRunRecordNDJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
-	}
-	for _, line := range lines {
-		var m map[string]string
-		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
-			t.Fatalf("ndjson line %q is not one object: %v", line, err)
-		}
-	}
-}
-
-func TestRunRecordCSVRoundTrips(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 4 {
-		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
-	}
-	if lines[0] != "first,last" {
-		t.Fatalf("csv header = %q, want first,last", lines[0])
-	}
-	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
-	if err != nil {
-		t.Fatalf("csv output is not valid CSV: %v", err)
-	}
-	if len(r) != 4 || len(r[0]) != 2 {
-		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
-	}
-}
-
-func TestRunRecordSQL(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
-		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
-	}
-}
-
-func TestRunRecordDefaultTable(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
-	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
-		t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
-	}
-}
-
-func TestRunRecordInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
-	if code != 0 {
-		t.Fatalf("inline record = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
-		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
-	}
-}
-
-func TestRunRecordMisuse(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
-		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
-		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
-		}
-	}
-}
-
-func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
-	if code != 1 {
-		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
-	}
-}
-
-func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
-	dir := t.TempDir()
-	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
-	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
-	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
-	}
-	code, _, errb = runOut("--format", "json", content)
-	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
-		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
-	}
-}
-
-func TestRunNumbersFollowTheShell(t *testing.T) {
-	_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
-	code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
-	if code != 0 || got != want {
-		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
-	}
-}
-
-// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
-// for columns: the record to write, and the quoting a shell needs to take it. The
-// argument is well-formed and only the data cannot serve it, so the code is 1.
-func TestRunRecordOfAFieldlessPath(t *testing.T) {
-	code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
-	if code != 1 {
-		t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
-	}
-	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("stderr = %q, want it to name %s", errb, want)
-		}
-	}
-	if out != "" {
-		t.Errorf("stdout = %q, want nothing", out)
 	}
 }

compose.yaml

@@ -1,21 +1,5 @@
-# Docker-based dev workflow — no local tooling beyond Docker is required.
-#
-#   docker compose run --rm test
-#   docker compose run --rm vet
-#   docker compose run --rm build
-#
-# The full gate — vet, formatting and tests — is `docker build .`, which is what
-# CI runs.
-#
-# Build caches persist in the `gocache` volume, so re-runs are fast and the
-# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
-# ownership when run with `--user "$(id -u):$(id -g)"`.
-#
-# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
-# supported Go with: GO_VERSION=1.22 docker compose run --rm test
-
 x-go: &go
-  image: golang:${GO_VERSION:-1.26.4}
+  image: golang:${GO_VERSION:-1.27.2}
   working_dir: /app
   volumes:
     - .:/app
@@ -33,12 +17,16 @@ services:
 
   cover:
     <<: *go
-    command: go test -cover ./...
+    command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
 
   bench:
     <<: *go
     command: go test -run XXX -bench . -benchmem ./...
 
+  generate:
+    <<: *go
+    command: go generate ./...
+
   vet:
     <<: *go
     command: go vet ./...
@@ -59,13 +47,12 @@ services:
     <<: *go
     command: go mod tidy
 
-  # Interactive shell for ad-hoc work: docker compose run --rm dev
   dev:
     <<: *go
     command: sh
 
   data-import:
-    image: python:3.14.7-slim
+    image: python:3.14.8-slim
     working_dir: /app
     volumes:
       - .:/app

data-import/geo-se.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
 
-    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
+    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import csv
 import io
 import json
-import math
 import os
 import re
 import sys
@@ -16,6 +14,7 @@ import urllib.request
 import zipfile
 from pathlib import Path
 
+import geo
 import source
 import tsv
 import xlsx
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
 POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
 NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
 NVDB_PAGE = 50000
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
-CACHE = Path(__file__).resolve().parent / "cache"
 TIMEZONE = "Europe/Stockholm"
+# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
 ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
+# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
 UNMATCHED_POPULATION = 200
+# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
+# or a description in one.
+NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
+PARTICLES = {"af", "de", "von"}
 
 
 def scb_codes(cache):
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
         yield name, float(lat), float(lon)
 
 
-class Nearest:
-    """Nearest point by an equirectangular distance, over a degree grid."""
-
-    def __init__(self, points, cell=0.05):
-        self.cell = cell
-        self.grid = collections.defaultdict(list)
-        for lat, lon, value in points:
-            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
-
-    def find(self, lat, lon):
-        ci, cj = int(lat // self.cell), int(lon // self.cell)
-        best, best_d = None, math.inf
-        ring = 0
-        while ring < 400:
-            for i in range(ci - ring, ci + ring + 1):
-                for j in range(cj - ring, cj + ring + 1):
-                    if max(abs(i - ci), abs(j - cj)) != ring:
-                        continue
-                    for plat, plon, value in self.grid.get((i, j), ()):
-                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
-                        if d < best_d:
-                            best, best_d = value, d
-            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
-                return best
-            ring += 1
-        return best
-
-
 def street_delivery(name, codes):
-    """The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
+    """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
+    least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
+    for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
+    docs/research/research-geo-se.md, Postal codes → Structure."""
     if name in ONE_POSITION:
         return [c for c in codes if c[1] != "0"]
     largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
 
 
 def well_cased(name):
+    """Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
+    capitals or lower case."""
     return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
 
 
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
         municipality, method = municipality_of(name, rows, tatorter, municipalities)
         how[method] += 1
         kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
-        with_point = [r for r in rows if r["lat"] is not None]
+        point = geo.centroid(rows)
-        if municipality is None or not kept or not with_point or not well_cased(name):
+        if municipality is None or not kept or point is None or not well_cased(name):
             continue
-        lat = sum(r["lat"] for r in with_point) / len(with_point)
+        lat, lon = point
-        lon = sum(r["lon"] for r in with_point) / len(with_point)
         out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
-    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr)
+    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
     return out
 
 
-def streets(segments, codes, localities, per_locality):
+def street_name(name):
+    # A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
+    opening = name[0].isupper() or name.split()[0] in PARTICLES
+    return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
+
+
+def streets(segments, codes, localities, share, minimum):
     """The names with most segments per locality, each segment at its nearest code centroid."""
-    nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
+    nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
     count = collections.Counter()
     for name, lat, lon in segments:
-        if name[0].isalpha():
+        if street_name(name):
             count[(nearest.find(lat, lon), name)] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "segments")
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
     key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
     if not key:
         sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
-    cache, out = Path(a.cache), Path(a.out)
+    cache, out = geo.directories(a)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
-    regions, municipalities = scb_codes(cache)
+    region_names, municipality_names = scb_codes(cache)
     population = scb_population(cache)
     codes = geonames(cache)
-    places = localities(codes, scb_tatorter(cache), municipalities, population)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
-    named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
+    named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
-    places = {name: l for name, l in places.items() if name in named}
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
-    if empty:
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
-        sys.exit(f"municipalities without a locality: {empty}")
+    geo.log_dropped_localities(dropped["locality"])
+    if dropped["municipality"] or dropped["region"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo-us.py

@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
-"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
+"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
 
-    data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
+    data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import concurrent.futures
 import csv
 import io
+import math
+import random
 import re
 import struct
 import sys
+import urllib.parse
 import zipfile
-from pathlib import Path
 
+import geo
 import source
 import tsv
 
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
 PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
 ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
 TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
-CACHE = Path(__file__).resolve().parent / "cache"
 ESTIMATE = "POPESTIMATE2025"
 CDP = "57"
+# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
+CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
+SELECT ?iso ?capitalLabel ?gnis WHERE {
+  ?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
+  OPTIONAL { ?capital wdt:P590 ?gnis. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}"""})
+SHIPPED_POPULATION = 100000
 HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
 SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
-# Places whose Census name is a merged government's; the postal city is what an address carries.
+# Places whose Census name is not the postal city an address carries.
-NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
+NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
 TIMEZONES = {
     "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
     "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -107,8 +115,8 @@ def place_name(geoid, name):
     return stripped
 
 
-def localities(cache, min_population, counties):
+def localities(cache, counties):
-    """Each shipped place with its county, population and centroid."""
+    """Each incorporated place with its county, population and centroid."""
     place_population, county_part = {}, collections.defaultdict(list)
     for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
         if r["SUMLEV"] == "162":
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
     out = {}
     for r in gazetteer(cache, "place"):
         geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
-        if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid):
+        if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
             continue
         county = max(county_part[geoid])[1]
         if county not in counties:
             print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
             continue
-        out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
+        out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
     return out
 
 
-def postal_codes(cache, localities):
+def capitals(cache, places, counties):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
+    """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
+    by_gnis = {p["gnis"]: code for code, p in places.items()}
+    named = collections.defaultdict(list)
+    for code, p in places.items():
+        named[(counties[p["municipality"]]["region"], p["name"])].append(code)
+    by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
+    gnis_of = collections.defaultdict(set)
+    for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
+        gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
+    out = set()
+    for (state, name), ids in sorted(gnis_of.items()):
+        code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
+        if code:
+            out.add(code)
+        else:
+            print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
+    return out
+
+
+def drawn(places, share):
+    """The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
+    sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
+    the sources moves only the places near the cut."""
+    peopled = [code for code, p in places.items() if p["population"] > 0]
+    key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
+    return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
+
+
+def zcta_localities(cache, localities):
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
     return {zcta: place for zcta, place in largest.items() if place in localities}
 
 
-def streets(cache, counties, locality_of_zcta, per_locality):
+def streets(cache, counties, locality_of_zcta, share, minimum):
     """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
     with concurrent.futures.ThreadPoolExecutor(3) as pool:
         list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
             if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
                 for z in zips.get(r["TLID"], ()):
                     count[(locality_of_zcta[z], r["FULLNAME"])] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "addresses"), addresses
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
-    return named, addresses
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
+    cache, out = geo.directories(a)
-    p.add_argument("--min-population", type=int, default=25000)
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
     state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
     regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
@@ -182,20 +208,29 @@ def main():
         counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
 
-    places = localities(cache, a.min_population, counties)
+    places = localities(cache, counties)
-    locality_of_zcta = postal_codes(cache, places)
+    owner_of_zcta = zcta_localities(cache, places)
-    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
+    capital_codes = capitals(cache, places, counties)
-    for geoid in [l for l in places if l not in named]:
+    owners = {p: places[p] for p in set(owner_of_zcta.values())}
-        print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr)
+    for code in sorted(capital_codes - set(owners)):
-        del places[geoid]
+        print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
-    kept_counties = {l["municipality"] for l in places.values()}
+    places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
+    locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
+    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped_localities(dropped["locality"])
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
-    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo.py

@@ -0,0 +1,98 @@
+"""The steps a country's geo import may share with another's."""
+import argparse
+import collections
+import math
+import sys
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
+CACHE = Path(__file__).resolve().parent / "cache"
+
+
+def parser(doc, country, street_share, **own):
+    """A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
+    "_" standing for "-"."""
+    options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
+    p = argparse.ArgumentParser(description=doc.splitlines()[0])
+    for name in sorted(options):
+        p.add_argument("--" + name.replace("_", "-"), **options[name])
+    return p
+
+
+def directories(args):
+    """The cache and out directories of the parsed options, both created."""
+    cache, out = Path(args.cache), Path(args.out)
+    cache.mkdir(parents=True, exist_ok=True)
+    out.mkdir(parents=True, exist_ok=True)
+    return cache, out
+
+
+def centroid(rows):
+    """The mean lat and lon of the rows with a point, or None where no row has one."""
+    points = [r for r in rows if r["lat"] is not None]
+    if not points:
+        return None
+    return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
+
+
+class Nearest:
+    """Nearest point by an equirectangular distance, over a degree grid."""
+
+    def __init__(self, points, cell=0.05):
+        self.cell = cell
+        self.grid = collections.defaultdict(list)
+        for lat, lon, value in points:
+            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
+
+    def find(self, lat, lon):
+        ci, cj = int(lat // self.cell), int(lon // self.cell)
+        best, best_d = None, math.inf
+        ring = 0
+        while ring < 400:
+            for i in range(ci - ring, ci + ring + 1):
+                for j in range(cj - ring, cj + ring + 1):
+                    if max(abs(i - ci), abs(j - cj)) != ring:
+                        continue
+                    for plat, plon, value in self.grid.get((i, j), ()):
+                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
+                        if d < best_d:
+                            best, best_d = value, d
+            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
+                return best
+            ring += 1
+        return best
+
+
+def top_streets(count, share, minimum, column):
+    """count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
+    them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
+    of = collections.defaultdict(list)
+    for (locality, name), n in count.items():
+        of[locality].append((n, name))
+    return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
+
+
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
+
+
+def log_dropped_localities(rows):
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)

data-import/httpmethod.py

@@ -23,7 +23,7 @@ CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["idempotent", "method", "safe"]
 CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
 BOOLEAN = {"yes": "true", "no": "false"}
-SELECTOR = '[]{}"|'  # mirrors inSelector in table.go
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
 EXPECTED = 9
 
 

data-import/protocol.py

@@ -21,7 +21,7 @@ OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
 CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["keyword", "name", "number"]
 SKIP = ("Reserved", "deprecated")
-SELECTOR = '[]{}"|'  # mirrors inSelector in table.go
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
 
 
 def rows(text):

data-import/source.py

@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
     if path.exists():
         return path.read_bytes()
     for attempt in range(1, 6):
-        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
+        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
         try:
             with urllib.request.urlopen(req, timeout=600) as r:
                 body = r.read()

data-import/tsv.py

@@ -5,14 +5,14 @@ from pathlib import Path
 
 
 def write(path, columns, rows):
-    """Write the rows as a TSV; the table needs two rows, and every cell must be non-empty and free of tabs, newlines and braces."""
+    """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
     lines = ["\t".join(columns)]
     for row in rows:
-        cells = [str(row[c]) for c in columns]
+        cells = [str(row[c]).strip() for c in columns]
         if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
             raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
         lines.append("\t".join(cells))
-    if len(lines) < 3:
+    if len(lines) < 2:
-        raise ValueError(f"{path}: has {len(lines) - 1} rows below its header; a table is at least two rows")
+        raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
     Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
     print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)

data.go

@@ -1,195 +1,175 @@
 package fejkdata
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
+// loadSources merges sources in order, the last winning a clash. A source whose manifest
-// label prefixes file names in errors; onDisk marks path as a directory that must
+// carries an index leaves its categories unloaded; every other source loads here, with
-// exist.
+// the unloaded categories it needs.
-type dataSource struct {
+// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
-	fsys   fs.FS
+func loadSources(sources []datafiles.Source) (folder, error) {
-	label  string
+	root := folder{children: map[string]node{}}
-	onDisk bool
-	path   string
-	root   string
-}
-
-func (s dataSource) name(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-// loadData loads every source into one namespace tree and returns its root
-// children. A directory becomes a folder; each *.json file in it compiles to a node
-// keyed by its base name (address.json -> "address"); each subdirectory becomes a
-// nested folder, so folders turn into dot-path segments. Sources merge left to right:
-// matching folders merge by their children, and any other clash is won by the last
-// source loaded. Once merged, linkRefs binds every reference against the
-// final tree.
-func loadData(sources []dataSource) (map[string]node, error) {
-	root := map[string]node{}
 	for _, src := range sources {
-		if src.onDisk {
+		m, err := src.Manifest()
-			if src.path == "" {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.path)
-			if err != nil {
-				return nil, fmt.Errorf("data path %s: %w", src.path, err)
-			}
-			if !info.IsDir() {
-				return nil, fmt.Errorf("%s is not a directory", src.path)
-			}
-		}
-		dir := src.root
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
 		if err != nil {
-			return nil, err
+			return folder{}, err
+		}
+		if err := addSource(&root, src, m); err != nil {
+			return folder{}, err
 		}
-		mergeChildren(root, g.children)
 	}
-	if len(root) == 0 {
+	return root, loadReached(&root, categorySites(&root), nil)
-		return nil, fmt.Errorf("no .json data found")
-	}
-	setTablePaths(root)
-	if err := linkTables(root); err != nil {
-		return nil, err
-	}
-	if err := linkRefs(root); err != nil {
-		return nil, err
-	}
-	if err := checkNoCycles(root); err != nil {
-		return nil, err
-	}
-	if err := checkScope(treeScope(root)); err != nil {
-		return nil, err
-	}
-	return root, nil
 }
 
-// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
+func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
-// by name, so the tree is built deterministically. Empty subdirectories (no JSON
+	src = src.Mounted(m.Namespace)
-// anywhere under them) are skipped rather than added as empty namespaces.
+	if m.Index != nil {
-func loadDir(src dataSource, dir string) (*folder, error) {
+		placeIndex(root, src, m.Index)
-	entries, err := fs.ReadDir(src.fsys, dir)
+		return nil
-	if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.name(dir), err)
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
-	for _, e := range entries {
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
-		}
-	}
-	for _, e := range entries {
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
-		}
-		full := path.Join(dir, e.Name())
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
-		}
-	}
-	return g, nil
-}
-
-// categoryFiles is what a category may name beside itself: the rows files of its
-// directory, each marked once a category names it.
-type categoryFiles struct {
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-// rows reads the rows file a category names beside it.
-func (c *categoryFiles) rows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
-	if err != nil {
 		return err
 	}
-	if len(child.children) == 0 {
+	mergeFolder(root, g)
-		return nil
-	}
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.name(full), err)
-	}
-	g.children[name] = child
 	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
+func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
-// is skipped.
+	return func(c datafiles.Category) error {
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
+		n, err := compileCategory(c)
-	if !strings.HasSuffix(file, ".json") {
+		if err != nil {
+			return err
+		}
+		place(c.Folders).put(c.Name, n)
 		return nil
 	}
-	name := strings.TrimSuffix(file, ".json")
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.name(full), err)
-	}
-	b, err := fs.ReadFile(src.fsys, full)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	g.children[name] = n
-	return nil
 }
 
-// mergeChildren overlays src onto dst. Two folders under the same key merge
+// madeFolder is the folder that dir names under root, made where it is missing or a category.
-// recursively (so locales/categories from several paths combine); every other
+func madeFolder(root *folder, dir []string) *folder {
-// key is replaced, making the last-loaded directory win on a conflict.
+	g := root
-func mergeChildren(dst, src map[string]node) {
+	for _, seg := range dir {
-	for k, v := range src {
+		sub, isFolder := g.children[seg].(*folder)
-		if dg, ok := dst[k].(*folder); ok {
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.put(seg, sub)
+		}
+		g = sub
+	}
+	return g
+}
+
+func mergeFolder(dst, src *folder) {
+	for k, v := range src.children {
+		if dg, ok := dst.children[k].(*folder); ok {
 			if sg, ok := v.(*folder); ok {
-				mergeChildren(dg.children, sg.children)
+				mergeFolder(dg, sg)
 				continue
 			}
 		}
-		dst[k] = v
+		dst.put(k, v)
 	}
 }
+
+// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
+// single inline node.
+type nodeSet func(fn func(label string, n node) error) error
+
+// categorySite is a loaded category, where it sits and its own dot path.
+type categorySite struct {
+	categoryAt
+	n    node
+	path string
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, siteIn(dir, g, name))
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func siteIn(dir []string, in *folder, name string) categorySite {
+	return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
+}
+
+func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
+
+func siteNodes(sites []categorySite) nodeSet {
+	return func(fn func(label string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func inlineNodes(n node, label string) nodeSet {
+	return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
+}
+
+// pipeline is the one load pipeline, run over one nodeSet; its fields are where
+// entry points differ.
+type pipeline struct {
+	nodes   nodeSet
+	resolve func() error
+	// Set by a caller that proves the columns against their Go types itself.
+	typedByGo bool
+}
+
+// categoryPipeline is the pipeline over the categories of one load, their references
+// resolving against root.
+func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
+	return pipeline{
+		nodes: siteNodes(sites),
+		resolve: func() error {
+			if err := linkTables(sites); err != nil {
+				return err
+			}
+			return resolveCategoryTemplates(sites, root)
+		},
+	}
+}
+
+func (p pipeline) run() error {
+	if err := p.resolve(); err != nil {
+		return err
+	}
+	if err := checkNoCycles(p.nodes); err != nil {
+		return err
+	}
+	settleRecords(p.nodes)
+	if !p.typedByGo {
+		if err := checkColumns(p.nodes); err != nil {
+			return err
+		}
+	}
+	return checkNodeFences(p.nodes)
+}
+
+// checkNodeFences runs the per-node fences over s, each over every node before the next,
+// so which of several broken nodes is reported does not depend on the walk. Its walks
+// recurse unguarded, so pipeline.run refuses cycles first.
+func checkNodeFences(s nodeSet) error {
+	mem := renderCounts{}
+	if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
+		return err
+	}
+	return s((&valueProof{}).checkDatatype)
+}

data/en_US/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"en_US.color": {"paths":[""]},
+		"en_US.company": {"paths":["","base","suffix"]},
+		"en_US.date": {"paths":[""]},
+		"en_US.email": {"paths":["","domain","local","local.n"]},
+		"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"en_US.ip": {"paths":[""]},
+		"en_US.itin": {"paths":["","area","group","serial"]},
+		"en_US.last-name": {"paths":["","count","name"]},
+		"en_US.person": {"paths":["","first","last","prefix","sex"]},
+		"en_US.phone": {"paths":["","area","exchange","line"]},
+		"en_US.phone-exchange": {"paths":[""]},
+		"en_US.price": {"paths":["","amt","cents"]},
+		"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
+		"en_US.ssn": {"paths":["","area","group","serial"]},
+		"en_US.time": {"paths":[""]},
+		"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
+		"en_US.url": {"paths":["","host","path"]},
+		"en_US.username": {"paths":["","n"]},
+		"en_US.version": {"paths":["","n","pre","suffix"]},
+		"en_US.word": {"paths":[""]}
+	},
+	"namespace": "en_US"
+}

data/en_US/address.json

@@ -1,8 +1,8 @@
 {
-  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
+  "format": "{/geo.US.address as a}{a}",
-  "street": "{/geo.US.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.US.address.street-number}",
+  "street-number": "{a.street-number}",
-  "locality": "{/geo.US.address.locality}",
+  "locality": "{a.locality}",
-  "region": "{/geo.US.address.region}",
+  "region": "{a.region}",
-  "postal-code": "{/geo.US.address.postal-code}"
+  "postal-code": "{a.postal-code}"
 }

data/en_US/fs.go

@@ -0,0 +1,19 @@
+// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
+// geo.US, which package geous holds.
+package enus
+
+import (
+	"embed"
+	"io/fs"
+
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.US.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }

data/en_US/fs_test.go

@@ -0,0 +1,22 @@
+package enus_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	enus "github.com/larvit/fejkdata/data/en_US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/en_US/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
-  "prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }],
+  "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data/en_US/phone-exchange.json

@@ -0,0 +1,17 @@
+[
+  { "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
+  { "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
+  { "format": "{int(2,9)}10", "weight": 8 },
+  { "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
+  { "format": "55{int(0,4)}", "weight": 5 },
+  { "format": "55{int(6,9)}", "weight": 4 },
+  { "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
+  { "format": "95{int(1,7)}", "weight": 7 },
+  { "format": "96{digits(1)}", "weight": 10 },
+  { "format": "97{int(0,5)}", "weight": 6 },
+  { "format": "97{int(7,9)}", "weight": 3 }
+]

data/en_US/phone.json

@@ -1,15 +1,15 @@
 [
   {
-    "format": "({area}) {exch}-{line}",
+    "format": "({area}) {exchange}-{line}",
     "weight": 2,
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   },
   {
-    "format": "{area}-{exch}-{line}",
+    "format": "{area}-{exchange}-{line}",
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   }
 ]

data/geo/SE/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
+		"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
+		"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
+		"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
+	},
+	"namespace": "geo.SE"
+}

data/geo/SE/address.json

@@ -1,6 +1,6 @@
 {
   "format": "{street} {street-number}\n{postal-code} {locality}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(1,9)}",
     "{int(10,99)}",
@@ -9,6 +9,6 @@
     { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
     { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
   ],
-  "postal-code": "{.postal-code.code}",
+  "postal-code": "{l.postal-code.code}",
-  "locality": "{.locality.name}"
+  "locality": "{l.name}"
 }

data/geo/SE/fs.go

@@ -0,0 +1,15 @@
+// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
+package geose
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/SE/fs_test.go

@@ -0,0 +1,22 @@
+package geose_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/geo/SE/locality.tsv

@@ -866,7 +866,6 @@ Nynäshamn	0192	16447	58.9034	17.9479
 Nyvång	1277	200	56.1333	12.9000
 Nyåker	2403	200	63.7500	19.3167
 Nälden	2309	986	63.3500	14.2500
-Nämdö	0120	200	59.1833	18.6833
 Nässjö	0682	18639	57.6531	14.6968
 Näsum	1272	1274	56.1667	14.4833
 Näsviken	2184	994	61.7667	16.8667

data/geo/SE/postal-code.tsv

@@ -949,7 +949,6 @@ code	locality
 130 25	Ingmarsö
 130 33	Gällnöby
 130 34	Skälvik
-130 36	Nämdö
 130 37	Stavsnäs
 130 38	Runmarö
 130 39	Sandhamn

data/geo/US/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
+		"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
+		"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
+		"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
+	},
+	"namespace": "geo.US"
+}

data/geo/US/address.json

@@ -1,13 +1,13 @@
 {
   "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 2 },
     { "format": "{int(1000,9999)}", "weight": 0.5 },
     { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
   ],
-  "locality": "{.locality.name}",
+  "locality": "{l.name}",
-  "region": "{.region.abbr}",
+  "region": "{l..municipality..region.abbr}",
-  "postal-code": "{.postal-code.code}"
+  "postal-code": "{l.postal-code.code}"
 }

data/geo/US/fs.go

@@ -0,0 +1,15 @@
+// Package geous is fejkdata's US places module, whose categories are geo.US.*.
+package geous
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/US/fs_test.go

@@ -0,0 +1,22 @@
+package geous_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/.fejkdata.json

@@ -0,0 +1,25 @@
+{
+	"index": {
+		"misc.car": {"paths":["","make","model"]},
+		"misc.coordinate": {"paths":["","lat","lon"]},
+		"misc.creditcard": {"paths":["","d"]},
+		"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
+		"misc.datetime": {"paths":[""]},
+		"misc.emoji": {"paths":[""]},
+		"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
+		"misc.httpstatus": {"paths":["","code","reason"]},
+		"misc.language": {"paths":["","code","code3","name"]},
+		"misc.loglevel": {"paths":["","code","keyword","severity"]},
+		"misc.mac": {"paths":[""]},
+		"misc.mimetype": {"paths":["","ext","type"]},
+		"misc.objectid": {"paths":[""]},
+		"misc.port": {"paths":["","number","service"]},
+		"misc.protocol": {"paths":["","keyword","name","number"]},
+		"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
+		"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
+		"misc.tld": {"paths":["","tld","type","unicode"]},
+		"misc.useragent": {"paths":["","browser","device","os","ua"]},
+		"misc.uuid": {"paths":["","variant"]}
+	},
+	"namespace": "misc"
+}

data/misc/currency.tsv

@@ -71,7 +71,7 @@ JPY	0	Yen	392	¥
 KES	2	Kenyan Shilling	404	KES
 KGS	2	Som	417	⃀
 KHR	2	Riel	116	៛
-KMF	0	Comorian Franc 	174	CF
+KMF	0	Comorian Franc	174	CF
 KPW	2	North Korean Won	408	₩
 KRW	0	Won	410	₩
 KWD	3	Kuwaiti Dinar	414	KWD

data/misc/fs.go

@@ -0,0 +1,15 @@
+// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
+package misc
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/misc/fs_test.go

@@ -0,0 +1,22 @@
+package misc_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	"github.com/larvit/fejkdata/data/misc"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/territory.tsv

@@ -48,7 +48,7 @@ CO	COL	57	Bogota	CO	COP	🇨🇴	es-CO	Colombia	170	.co
 CR	CRI	506	San Jose	CR	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
 CU	CUB	53	Havana	CU	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
 CV	CPV	238	Praia	CV	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
-CW	CUW	599	 Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
+CW	CUW	599	Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
 CX	CXR	61	Flying Fish Cove	AU	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
 CY	CYP	357	Nicosia	CY	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
 CZ	CZE	420	Prague	CZ	CZK	🇨🇿	cs,sk	Czechia	203	.cz

data/modules.go

@@ -0,0 +1,19 @@
+// Package data lists every module fejkdata ships.
+package data
+
+import (
+	"io/fs"
+
+	enus "github.com/larvit/fejkdata/data/en_US"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+	"github.com/larvit/fejkdata/data/misc"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
+
+// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS {
+	return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
+}

data/sv_SE/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
+		"sv_SE.color": {"paths":[""]},
+		"sv_SE.company": {"paths":["","base","suffix"]},
+		"sv_SE.date": {"paths":[""]},
+		"sv_SE.email": {"paths":["","domain","local","local.n"]},
+		"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"sv_SE.ip": {"paths":[""]},
+		"sv_SE.last-name": {"paths":["","count","name"]},
+		"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
+		"sv_SE.personnummer": {"paths":[""]},
+		"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
+		"sv_SE.price": {"paths":["","amt","ore"]},
+		"sv_SE.samordningsnummer": {"paths":[""]},
+		"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
+		"sv_SE.time": {"paths":[""]},
+		"sv_SE.title": {"paths":["","name","share"]},
+		"sv_SE.url": {"paths":["","host","path"]},
+		"sv_SE.username": {"paths":["","n"]},
+		"sv_SE.version": {"paths":["","n","pre","suffix"]},
+		"sv_SE.word": {"paths":[""]}
+	},
+	"namespace": "sv_SE"
+}

data/sv_SE/address.json

@@ -1,7 +1,7 @@
 {
-  "format": "{street} {street-number}\n{postal-code} {locality}",
+  "format": "{/geo.SE.address as a}{a}",
-  "street": "{/geo.SE.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.SE.address.street-number}",
+  "street-number": "{a.street-number}",
-  "postal-code": "{/geo.SE.address.postal-code}",
+  "postal-code": "{a.postal-code}",
-  "locality": "{/geo.SE.address.locality}"
+  "locality": "{a.locality}"
 }

data/sv_SE/fs.go

@@ -0,0 +1,19 @@
+// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
+// geo.SE, which package geose holds.
+package svse
+
+import (
+	"embed"
+	"io/fs"
+
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.SE.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }

data/sv_SE/fs_test.go

@@ -0,0 +1,22 @@
+package svse_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/sv_SE/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
   "prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data_test.go

@@ -1,7 +1,10 @@
 package fejkdata
 
 import (
+	"io/fs"
+	"os"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
 			regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
 	}
 
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, c := range cases {
 		for i := 0; i < 200; i++ {
 			if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
 	}
 }
 
-// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper
+// TestShippedMiscCategories covers the locale-neutral misc module: a proper
 // v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
 // whose trailing {luhn()} check passes.
 func TestShippedMiscCategories(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
 	mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
 	objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
@@ -93,7 +96,7 @@ func TestShippedMiscCategories(t *testing.T) {
 // codes follow their standard shape, dotted sub-paths resolve, emoji is a real
 // glyph, and a coordinate parses to a point within geographic bounds.
 func TestShippedMiscReferenceData(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	re := map[string]*regexp.Regexp{
 		"currency":              regexp.MustCompile(`^[A-Z]{3}$`),
 		"currency.name":         regexp.MustCompile(`\p{L}`),
@@ -156,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
 // TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
 // regexes miss: no generated name repeats a character three times over.
 func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
 		for i := 0; i < 20000; i++ {
 			name := fake(t, f, path)
@@ -175,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
 // male), the trailing digit is a Luhn checksum over the other nine, and a
 // samordningsnummer is the same with 60 added to the day.
 func TestSwedishPersonnummer(t *testing.T) {
-	sv := newGenerator(t, "data", WithSeed(1))
+	sv := shipped(t, WithSeed(1))
 	re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
 	sawLongMonthEnd, birth := false, map[string]bool{}
 	for i := 0; i < 2000; i++ {
@@ -208,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
 	if !birth["238"] || !birth["239"] {
 		t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
 	}
-	for i := 0; i < 200; i++ {
-		got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
-		sex, id, _ := strings.Cut(got, " ")
-		if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
-			t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
-		}
-	}
 }
 
 // TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
 // 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
 // in 50-65, 70-88, 90-92 or 94-99.
 func TestShippedUSTaxIds(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
 	itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
 	for i := 0; i < 2000; i++ {
@@ -245,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
 // TestShippedPersonNames pins the name tables: a sex pins its first names, a name
 // both sexes carry appears under both, and the record's sex agrees with its name.
 func TestShippedPersonNames(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(9))
+	f := shipped(t, WithSeed(9))
 	for path, want := range map[string]string{
 		"sv_SE.sex[f]":                        "kvinna",
 		"sv_SE.sex[man].code":                 "m",
@@ -273,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
 		seen := map[string]bool{}
 		for i := 0; i < 2000; i++ {
 			seen[fake(t, f, locale+".sex[f].first-name")] = true
-			first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex")
+			got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
-			if first == "" || sex == "" {
+			first, sex, _ := strings.Cut(got, "|")
-				t.Fatalf("%s.person lacks a first name or a sex", locale)
+			fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
-			}
 		}
 		if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
 			t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
 		}
-		got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`)
+	}
-		code, first, _ := strings.Cut(got, "|")
+}
-		if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
+
-			t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code)
+// TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
+// from one pick, and that two names bound to one person category are two people.
+func TestShippedPersonIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(9))
+	for _, locale := range []string{"sv_SE", "en_US"} {
+		apart := false
+		for i := 0; i < 300; i++ {
+			got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
+			parts := strings.Split(got, "|")
+			for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
+				fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
+			}
+			if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
+				fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
+			}
+			apart = apart || parts[0] != parts[3]
+		}
+		if !apart {
+			t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
 		}
 	}
 }
 
 func TestShippedSwedishPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
 	for i := 0; i < 50; i++ {
 		if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
@@ -300,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
 }
 
 func TestShippedSwedishAddress(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(3))
+	f := shipped(t, WithSeed(3))
 	digit := regexp.MustCompile(`\d`)
 	for i := 0; i < 30; i++ {
 		a := fake(t, f, "sv_SE.address")
@@ -317,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
 	}
 }
 
+// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
+// locality and region are one place in the geo tables, read through a name in a template or
+// across a struct's tags, and that a reference beside the name is another address.
+func TestShippedAddressIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(3))
+	for _, c := range []struct{ locale, country, template, largest string }{
+		{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
+		{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
+	} {
+		places := geoPlaces(t, c.country)
+		apart, localities := false, map[string]int{}
+		for i := 0; i < 2000; i++ {
+			got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
+			parts := strings.Split(got, "|")
+			if !places[strings.Join(parts[:4], "|")] {
+				t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
+			}
+			apart = apart || parts[0] != parts[4]
+			localities[parts[2]]++
+		}
+		if !apart {
+			t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
+		}
+		if c.country == "SE" {
+			var row struct {
+				Street     string `fake:"{/sv_SE.address as a}{a.street}"`
+				PostalCode string `fake:"{a.postal-code}"`
+				Locality   string `fake:"{a.locality}"`
+			}
+			for i := 0; i < 200; i++ {
+				if err := f.FakeStruct(&row); err != nil {
+					t.Fatal(err)
+				}
+				if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
+					t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
+				}
+			}
+		}
+		for l, n := range localities {
+			if n > localities[c.largest] {
+				t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
+			}
+		}
+	}
+}
+
+// 0644000 is Los Angeles.
+func TestShippedPlacesSpreadAddresses(t *testing.T) {
+	for country, city := range map[string]struct {
+		key   string
+		least int
+	}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
+		streets := 0
+		for _, r := range geoRows(t, country, "street") {
+			if r[1] == city.key {
+				streets++
+			}
+		}
+		if streets < city.least {
+			t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
+		}
+	}
+	header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
+	population := slices.Index(strings.Split(header, "\t"), "population")
+	if population < 0 {
+		t.Fatal("geo.US locality.tsv has no population column")
+	}
+	for _, r := range geoRows(t, "US", "locality") {
+		n, err := strconv.Atoi(r[population])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n < 2500 {
+			return
+		}
+	}
+	t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
+}
+
+// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
+// draw by population leaves out.
+func TestShippedCapitalsAndBigPlaces(t *testing.T) {
+	state := map[string]string{}
+	for _, r := range geoRows(t, "US", "municipality") {
+		state[r[0]] = r[2]
+	}
+	shipped := map[string]bool{}
+	for _, r := range geoRows(t, "US", "locality") {
+		shipped[r[1]+", "+state[r[2]]] = true
+	}
+	for _, place := range []string{
+		"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
+		"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
+		"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
+		"Ventura, CA",
+	} {
+		if !shipped[place] {
+			t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
+		}
+	}
+}
+
+func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
+	refused := map[string]bool{
+		"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
+		"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
+		"Öresundsvägen, Bondevägen": true,
+	}
+	for _, r := range geoRows(t, "SE", "street") {
+		if refused[r[0]] {
+			t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
+		}
+	}
+}
+
+func geoTSV(t *testing.T, country, table string) string {
+	b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func geoRows(t *testing.T, country, table string) [][]string {
+	var out [][]string
+	for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
+		out = append(out, strings.Split(line, "\t"))
+	}
+	return out
+}
+
+// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
+// one locality holds, the region empty for SE, whose address carries none.
+func geoPlaces(t *testing.T, country string) map[string]bool {
+	rows := func(table string) [][]string { return geoRows(t, country, table) }
+	name, region := map[string]string{}, map[string]string{}
+	if country == "US" {
+		county := map[string]string{}
+		for _, r := range rows("municipality") {
+			county[r[0]] = r[2]
+		}
+		for _, r := range rows("locality") {
+			name[r[0]], region[r[0]] = r[1], county[r[2]]
+		}
+	} else {
+		for _, r := range rows("locality") {
+			name[r[0]] = r[0]
+		}
+	}
+	codes := map[string][]string{}
+	for _, r := range rows("postal-code") {
+		codes[r[1]] = append(codes[r[1]], r[0])
+	}
+	places := map[string]bool{}
+	for _, r := range rows("street") {
+		for _, code := range codes[r[1]] {
+			places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
+		}
+	}
+	return places
+}
+
 func TestShippedPersonHasParts(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(7))
+	f := shipped(t, WithSeed(7))
 	for _, path := range []string{"sv_SE.person", "en_US.person"} {
 		for i := 0; i < 30; i++ {
 			if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
@@ -328,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
 	}
 }
 
+// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
 func TestShippedUSPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
-	re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`)
+	re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
-	for i := 0; i < 50; i++ {
+	unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
-		if n := fake(t, f, "en_US.phone"); !re.MatchString(n) {
+	for i := 0; i < 2000; i++ {
+		n := fake(t, f, "en_US.phone")
+		m := re.FindStringSubmatch(n)
+		if m == nil {
 			t.Fatalf("phone %q does not match %s", n, re)
 		}
+		if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
+			t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
+		}
 	}
 }
 
-// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and
+func TestShippedTableCellsAreTrimmed(t *testing.T) {
-// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person.
+	err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil || !strings.HasSuffix(path, ".tsv") {
+			return err
+		}
+		b, err := fs.ReadFile(os.DirFS("data"), path)
+		if err != nil {
+			return err
+		}
+		for i, line := range strings.Split(string(b), "\n") {
+			for _, cell := range strings.Split(line, "\t") {
+				if cell != strings.TrimSpace(cell) {
+					t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
+// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
 func TestShippedNamespacedTree(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
 		if got := fake(t, f, path); got == "" {
 			t.Fatalf("Fake(%q) returned empty", path)
@@ -385,7 +590,7 @@ func luhnValid(s string) bool {
 var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
 
 func TestShippedStreetIsARegisteredName(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
@@ -397,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
 func TestShippedLastNameComposition(t *testing.T) {
 	// last is a choice of patronymic {first}sson templates, compound
 	// {first}{last} templates and literal surnames.
-	f := newGenerator(t, "data", WithSeed(6))
+	f := shipped(t, WithSeed(6))
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
 			t.Fatalf("last name %q is not a Swedish surname", s)
@@ -407,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
 
 func TestShippedStreetNumberFormats(t *testing.T) {
 	// Reachable via a hyphenated path; covers all five weighted number variants.
-	f := newGenerator(t, "data", WithSeed(8))
+	f := shipped(t, WithSeed(8))
 	re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
 	for i := 0; i < 300; i++ {
 		if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
@@ -419,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
 // TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
 // that the everyday titles are reachable.
 func TestShippedUSTitleAgreesWithSex(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(4))
+	f := shipped(t, WithSeed(4))
 	female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
 	seen := map[string]bool{}
 	for i := 0; i < 3000; i++ {
-		got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`)
+		got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
 		title, sex, _ := strings.Cut(got, "|")
 		title = strings.TrimSpace(title)
 		seen[title] = true

datatype.go

@@ -3,97 +3,83 @@ package fejkdata
 import (
 	"errors"
 	"fmt"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
-// DataType is what a record column holds, which decides how a record writes its value.
+// DataType is what a record column holds, which decides how a record writes its value. It
-type DataType int
+// prints as data spells it: string, integer, number or boolean.
+type DataType = datatype.DataType
 
 // The datatypes a column declares with "datatype"; a column without one is a string.
 const (
-	DataTypeString DataType = iota
+	DataTypeString  DataType = datatype.String
-	DataTypeInteger
+	DataTypeInteger DataType = datatype.Integer
-	DataTypeNumber
+	DataTypeNumber  DataType = datatype.Number
-	DataTypeBoolean
+	DataTypeBoolean DataType = datatype.Boolean
 )
 
-var (
+// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
-	dataTypeNames = [...]string{"string", "integer", "number", "boolean"}
+func datatypeOf(m map[string]any, pos position) (*DataType, error) {
-	dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
-)
-
-// String is the datatype as data spells it.
-func (d DataType) String() string {
-	if d < 0 || int(d) >= len(dataTypeNames) {
-		return fmt.Sprintf("DataType(%d)", int(d))
-	}
-	return dataTypeNames[d]
-}
-
-// position is where a JSON value sits, which decides whether it may carry a datatype or
-// be null.
-type position int
-
-const (
-	inFormat position = iota // rendered by a format, so neither
-	atTop                    // a category or an inline template, whose fields may be columns
-	inColumn                 // a column, or a choice item standing in for one
-)
-
-// datatypeOf reads a template's "datatype" (default DataTypeString).
-func datatypeOf(m map[string]any, pos position) (DataType, error) {
 	v, ok := m["datatype"]
 	if !ok {
-		return DataTypeString, nil
+		return nil, nil
 	}
 	name, ok := v.(string)
 	if !ok {
-		return 0, fmt.Errorf("datatype must be a string, got %T", v)
+		return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
 	}
-	if name == DataTypeString.String() {
+	for d := DataTypeString; d < datatype.Count; d++ {
-		return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
-	}
-	for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
 		if name != d.String() {
 			continue
 		}
-		if pos != inColumn {
+		if !pos.column() {
-			return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here")
+			return nil, nil
 		}
-		return d, nil
+		return &d, nil
 	}
-	return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name)
+	return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
 }
 
 // checkColumns rejects a record column whose items hold different datatypes, checking a column
 // after the columns its items read, so a column read is named before its readers.
-func checkColumns(s nodeScope) error {
+func checkColumns(s nodeSet) error {
 	checked := map[node]bool{}
-	var check func(path, name string, column node) error
+	var check func(label, name string, column node) error
-	check = func(path, name string, column node) error {
+	check = func(label, name string, column node) error {
 		if checked[column] {
 			return nil
 		}
 		checked[column] = true
 		items, _ := columnItems(column)
 		for _, it := range items {
-			if r := it.readsColumn; r != nil {
+			if r := it.readsColumn; r != nil && r.category != "" {
-				if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil {
+				if err := check(r.category, r.field, r.column); err != nil {
 					return err
 				}
 			}
 		}
-		if _, err := columnDatatype(column); err != nil {
+		if len(items) == 0 {
-			return fmt.Errorf("%s: field %q: %w", path, name, err)
+			return nil
+		}
+		first := columnDatatype(column)
+		for _, it := range items[1:] {
+			if d := itemDatatype(it); d != first {
+				return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
+			}
 		}
 		return nil
 	}
-	return s(func(path string, n node) error {
+	return s(func(label string, n node) error {
 		t, ok := n.(*template)
-		if !ok || !t.record {
+		if !ok || !t.isRecord {
 			return nil
 		}
-		for _, name := range recordColumns(t) {
+		for _, name := range sortedNames(t.fields) {
-			if err := check(path, name, t.fields[name]); err != nil {
+			if err := check(label, name, t.fields[name]); err != nil {
 				return err
 			}
 		}
@@ -101,41 +87,25 @@ func checkColumns(s nodeScope) error {
 	})
 }
 
-// columnDatatype is the datatype a column's items hold. They must agree, since a
+// columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
-// column holds one; a column only ever null is a string.
+// a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
-func columnDatatype(n node) (DataType, error) {
+func columnDatatype(n node) DataType {
 	items, _ := columnItems(n)
 	if len(items) == 0 {
-		return DataTypeString, nil
+		return DataTypeString
 	}
-	first := itemDatatype(items[0])
+	return itemDatatype(items[0])
-	for _, t := range items[1:] {
-		if d := itemDatatype(t); d != first {
-			return first, disagreement(items[0], first, t, d)
-		}
-	}
-	return first, nil
 }
 
-// itemDatatype is the datatype a column item declares, else that of the column it is.
+// itemDatatype is the datatype a column item declares, else that of the column it reads.
 func itemDatatype(t *template) DataType {
-	if t.datatype != DataTypeString {
+	switch {
-		return t.datatype
+	case t.datatype != nil:
+		return *t.datatype
+	case t.readsColumn != nil:
+		return columnDatatype(t.readsColumn.column)
 	}
-	return readDatatype(t)
+	return DataTypeString
-}
-
-// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
-// when t reads none.
-func readDatatype(t *template) DataType {
-	if t.readsColumn == nil {
-		return DataTypeString
-	}
-	items, _ := columnItems(t.readsColumn.column)
-	if len(items) == 0 {
-		return DataTypeString
-	}
-	return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
 }
 
 // columnItems is a column's template items, its choices unwrapped, and whether one is null.
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
 			}
 		case *template:
 			items = append(items, n)
-		case *null:
+		case *nullItem:
 			nullable = true
+		case *tableColumn:
+		default:
+			panic(invariant.Broken("columnItems has no case for node %T", n))
 		}
 	}
 	collect(n)
 	return items, nullable
 }
 
-// disagreement names the fix for two items of one column holding different datatypes.
+// itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
-func disagreement(a *template, da DataType, b *template, db DataType) error {
+type itemKind int
-	bare, typed, want := b, a, da
+
-	if da == DataTypeString {
+const (
-		bare, typed, want = a, b, db
+	kindText itemKind = iota
-	}
+	kindReads
+	kindDeclares
+)
+
+func kindOf(t *template) itemKind {
 	switch {
-	case da != DataTypeString && db != DataTypeString:
+	case t.datatype != nil:
-		return bothTyped(a, da, b, db)
+		return kindDeclares
-	case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "":
+	case itemDatatype(t) != DataTypeString:
-		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed))
+		return kindReads
-	case bare.fromString: // an object may carry a weight, which this spelling would drop
-		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
 	}
-	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)
+	return kindText
 }
 
-// bothTyped names the fix for two items holding different datatypes: the one taking its datatype
+// disagreement names the fix for items a and b of one column holding different datatypes, fixing
-// from the column it reads is typed as the other where its values prove it, else read as text.
+// the one that declares least, a when both declare alike.
-func bothTyped(a *template, da DataType, b *template, db DataType) error {
+func disagreement(a *template, da DataType, b *template, db DataType) error {
-	read, other := a, db
+	c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
-	if a.datatype != DataTypeString {
+	if kindOf(b) < kindOf(a) {
-		read, other = b, da
+		c.fix, c.other, c.want = b, a, da
 	}
-	switch {
+	message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
-	case read.datatype != DataTypeString:
+	if !ok {
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
+		panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
-	case (&valueProof{}).columnItem(read).not[other] == "":
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
 	}
-	return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read))
+	return message(c)
+}
+
+// clash is two items of one column holding different datatypes: other holds want; held says
+// what each holds.
+type clash struct {
+	fix, other *template
+	want       DataType
+	held       string
+}
+
+// disagreements is a clash's message by the kinds of its items, fix's first.
+var disagreements = map[[2]itemKind]func(clash) error{
+	{kindDeclares, kindDeclares}: clash.bothDeclare,
+	{kindReads, kindReads}:       clash.retypeRead,
+	{kindReads, kindDeclares}:    clash.retypeRead,
+	{kindText, kindReads}:        clash.textBesideRead,
+	{kindText, kindDeclares}:     clash.declareText,
+}
+
+func (c clash) bothDeclare() error { return errors.New(c.held) }
+
+// fits reports whether fix may hold want.
+func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
+
+func (c clash) retypeRead() error {
+	if c.fits() {
+		return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
+	}
+	return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
+}
+
+func (c clash) textBesideRead() error {
+	if !c.fits() {
+		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
+	}
+	return c.declareText()
+}
+
+func (c clash) declareText() error {
+	if c.fix.fromString { // an object may carry a weight, which this spelling would drop
+		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
+	}
+	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
 }
 
 // typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
 	return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
 }
 
-// asText names the spelling that reads the column a column-read item reads as text, keeping the
+// columnRead is a record's column read by a format that only reads one reference or name,
-// other keys an object item carries.
+// which is the column: it takes the column's datatype and null. category and field name it;
-func asText(t *template) string {
+// category is "" for a column of the reading template's own record, which checkColumns reaches
-	if t.fromString {
+// on its own.
-		return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format)
+type columnRead struct {
-	}
+	a               arm
-	return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format)
+	category, field string
+	column          node
+}
+
+// columnReadOf is the record's column t's format reads, where the format only reads one reference
+// or name, and nil where it does not or t declares a string, which reads the column as text.
+func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
+	ops := t.compiled.ops
+	if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
+		return nil
+	}
+	a := ops[0].arms[0]
+	var start node
+	var category string
+	var tail []string
+	switch b := a.named; {
+	case a.kind == namedRead && b.bindsField():
+		// {f as n}{n}: column f of the record binding n, in t's own category.
+		start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
+	case a.kind == namedRead:
+		// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
+		start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
+		tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
+	case grammar.IsRef(a.head):
+		// {/c.x}: column x of category c.
+		start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
+	default:
+		return nil
+	}
+	if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
+		return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
+	}
+	return nil
 }

datatype_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 )
 
-func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
+func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
 	for _, src := range []string{
 		`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
 		`{"format":"","n":{"format":"42","datatype":"integer"}}`,
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 		`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
 		`{"format":"","pick":[[null,"a"],"b"]}`,
 	} {
-		if _, err := compile(parse(t, src)); err != nil {
+		if _, err := resolved(t, src); err != nil {
 			t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
 		}
 	}
 	for src, want := range map[string]string{
-		`{"format":"","n":{"format":"1","datatype":"int"}}`:                             `datatype takes "integer", "number" or "boolean", got "int"`,
+		`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
-		`{"format":"","n":{"format":"1","datatype":1}}`:                                 "datatype must be a string",
+		`{"format":"","n":{"format":"1","datatype":1}}`:     "datatype must be a string, not a number",
-		`{"format":"{int(1,9)}","datatype":"integer"}`:                                  "datatype only types a record column",
-		`[{"format":"1","datatype":"integer"},"x"]`:                                     "datatype only types a record column",
-		`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
-		`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`:           "datatype only types a record column",
-		`null`: `so write ""`,
-		`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
 		}
 	}
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 	tree := map[string]string{
 		"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
-		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
 	}
 	for _, c := range []struct{ name, column, want string }{
 		{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
 		{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
-		{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
 		{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
 		{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
-		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
-		{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
 		{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
-		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
 		{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
-		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`},
+		{"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
+		{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
+		{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
+		{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
 		{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
 		{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
 		{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
 		{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
 		{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
 		{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
-		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"},
+		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
 		{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
 		{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
 		{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
-		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"},
+		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
 		{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
 		{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
-		{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
-		{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
 	} {
 		row := `{"format":"","col":` + c.column + `}`
 		files := map[string]string{"row": row}
 		for name, body := range tree {
 			files[name] = body
 		}
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
+		_, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
 		if err == nil || !strings.Contains(err.Error(), c.want) {
 			t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
 		}
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 			t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
 		}
 	}
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
+	for column, want := range map[string]string{
+		`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
+		`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`:   "its items hold integer and number; a column holds one datatype",
+	} {
+		row := `{"format":"","col":` + column + `}`
+		if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
+		}
+		if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
+		}
+	}
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
 		"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
 		"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
 	})))
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
 	} {
 		row := `{"format":"","col":` + column + `}`
 		src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
-		f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
+		f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
 		if err != nil {
 			t.Errorf("%s: New = %v, want it loaded", column, err)
 			continue
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
 		t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
 	}
 }
+
+func TestDisagreementRefusesTwoTextItems(t *testing.T) {
+	mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
+}
+
+func TestASignedZeroIsZero(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
+	r, err := f.FakeRecord("row")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
+		t.Errorf("columns = %+v, want each written as its format", c)
+	}
+}
+
+func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{
+		"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
+	}), WithSeed(1))
+	empty := 0
+	for i := 0; i < 50; i++ {
+		r, err := f.FakeRecord("row")
+		if err != nil {
+			t.Fatal(err)
+		}
+		same, text := r.Columns()[0], r.Columns()[1]
+		if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
+			t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
+		}
+		if text.Value == "" {
+			empty++
+		}
+	}
+	if empty == 0 {
+		t.Error(`50 records never read a null as "", want the string column to render one so`)
+	}
+}

doc.go

@@ -0,0 +1,46 @@
+// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
+//
+// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
+// or a folder. A generator loads the modules you name, such as data.Modules(), every
+// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
+// become a dot-path namespace, and you generate values by path:
+//
+//	f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
+//	f.Fake("sv_SE.address")          // "Trekungagatan 43\n442 42 Kungälv"
+//	f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
+//
+// Several sources merge in order, the last winning a name clash, so custom data
+// layers over the built-ins. The JSON template format is documented in the [README].
+//
+// [README]: https://github.com/larvit/fejkdata#readme
+package fejkdata
+
+// Vocabulary
+//
+//   - draw — one item taken from a choice, or one row from a table: `drawItem`,
+//     `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
+//   - expansion — one render of one format: `expand`.
+//   - render — one walk of the tree from an entry point, over one `frameStack`:
+//     `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
+//     start one each, and `FakeStruct` one per record.
+//   - env — what a render reads names and rows through: `renderEnv`.
+//   - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
+//   - named pick — the draw a token such as {/person as p} binds to a name, drawn
+//     on its first read and kept while its scope renders: `nameBinding`,
+//     `namedPick`, `pickFrame`.
+//   - level, pick key, addressed key — `pickKey`.
+//   - memo — what a named pick keeps its draws in, the variant drawn at each level
+//     and the value each path read: `drawMemo`.
+//   - pin — fixing which row of a table one path, or one named pick, uses:
+//     `pinSet`, `namedPick.pins`, `table.followStep`.
+//   - link — a table's tie to its parent table, each row to its parent row:
+//     `table.linkParent`.
+//   - family — a table and every table reaching it through a chain of parent
+//     links.
+//   - resolve — tying a reference or a name to what it names in the assembled tree:
+//     `resolveTemplates`, `resolveRefs`.
+//   - head — the string a read starts from: a field or a name as written, or a
+//     reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
+//     the read's start: `template.startOf`, `nameTarget.start`.
+//   - fence — a load-time check, run over each category, template or struct as it
+//     loads or compiles, so rendering a compiled tree cannot fail.

doc_test.go

@@ -0,0 +1,286 @@
+package fejkdata
+
+import (
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"os"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Every backtick in the Vocabulary is a symbol.
+var backticked = regexp.MustCompile("`([^`\n]+)`")
+
+// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
+func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
+	_, files := sourceFiles(t)
+	declared := declaredSymbols(files)
+	if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
+		t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
+	}
+}
+
+// namesDeclared is every symbol text backticks, failing for each one the package
+// declares nowhere.
+func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
+	t.Helper()
+	var named []string
+	for _, m := range backticked.FindAllStringSubmatch(text, -1) {
+		if !declared[m[1]] {
+			t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
+		}
+		named = append(named, m[1])
+	}
+	return named
+}
+
+// namesAPass fails unless one symbol text backticks is a function: a field names
+// what a pass reads or writes, never the pass.
+func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
+	t.Helper()
+	named := namesDeclared(t, where, text, declared)
+	if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
+		t.Errorf("%s names no function, so it names no pass", where)
+	}
+}
+
+func packageDir(t *testing.T) *build.Package {
+	t.Helper()
+	pkg, err := build.ImportDir(".", 0)
+	if err != nil {
+		t.Fatalf("read the package directory: %v", err)
+	}
+	return pkg
+}
+
+func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	return parseFiles(t, packageDir(t).GoFiles)
+}
+
+func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	fset := token.NewFileSet()
+	var files []*ast.File
+	for _, name := range names {
+		f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
+		if err != nil {
+			t.Fatalf("parse %s: %v", name, err)
+		}
+		files = append(files, f)
+	}
+	return fset, files
+}
+
+func vocabulary(t *testing.T, files []*ast.File) string {
+	t.Helper()
+	var sections strings.Builder
+	for _, f := range files {
+		for _, c := range f.Comments {
+			section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
+			if found && c.Pos() > f.Package {
+				sections.WriteString(section)
+			}
+		}
+	}
+	if sections.Len() == 0 {
+		t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
+	}
+	return sections.String()
+}
+
+// declaredSymbols is every name the package declares at the top level, a method and
+// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
+func declaredSymbols(files []*ast.File) map[string]bool {
+	names := declaredFuncs(files)
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			if d, isGen := decl.(*ast.GenDecl); isGen {
+				addSpecs(names, d)
+			}
+		}
+	}
+	return names
+}
+
+// declaredFuncs is every function the package declares, a method keyed under its
+// type: template.fixedText.
+func declaredFuncs(files []*ast.File) map[string]bool {
+	names := map[string]bool{}
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isFunc := decl.(*ast.FuncDecl)
+			if !isFunc {
+				continue
+			}
+			names[d.Name.Name] = true
+			if recv := receiverType(d.Recv); recv != "" {
+				names[recv+"."+d.Name.Name] = true
+			}
+		}
+	}
+	return names
+}
+
+func addSpecs(names map[string]bool, d *ast.GenDecl) {
+	for _, spec := range d.Specs {
+		switch s := spec.(type) {
+		case *ast.TypeSpec:
+			names[s.Name.Name] = true
+			addFields(names, s)
+		case *ast.ValueSpec:
+			for _, id := range s.Names {
+				names[id.Name] = true
+			}
+		}
+	}
+}
+
+func addFields(names map[string]bool, s *ast.TypeSpec) {
+	st, isStruct := s.Type.(*ast.StructType)
+	if !isStruct {
+		return
+	}
+	for _, field := range st.Fields.List {
+		for _, id := range field.Names {
+			names[s.Name.Name+"."+id.Name] = true
+		}
+	}
+}
+
+func receiverType(recv *ast.FieldList) string {
+	if recv == nil || len(recv.List) == 0 {
+		return ""
+	}
+	e := recv.List[0].Type
+	if star, isPointer := e.(*ast.StarExpr); isPointer {
+		e = star.X
+	}
+	id, isIdent := e.(*ast.Ident)
+	if !isIdent {
+		return ""
+	}
+	return id.Name
+}
+
+const fillHeader = "Filled by "
+
+// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
+// fills, and to the template doc.
+func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
+	fset, files := sourceFiles(t)
+	declared, funcs := declaredSymbols(files), declaredFuncs(files)
+	doc, st, headers := templateStruct(t, fset, files)
+	namesAPass(t, "the template doc", doc, declared, funcs)
+	for _, field := range st.Fields.List {
+		if len(field.Names) == 0 {
+			t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
+		}
+		for _, id := range field.Names {
+			header := headerAbove(headers, id.Pos())
+			if header == "" {
+				t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
+				continue
+			}
+			namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
+		}
+	}
+}
+
+// templateStruct is the template struct, its doc, and the fill headers standing
+// between its fields, in source order.
+func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
+	t.Helper()
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isGen := decl.(*ast.GenDecl)
+			if !isGen {
+				continue
+			}
+			for _, spec := range d.Specs {
+				s, isType := spec.(*ast.TypeSpec)
+				if !isType || s.Name.Name != "template" {
+					continue
+				}
+				st, isStruct := s.Type.(*ast.StructType)
+				if !isStruct {
+					t.Fatal("template is not a struct")
+				}
+				return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
+			}
+		}
+	}
+	t.Fatal("the package declares no template struct")
+	return "", nil, nil
+}
+
+// fillHeaders skips a header trailing a field: it names that field, never the
+// fields below it.
+func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
+	trailing := map[int]bool{}
+	for _, field := range st.Fields.List {
+		trailing[fset.Position(field.End()).Line] = true
+	}
+	var found []*ast.CommentGroup
+	for _, c := range f.Comments {
+		if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
+			continue
+		}
+		if strings.HasPrefix(c.Text(), fillHeader) {
+			found = append(found, c)
+		}
+	}
+	return found
+}
+
+// headerAbove is the last fill header standing above pos.
+func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
+	text := ""
+	for _, c := range headers {
+		if c.Pos() < pos {
+			text = c.Text()
+		}
+	}
+	return text
+}
+
+var (
+	printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
+	printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
+)
+
+// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
+func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
+	for _, name := range []string{"doc.go", "README.md"} {
+		src, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines := printedCall.FindAllString(string(src), -1)
+		if lines == nil {
+			t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
+		}
+		f, err := New(withShipped(), WithSeed(42))
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, line := range lines {
+			c := printedFake.FindStringSubmatch(line)
+			if c == nil {
+				t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
+			}
+			want, err := strconv.Unquote(c[2])
+			if err != nil {
+				t.Fatalf("%s prints %s: %v", name, c[2], err)
+			}
+			if got := fake(t, f, c[1]); got != want {
+				t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
+			}
+		}
+	}
+}

docs/comprehension-history.md

@@ -0,0 +1,684 @@
+# Comprehension history
+
+## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
+- `datatype.go:217` `columnRead`: beside the code that reads it
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
+- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
+- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
+
+## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
+- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
+- `node.go:70` `template.readsColumn`: names the four readers that change together
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
+- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
+
+## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
+- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
+- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
+- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
+- `path.go:119` `compileStep`: six parameters, the one place that reads harder
+
+## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
+- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
+- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
+- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
+- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
+
+## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
+- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
+- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
+- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
+- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
+
+## 2026-10-07, scoring run at ad59967
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
+
+## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
+| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
+| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
+| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
+
+Junior A, hardest first:
+
+- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
+- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
+
+Mid A, hardest first:
+
+- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
+- `node.go:70` `readsColumn`: four functions in three files must change together
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
+- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
+
+Inherited architect, would restructure first:
+
+- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
+- `node.go:52` `template`: filled in four phases by four functions
+
+## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
+- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
+- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
+
+Inherited architect, decided by:
+
+- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
+- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
+- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
+
+## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
+- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
+
+Inherited architect, decided by:
+
+- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
+- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
+- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
+- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
+
+## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
+- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
+- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
+- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
+- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
+- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
+
+## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
+- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
+- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
+- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
+- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
+- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
+- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
+
+## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
+- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
+- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
+- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
+
+Inherited architect, decided by:
+
+- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
+- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
+- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
+
+## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | better | same | better | better |
+| Junior A | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
+- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
+- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
+- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
+- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
+
+Junior A, decided by:
+
+- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
+- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
+- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
+
+## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
+- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
+- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
+- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
+- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
+- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
+- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
+- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
+- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
+- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
+
+## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | worse | worse | better | same | worse |
+| Maintainability senior | worse | same | worse | same | worse |
+| Junior A | same | worse | same | same | same |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
+- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
+- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
+- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
+- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
+- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
+- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
+- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
+
+Maintainability senior, decided by:
+
+- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
+- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
+- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
+- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
+
+Junior A, decided by:
+
+- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
+- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
+
+## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | worse | same | worse |
+| Inherited architect | worse | same | worse | same | worse |
+
+Mid A, decided by:
+
+- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
+- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
+- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
+- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
+
+Inherited architect, decided by:
+
+- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
+- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
+- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
+- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
+- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
+
+## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | worse | worse | same | worse |
+| Inherited architect | worse | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
+- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
+- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
+- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
+- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
+
+## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
+
+Ruling: worse
+
+Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | better | same | same | same |
+| Inherited architect | worse | same | same | same | same |
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6 | 6 | 6 |
+| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
+| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
+- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
+- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
+- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
+
+Inherited architect, decided by:
+
+- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
+- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
+- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
+
+## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | better | better | same | better | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
+- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
+
+Inherited architect, decided by:
+
+- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
+- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
+- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
+- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
+
+## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | same | worse | worse | same | worse |
+| Maintainability senior | same | worse | same | same | worse |
+| Junior A | same | worse | worse | better | worse |
+
+Mid A, decided by:
+
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
+- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
+- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
+- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
+- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
+- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
+- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
+- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
+- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
+
+Junior A, decided by:
+
+- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
+- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
+
+## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
+- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
+- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
+- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
+- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
+- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
+- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
+- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
+
+## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
+- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
+- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
+- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
+- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
+- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
+
+## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | better | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
+- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
+- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
+- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
+- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
+- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
+- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
+
+Inherited architect, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
+- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
+- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
+
+## 2026-10-12T16:30:06Z, PR #216 at 52e0304
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
+| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
+| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
+
+Junior A, hardest first:
+
+- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
+- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
+- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
+- `node.go:65` `template`: filled in four phases by different files, with a change-together set
+- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
+
+Mid A, hardest first:
+
+- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
+- `render.go:146` `readField`: four read paths, and the name undersells it
+- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
+- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
+- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
+- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
+- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
+- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
+- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
+
+Inherited architect, would restructure first:
+
+- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
+- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
+- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
+- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
+- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
+
+## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | better | worse | same | same | better |
+| Inherited architect | better | worse | same | same | same |
+| Maintainability senior | better | worse | same | same | same |
+| Junior A | better | worse | same | better | better |
+
+Mid A, decided by:
+
+- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
+- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
+- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
+
+Inherited architect, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
+- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
+- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
+- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
+
+Maintainability senior, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
+- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
+- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
+
+Junior A, decided by:
+
+- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
+- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
+- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
+- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
+
+## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |

docs/decisions.md

@@ -0,0 +1,991 @@
+# Decisions
+
+## GitHub is canonical, and the module path names it
+
+2026-09-20, Lilleman auf Larv.
+
+Goal 1 wants the usage the tool earns and goal 3 expects extenders who did not write it;
+both need a stranger to file an issue and open a pull request. Gitea has no anonymous
+issue, and no cross-host pull request at all, so a contributor would need an account and
+a fork on a personal instance. Valid while the project wants contribution from outside.
+
+## The Gitea copy stays, as a pull mirror of GitHub
+
+2026-10-03, decided by the maintainer. Technical principle: one owner per value
+(`~/.claude/principles/technical-principles.md`). GitHub owns the history, and
+`gitea.larvit.se/larvit/fejkdata` stays as a read-only pull mirror of it, so it is never
+a second owner. Until `todo.md` item 68 converts it, the Gitea copy goes stale. Valid
+while the maintainer keeps a Gitea copy.
+
+## The vocabulary sits below `doc.go`'s package clause, not in the package doc
+
+2026-09-21, Lilleman auf Larv.
+
+Goal 1 is what a developer choosing a library reads first, and the package doc is that
+page: the glossary would fill two thirds of it with the render path's units, mostly
+unexported, and Go doc comments have no code markup, so every name would render its
+backticks literally. Below the clause it reaches goal 3's reader — who opens the file —
+and nothing else. Valid while the glossary's subject is the render path; one written to
+teach the exported API belongs above the clause.
+
+## Options and fields share one namespace
+
+2026-09-02, Lilleman auf Larv.
+
+`format`, `weight`, `repeat`, `separator` and `datatype` are reserved; every other key
+is a field. Nesting fields under a key, or prefixing options, would tax every template to
+guard against a misspelt option.
+
+## `{a|b}` stays beside nested choices
+
+2026-09-02, Lilleman auf Larv.
+
+`[[…], […]]` picks the same way, but its arms are anonymous; `{female|male}` keeps
+`person.female` addressable.
+
+## Flags follow getopt_long
+
+2026-09-02, Lilleman auf Larv.
+
+`--name value` and `--name=value` both work; a short flag's value attaches or follows
+(`-s42`, `-s 42`) and short flags bundle (`-hn 3`), as every shell user expects. A
+single-dash long flag is rejected naming the double-dash spelling, and `-s=42` is
+rejected naming both short spellings: `=` belongs to the long form, and reading `=42` as
+the value would make `-d=./x` a directory named `=./x`.
+
+## A struct tag is a template by its shape
+
+2026-09-03, Lilleman auf Larv.
+
+A JSON object, array or string, or a string carrying a `{` token, is an inline template;
+anything else is a path. A name may not contain a brace, a bracket or a quote, so a path
+can never collide with any of those spellings, and the leading `[` or `"` is gated on
+valid JSON so a stray copied bracket never swallows a tag — it names nothing, and says
+so. Reserving the characters whole — though only a leading one
+could collide — keeps one simple name rule instead of a leading-position special case.
+A JSON string stays a template even where its bare text renders alike, `"x {/a}"` beside
+`x {/a}`: inside the quotes, `\t` and `\n` let a one-line template write a tab or a newline. A
+folder-relative `{.name}` or `{..name}` is refused naming
+`{/name}`, since an inline template sits in no folder. `IsTemplate` exports the rule, so
+struct tags and any other caller read one.
+
+## The CLI renders exactly the template on its stdin, and adds nothing
+
+2026-10-07, Lilleman auf Larv. Serves goals 7, 7.1 and 7.2.
+
+The template comes from stdin, from `echo`, a file or a heredoc, so one command line gets
+a value and a quoted heredoc needs no escape. A format string is stdin's every byte, and
+fejkdata adds no newline of its own under `--format text`: `echo` ends each render with its newline,
+`printf '%s'` sends none, and `-n` joins renders with `--separator`, empty by default. A
+bench of the README's personas picked this on 2026-10-07 over dropping a newline or
+adding one: a rule a user states in one sentence, at the cost of `printf` or a
+`--separator` now and then.
+
+Stdin holding only whitespace is a format string, and renders as sent.
+
+A lone reference, `{/users}` or `"{/users}"`, names the record `users` under `--format`,
+one newline after it allowed, with `users` the default `--table`; one to nothing there
+exits 1. `todo.md` item 129 moves that reading into the library; until then it works
+against goal 3.2.
+
+The CLI takes no argument but its flags: one is misuse naming the spelling that reads what
+it likely means, `fejkdata < FILE` for a file's name, dropping it for `-`, and the template
+piped in otherwise; so is a terminal with nothing piped in, which would otherwise wait for
+typed input. `--list` prints paths, each read as `{/path}`. Valid while the CLI reads one
+template per run.
+
+## The CLI reads only the library's public API
+
+2026-10-07, Lilleman auf Larv; revised 2026-10-11 to name package `cli`. Serves goal 9.6: a
+CLI carrying your own modules is a few lines of Go over `cli.Main` and works just like
+`fejkdata`. `cli`'s non-test code imports nothing under `internal/`, so whatever the CLI
+does, a program of your own can do through the public API. Until `todo.md` item 129 moves
+the lone-reference check into the library, the CLI copies how the grammar tells a lone reference, in `isPath`,
+`hasEmptySegment` and `withoutSelectors`, and a test pins the copy to the grammar.
+
+## A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+
+2026-09-24, Lilleman auf Larv.
+
+The whole template is the spelling under test, and `NewTemplate` compiles, links and
+validates as one step. Under the CLI's `--format`, a lone reference to nothing exits 1
+until item 129: it reads a path, and only the data is absent.
+
+## Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+
+2026-10-07, Lilleman auf Larv. Applies KISS.
+
+Padding is where the two readings disagree: a format string renders it, JSON drops it. A
+template that parses as a JSON object, array, string or `null` is its JSON, so its padding
+is dropped, as `encoding/json` drops it. `null` stays JSON, since it is how a template
+renders nothing. A bare number or boolean gains nothing from the JSON reading, so it is a
+format string, and `echo 42` prints its newline as `echo x` does (goal 5.1).
+
+## `FakeTemplate` and `NewTemplate` both stay
+
+2026-09-03, Lilleman auf Larv.
+
+They reach the same value but not at the same cost: `NewTemplate` pays the compile and
+validation once and renders many times, `FakeTemplate` is the one-shot call, and
+`--repeat` is exactly the case that needs the first. The pair is `regexp.MustCompile`
+and `regexp.Match`, not two spellings of one result.
+
+## The shipped data is embedded, not discovered
+
+2026-09-02, Lilleman auf Larv. Serves goal 2: nothing is fetched at run time.
+
+A directory a machine happens to have would make `--seed 42` machine-dependent;
+`--data-path` layers over the embedded packages.
+
+## The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+
+2026-10-04, Lilleman auf Larv; revised 2026-10-09 to put the packages in the core's Go module. Serves goals 2.2, 6.2, 7.3, 9 and 10.1. Carried out by `todo.md` items 181, 186 and 196, all in 0.2.0. The CLI binary may grow to hundreds of MB.
+
+- One Go package per locale, one per country's `geo` tree, and one for `misc`, all in the core's Go module. A package registers nothing when imported.
+- A bare `New()` loads no data and fails, naming the option to add. `WithoutShippedData` goes.
+- A data package's `FS` is an `fs.FS` passed to `WithDataFS`, as anyone's data is, and `New` takes several. It is an exported `var FS embed.FS`, the shape `//go:embed` gives; a function returning it would only stop a program reassigning the FS variable of a package it imports. A module carrying functions loads through an option of its own, so a data module that starts carrying functions breaks its users. Such a function reaches the randomness through an interface the root declares; `internal/drawstate`'s type stays internal.
+- A module names, in a manifest, every module it reads by default, directly or through another, so the first error names the whole set to import; `todo.md` item 196 carries this out. A manifest is optional: a `--data-path` folder without one is a module. Nothing loads a default on its own.
+- A manifest names no fejkdata version, and `New` checks none: the shipped packages ship in the core's Go module, and a module from anyone else keeps loading across a minor or a patch, as the README's Versioning promises. The maintainer chose so on 2026-10-09.
+- In the CLI, a flag of its own says a `--data-path` replaces what it clashes with, so flags still go anywhere on the line. Data authors and hand fixture authors bench it.
+- The README asks for functions with no side effects of their own.
+- The CLI is a library call. One exported list names every shipped package, and `fejkdata`'s own `main` passes it to that call. The list lives in a package of its own, so a library importing the core links no data it did not import. A custom CLI is the same few lines with other modules added. No API and no flag exists only for custom CLIs; `Main` tells one apart only by its build info and file name, so its help and errors carry its name and its `--version` its own module's version beside fejkdata's. The maintainer chose so on 2026-10-11.
+- The help's examples and its hint for an empty stdin read no data, so each renders in any custom CLI, and both point to `--list` for the paths. "Shipped data", in `--no-shipped-data` and in `-d`'s help, names the modules passed to `Main`, a custom CLI's own included. The audience bench's custom-CLI builder seat ruled so on 2026-10-11, before 0.1.0.
+
+Valid while the shipped data stays under 100 MiB, which `TestShippedDataStaysUnder100MiB` checks: `go get` fetches the whole Go module, and the `go` command refuses a Go module zip over 500 MiB (`MaxZipFile` in `golang.org/x/mod/zip`). Past it, `todo.md` item 193 moves the largest packages into nested Go modules that keep their import paths, each requiring the core release that drops the package, so no build sees it twice.
+
+## A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+
+2026-10-11, Lilleman auf Larv, from two benches of a Go test author, a data author and a
+custom-CLI builder. Serves goals 1 and 3.3. `sv_SE` ships in `data/sv_SE` as package
+`svse`, `en_US` in `data/en_US` as `enus`, `geo.SE` in `data/geo/SE` as `geose`, `geo.US` in
+`data/geo/US` as `geous`, and `misc` in `data/misc`: Go linters flag a capital or an
+underscore in a package name, and `SE.FS` read as a type, while a folder spelling its
+namespace takes a reader from a dot path in an error or a template to its file. The second
+bench chose that over flat folders such as `data/svse`, 2 to 1, at the cost of a package
+name that differs from its folder. A locale is one folder, never a language folder holding
+regions: `en_US` and `en_GB` are different languages that share a name. Each package
+holds its files at its root, and its manifest's `namespace` puts them under the namespace,
+so no path repeats it. Valid until a locale's lowered name clashes with another's.
+
+## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+
+2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09.
+
+Goal 13: `New` pays nothing for a category a run never reaches. A manifest,
+`.fejkdata.json` at a source's root, may carry an index: each category's parent table and
+the paths `List` advertises below it. A category of an indexed source loads with every
+category it references and its whole table family, and binds as a whole load does, so
+it renders the same as after a whole load. `go generate` writes each shipped
+module's manifest; `TestShippedManifestsAreCurrent` fails while one is stale. CI loads
+the whole shipped set, and each shipped category alone into a fresh `New`, so none fails
+at first reach.
+
+Goal 6: a source without an index loads whole in `New`, with every indexed category it
+reads, so each mistake in it is `New`'s error. Goal 6.3: an indexed source from anyone
+else may fail at first reach; the call then returns the load's error, matching
+`ErrLoad`, the same every call. An indexed category reading one that another source
+replaced fails the same way, at its first reach, when the replacement breaks it. The
+maintainer chose that on 2026-10-10 over loading in `New` every indexed category
+depending on a replaced one: three rounds of the comprehension panel ruled that rule
+harder to follow (goal 3), and it needed every entry to list what its templates read.
+
+A first reach checks only a table's `parent` against its entry, so a table loads with its
+parent and a failed load puts back whole. Proving the rest of an index is its author's,
+with the tool `todo.md` item 194 plans.
+
+Valid while CI proves the shipped set whole and each category alone.
+
+## Every `{…}` draws afresh, and only a name keeps a pick
+
+2026-10-05, larv-review on the change the maintainer planned on 2026-10-03.
+Serves goals 5.2 and 4.1: each `{…}` makes a new pick, and values that belong together come
+from the one pick a name keeps.
+
+- `{p.a} {p.b}`, `{w} {uppercase(w)}`, `{/p.first} {/p.last}` and `{/p} {/p}` are two draws
+  each, and `{p as q}{q.a} {q.b}` is one. Nothing is kept per format or per render, so the
+  only load checks on draws are names, paths through names, cycles and the repeat cap.
+- `{x as n}` binds a reference, or a field of the template binding it or a path into one,
+  `{place as p}`: an inline template, and a choice of rows inside one category, have no reference to
+  bind, and goal 4.1 holds there too.
+
+Valid while goal 5.2 has each `{…}` make a new pick.
+
+## Reference sigils follow the filesystem
+
+2026-09-02, Lilleman auf Larv.
+
+`/` is the root, `.` this file's folder, `..` the folder above — what those spellings
+already mean to anyone who has typed a path. A locale's files reach each other without
+naming the locale, so a folder renames and copies without editing its references.
+
+## A change to what exists is a major; a minor only adds
+
+2026-09-16, Lilleman auf Larv.
+
+Data files, the CLI and the Go API are the public API, and a consumer must be able to
+take a minor without an edit — so an added column is a major, since it changes the CSV
+header and the `INSERT` column list, as is a removed value, which changes what a fixture
+holds, and a new option, which reserves a field name. A new refusal tightens what
+loads, so every new fence invalidates some file. Each such release names the rejected
+spelling and its replacement in the changelog and in the load error, and that is the
+whole migration: a fence rejects one spelling with one replacement, so the fix is local
+to each site. A fence that would need a non-local rewrite ships a converter with its
+release instead. Before `v1.0.0` a minor carries what a major would.
+
+## An error is a contract by what it names, not its bytes
+
+2026-09-16, Lilleman auf Larv.
+
+Goal 6. A script branches on the exit code and reads the named path or spelling, so
+those hold; wording improves in a minor.
+
+## A load reports every mistake at once
+
+2026-09-30, Lilleman auf Larv. Goal 6. An author learns everything wrong with the data
+in one load, not one mistake per run: `New` and `NewTemplate` report every mistake in one
+error, ordered by the path each names. A mistake following only from another, such as a
+reference into a category that did not compile, is left out, so each one names something
+to fix. Valid while a load checks the whole set before its first render.
+
+## Formats are checked after all data is loaded, so their errors name the category
+
+2026-09-28, larv-review; approved 2026-09-28 by lilleman. Goal 3. Loading reads every
+file first, then connects references across them. A format holding a reference can only
+be checked once it is connected, so every format is checked then, and all checks run at
+one moment. An error from those checks names the category path you would type (`sub.x`),
+or a table cell's category and line, not the file. Valid while checking a format needs
+its references connected.
+
+## Raising the lowest supported Go is a major
+
+2026-09-16, Lilleman auf Larv.
+
+A consumer building on it breaks, which is the one test every rule above applies; Go's
+convention of a minor is not followed.
+
+## The format check runs on the latest Go only
+
+2026-09-21, Lilleman auf Larv.
+
+`gofmt`'s output is the toolchain's, not the code's: 1.27 stopped padding a map
+literal's values out to a lone long key, so no source satisfies both it and 1.22's. A
+consumer on the lowest supported Go depends on the code compiling and its tests passing
+there, which is the `portable` stage, and never runs `gofmt` over this source, so the
+latest toolchain alone defines the one canonical form goal 3 asks a reader to meet.
+Valid while the lowest supported Go is not the latest.
+
+## The changelog heading is the one spelling of a release; CI cuts the tag
+
+2026-09-16, Lilleman auf Larv.
+
+A tag pushed by hand is served by `go get` at once, so a tag whose commit lacks its
+heading is burnt, not fixed. The heading on a gate-passed `main` commit is the trigger
+instead: the tag can land only there, and the GitHub release the same job publishes
+keeps one text as its body and is where prebuilt binaries will attach.
+
+## A `--data-path` override rebinds every reference to the category it replaces
+
+2026-09-24, Lilleman auf Larv.
+
+References bind against the merged tree, so once shipped data uses `{.person}`, a
+consumer's `sv_SE/person.json` is what every shipped reference into `person` reads, and
+`New` fails on shipped data the consumer never wrote when that file lacks a field those
+references read. Accepted: overriding is the point of layering, the error names the
+reference and the field, and the fix is the consumer's file carrying the fields the
+shipped tree reads.
+
+A `--data-path` category replaces a shipped one without saying so, against goal 9.3, until `todo.md` item 181 ends it.
+
+## The repeat cap bounds renders, not bytes
+
+2026-09-02, Lilleman auf Larv.
+
+A repeat, alone or nested, may ask for at most 1 048 576 renders; how large each render
+is stays what the data asked for, so `{hex(1048576)}` repeated to the cap is a terabyte,
+loaded without complaint. A byte estimate would need every builtin to declare a width to
+fence a shape no data comes near, and the harm lands on the author who wrote it.
+
+## 64-bit targets only
+
+2026-09-02, Lilleman auf Larv.
+
+The gate builds amd64, and the buffer sizing a render pre-computes (renders × bytes)
+assumes a 64-bit int; on a 32-bit target it could overflow and panic.
+
+## A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+
+2026-09-24, Lilleman auf Larv. Serves goal 6.1: a value that is always `NaN` or `Inf` is
+no valid value.
+
+`1/0` and a fixed `"0"` field are decidable, so they join the never-numeric operand as a
+load error; the fold stops where an operand varies, so `a/(b*c)` with `b` fixed at `0`
+and `c` varying loads and prints `Inf` every draw — catching it needs zero-absorbing
+algebra for a shape nobody writes. A [typed column](../README.md#datatype) bounds its operands
+instead and refuses a divisor it cannot keep from zero. In a string column, an operand that
+is sometimes not a number loads, and prints `NaN` on each draw where it is not one. `Inf`
+and `NaN` in a string column work against goals 6.1 and 4 until `todo.md` item 81
+refuses them there too.
+
+## Samples say what they emit, transforms what they do
+
+2026-09-02, Lilleman auf Larv.
+
+`{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both
+would turn on whether the argument looks like a number.
+
+## A record is a template seen as columns; a Go struct is the one second schema
+
+2026-09-15, Lilleman auf Larv.
+
+A template's `format` composes its fields into one string; `FakeRecord` and `--format`
+project the same fields as columns. Two views of one dataset, so a record author writes
+the same JSON they already know, and a column is the same field `Fake` renders by dotted
+path. The `format` is inert to a record — a record-only template writes `"format": ""` —
+but it is compiled and fenced, so a template that loads renders as whichever shape is
+asked for. `FakeStruct` takes its columns from a struct instead, because a Go caller has
+already written that schema: the fields name the columns and their types are the
+datatypes, so a tag says only what to draw: a `datatype` in it loads where it names the
+one the Go type sets, and is refused where it names another. The Go type is a struct column's one datatype, so its items need
+not agree on one among themselves; each must only hold that type.
+
+## A struct's records follow Go's field access, and compile on first use
+
+2026-09-15, Lilleman auf Larv.
+
+The fields an embedded struct promotes are the struct's own — `e.First`, as
+`encoding/json` and SQL mappers read them — so they are columns of its record and share
+its names: one tag may bind `{/person as p}{p.first}` and another read `{p.sex}` (goal 4.1),
+while a path tag draws afresh as `{/path}` does (goal 5.2).
+
+A tagged field that
+another field hides is refused, not dropped. A named
+struct field is another entity and a record of its own. `fake:"-"` leaves any struct
+field unfilled, so no name may be `-`; a pointer back to a struct
+already being filled is left alone, since filling it would never end. `New` cannot see a
+caller's types, so the first `FakeStruct` for a type compiles its tags and the answer,
+error included, is kept per type: a test's first call is its load, and no `NewStruct`
+handle is needed, as the cache already compiles once.
+
+## A category may reference itself, and a record's fences run at load
+
+2026-09-16, Lilleman auf Larv; revised 2026-10-07. Serves goals 5.2 and 5.7.
+
+A reference back into a category, `{/users.first}` inside `users`, is a fresh draw, as
+every `{…}` is; only a cycle is refused. A record's column fences run at `New` too, so a category that loads renders as a value and as a record alike.
+
+## A record's column set is fixed before the first draw
+
+2026-09-04, Lilleman auf Larv.
+
+Only a category-level template is a record: a path descending into a field, or naming a
+folder or a choice of more than one item, errors. A tail may pass through a choice whose variants carry
+different fields, so the columns — and with them the CSV header written once ahead of
+every row — would vary per draw. A fixed column set is what the CSV and `INSERT`
+contracts rest on, so the restriction holds even where a particular choice would happen
+to agree.
+
+## Null is a `null` item, not a rate
+
+2026-09-15, Lilleman auf Larv.
+
+A null is one more outcome of a column's draw, so a choice's weights skew it like any
+other; a null-rate option would be a second way to state odds. Outside a record column a
+null renders `""`.
+
+## A typed column holds one value, not composed text
+
+2026-09-15, Lilleman auf Larv. Serves goal 6.1: a typed column's values are proven valid at
+load, and composed text is not proven.
+
+Its bounds come from a literal or a call's arguments, so a load error names a real
+value and a range check is one comparison: `1{digits(2)}` is refused in a typed column,
+where `{int(100,199)}` states its range.
+
+## A column that only reads one reference or name is the column it reads
+
+2026-09-15, Lilleman auf Larv.
+
+`{/src.score}` renders exactly what `src.score` draws, and so do `{s.score}` where `s` is
+bound to `/src` and `{p}` where `p` is bound to the column `place` of its own record, so it
+takes that column's datatype and null. A `datatype` of its own, `string` included, wins
+over the one it reads: the one way to type a column someone else wrote.
+
+## A typed column's calc is refused unless proven
+
+2026-09-15, Lilleman auf Larv. Serves goal 6.1, read as: a template fails to load where
+the loader cannot prove it renders no forbidden value.
+
+Operand bounds must keep each divisor from zero and the result finite; what they cannot
+show is refused rather than trusted, since a bare `NaN` breaks the JSON and SQL it lands
+in.
+
+## `Column` carries text, not a Go value
+
+2026-09-15, Lilleman auf Larv.
+
+`Value` is the rendered string beside `DataType` and `Null`, which each serializer
+writes as the load check proved it; a `Value any` would hand every caller a type switch.
+
+## A unit takes the stem of what it is, and a file the stem of the units it holds
+
+2026-09-29, Lilleman auf Larv.
+
+Goal 3 wants a name to reach one unit: `drawMemo` is what a draw is kept in, `readValue` what
+a read drew.
+
+## One name, one meaning
+
+Two declarations of any kind — function, method, field, type — share a name only where one
+definition covers both: the same kind of value, counted or addressed the same way, so a
+reader landing on the wrong one concludes nothing false, as `Fake` renders one value on
+`Generator`, `Template` and `RecordTemplate`. `TestNoFunctionSpellsAMethod` keeps a bare function off a method's
+name, since a search returns both and a call site shows no receiver; review judges the
+rest. Serves goal 3: names that tell the truth, and reaching the unit behind a symptom
+without asking a person. Decided 2026-09-25 by Lilleman; valid while goal 3 counts reading
+cost.
+
+## The performance gate asserts allocations, not wall-clock time
+
+2026-09-03, Lilleman auf Larv. Goal 13.
+
+`AllocsPerRun` is deterministic across machines, so a 10% ceiling does not flake under
+CI load, while time varies with the machine and its neighbours. A rendering slowdown
+almost always costs an allocation too (a lost pre-size, a per-item map, an extra copy).
+The benchmark suite (the README's Development) reports time for a human, not as a pass/fail gate.
+
+A case guarding against one allocation per render, such as a render's scope reaching the
+heap through `pickKey.under`, holds at its baseline with no margin: from a baseline of
+10, a 10% margin lets that allocation through. Lilleman auf Larv decided it on 2026-10-06.
+
+## Rows live in a TSV, the shape in JSON
+
+2026-09-17, Lilleman auf Larv. Goals 8 and 13.
+
+`New` allocates once per node, so a register of thirty thousand rows written as JSON
+objects would cost it a second; a TSV is one allocation whose cells are substrings, and
+the JSON says only how a row is composed. The TSV sits beside its category file, named
+by `rows`, so a data directory stays a directory of categories, and one nothing names is
+ignored.
+
+## A selector is bracketed, and a dot inside it is literal
+
+2026-09-17, Lilleman auf Larv.
+
+`municipality[0180]` reads as selection to anyone who has indexed an array, and `[St.
+Louis]` keeps a name whole where a colon or a dot-separated spelling could not; zsh
+needs the brackets quoted, which the README's examples show.
+
+## `parent` names the link column and the table alike
+
+2026-09-17, Lilleman auf Larv.
+
+One word says both, so a child table sits in its parent's folder and links on a column
+of the parent's name; the geo plan wants exactly that, and a table needing another
+folder or another column name would be asking for a second spelling.
+
+## After a row, a path names a column or a linked table
+
+2026-09-17, Lilleman auf Larv.
+
+`locality[Lund].address`, a template beside the family under a selected row, is refused
+today; admitting it later is additive, since a refused spelling gains a meaning and no
+accepted one changes, so the door stays open for the address records the plan describes.
+
+## A selected row is fixed, not drawn
+
+Decided 2026-09-27 by the maintainer, for goals 4 and 5; valid while a selector names
+exactly one row.
+
+Each selector reads the row it names: `{/misc.territory[SE].capital} / {/misc.territory[FI].capital}`
+renders `Stockholm / Helsinki`, and two items of one choice may select different rows.
+
+## `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+
+2026-10-05, larv-review in PR #158. Serves goal 5.4, which has `..` go up one level, and goal
+5.2, which has each `{…}` make a new pick.
+
+- `..` names the parent table next, `city..country`, as a folder path names the folder it
+  enters. `.country` after a row stays the link column's cell.
+- A step down after a `..` draws afresh inside the row stepped up to (`drawSteps`). Under a
+  name, the paths stepping down from one `..` share that draw through the pick's memo
+  (`drawMemo.stepDownPins`), as every level a name's reads pass is shared, so two reads of
+  one step down describe one row.
+- A route through another row is another path to the same data, not a second spelling:
+  `city[Oslo]..country.name` and `country[NO].name` both load.
+
+## A table selects by one key column, a code, never a free-form name
+
+Decided 2026-09-27 by the maintainer, for goal 5.5: a name may repeat across rows or change
+with its register, while a code selects one row for good. Valid while
+the key is the code a user writes, as a database keys a table. `country[NO]` loads, and
+`country[Norway]` is refused naming `[NO]`, found by the row whose cell spells the
+selector. The key is the friendliest code the register holds, `misc.territory[SE]` and
+`misc.currency[EUR]`. There is no `name` option, and a table nothing selects into, a name
+table, `street`, `car` or `useragent`, carries no key and is only drawn.
+
+- `geo.US.locality` keys on the Census GEOID, `[1714000]` for Chicago.
+- `geo.SE.locality` keys on an integer the import assigns, since a postort has no code of
+  its own. The first import numbers the postorter in name order; later imports keep each
+  id by the postort's name, unique across Sweden as the key it replaces, give a new
+  postort the next unused integer, and retire a removed or renamed one's id for good.
+
+The code still takes a `name` option and selects by it; `todo.md` item 41 brings it to this decision.
+
+## A parent row with no child row is a load error
+
+2026-09-17, Lilleman auf Larv.
+
+A descendant is drawn inside the nearest pinned ancestor, so every ancestor row must
+lead to a row at every level below it, or a render could find nothing to draw. The
+import script drops or fills such rows; the alternative, falling back to a free draw,
+would break the consistency the link exists for without saying so.
+
+## A table is a record of string columns
+
+2026-09-17, Lilleman auf Larv.
+
+Its columns are the CSV header and the `INSERT` column list, fixed by the TSV header, so
+a table is a record by construction; every column is a string until a typed column
+option earns its place.
+
+## The key map is built at load, the rest on first draw
+
+2026-09-28, Lilleman auf Larv.
+
+A link is proved against the parent's keys and a key's uniqueness is a data mistake, so
+both are load-time; the row lookup, by name and by parent, serves only a draw, so it waits
+for the first one, keeping `New` linear in the bytes read. On 2026-10-05, building every
+shipped lookup at load added 13 ms and 3.2 MiB to a 60 ms `New`. That cost grows with every
+locale and every place the geo tables ship, and most tables are never drawn by name or parent. The maintainer
+kept building the lookup on first draw, once per table.
+
+## Two categories may name one TSV
+
+2026-09-17, Lilleman auf Larv.
+
+Each is a view of the file with its own format and options, at the cost of holding the
+rows twice, which is what a category over a register with two natural formats asks for.
+
+## A cell may read a reference, and each row its own
+
+2026-09-29, Lilleman auf Larv.
+
+Goal 4: facts that belong together come from one draw, so a shop row's `phone` cell reads
+`{/sv_SE.phone}` on a Swedish shop and `{/en_US.phone}` on a US one, and the table's
+format, the same for every row, cannot say that. Valid while a table's rows need
+generators of their own.
+
+## A path is walked once without drawing before it is walked for real
+
+2026-09-17, Lilleman auf Larv.
+
+Goals 10 and 13: a path that fails moves no seeded stream, at the cost of one draw-free
+walk per call, compiling its steps into a stack buffer of 16. Valid while that walk
+costs little against the draw it guards.
+
+## A country's postal codes and streets are siblings under its locality
+
+2026-09-18, Lilleman auf Larv.
+
+No open source pairs a Swedish street with its postnummer, and pairing the US through
+its ZIPs would shape the two trees differently, so both draw inside the pinned locality
+and an address agrees at that level. A street's own code is the exact pairing to add
+when a source carries it.
+
+## A locale package imports its country's geo package, and its `Modules()` returns both
+
+2026-09-24, Lilleman auf Larv; revised 2026-10-11, when the shipped data became Go packages,
+from a bench of a Go test author, a data author and a custom-CLI builder, 3 to 0. Serves
+goals 6.2 and 9.2. A locale's `address` reads its country's `geo` tree, so `svse` imports
+`geose` and `svse.Modules()` returns both: the Go import graph states the dependency, and a
+program passing `Modules()` cannot leave the places out. `svse.FS` alone loads for a stand-in
+providing `geo.SE`'s paths, and without one `sv_SE.address` fails at first reach with
+`no entry "geo"`; until `todo.md` item 196 names the missing module in that error, the bare
+`FS` works against goal 6.2. Every package exports `Modules()`, so one spelling loads any of
+them; a data package imports only the packages it reads.
+
+## The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+
+2026-10-11, Lilleman auf Larv; revised the same day, after a tenth of the streets gave
+Göteborg 163 and Stockholm 37. Goals 2 and 14: ten streets per place put 500 Stockholm
+customers on ten streets, and US places of 25,000 or more left out every town and
+village. `geo-us.py` draws with Efraimidis and Spirakis's weighted sampling. It draws
+only among places holding most of some ZCTA's land, since no other place can carry a
+postal code. The draw alone kept 310 of the 343 places of 100,000 or more and left out
+13 state capitals, so a test author pinning a city they know hit `no row of … has key
+or name`; every capital and every place of 100,000 or more ships beside the draw. The
+draw's smallest place holds 93 people. The capitals are each state's `capital` (P36) in
+[Wikidata](https://www.wikidata.org/), matched to a Census place by its GNIS id and
+else by a name no other place in its state carries. No census-designated place ships,
+so Honolulu is the one capital missing. Every Swedish street is 174k rows; a quarter
+of the US streets 475k. On 2026-10-11 the embed grew from 5.2 MB to 16 MB, and a CLI
+run printing the first `en_US.address` took 145 ms, `sv_SE.address` 95 ms and
+`misc.territory` 3 ms, within goal 13. `--place-share`, `--street-share` and
+`--min-streets` on the import scripts build another set. Valid while the first reach of
+a country stays unnoticed beside a test run.
+
+## A locale's `address` binds its country's record once and prints it whole
+
+2026-09-18, Lilleman auf Larv; revised 2026-10-05. Goal 4.1.
+
+A record cannot read another whole and keep its columns, so `sv_SE.address` names the
+same four columns as `geo.SE.address`. Its format binds one name,
+`{/geo.SE.address as a}`, and prints it whole as `{a}`. Each column reads one part of it,
+`{a.street}`, so the country's format is written once and every column describes the
+address printed. Valid while a record cannot read another whole and keep its columns.
+
+## A postort's kommun comes from its name, its tätort or its codes, never from distance
+
+2026-09-24, Lilleman auf Larv.
+
+GeoNames leaves a fifth of Sweden's codes without a kommun and carries stale spellings;
+the nearest code across a border named the wrong kommun half the time it was tried, so a
+postort none of the three rules place is dropped, as is one not cased like a place name.
+
+## A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+
+2026-09-24, Lilleman auf Larv.
+
+`I- 55 Bus` and `US Hwy 1` carry the most address ranges in many places and would head
+every address, so the import drops names spelled as a route. A ZCTA goes to the place
+its largest in-place part lies in, census-designated places left out since they never
+ship, and ships only when that place does, so a place whose every code lies mostly in a
+bigger neighbour is never drawn; counting the land outside every place too
+would drop a quarter of the places, whose codes straddle unincorporated land, for a
+postal city the USPS mostly names the same way.
+
+## `--list` stays a plain list of paths
+
+2026-09-18, Lilleman auf Larv.
+
+It is what a script reads, so every line has to be a path that `Fake` takes; a marker
+for the tables a `[selector]` follows, or a legend above them, would make the output
+something to parse before use. `--help` names the selector spelling instead, and the
+Table section teaches it.
+
+## A layout is always quoted
+
+2026-09-18, Lilleman auf Larv. Serves goal 5.6.
+
+A layout may carry the comma that separates arguments,
+`'January 2, 2006'`. A bare `2006-01-02` reads like a third date, and a double-quoted
+layout could be quotes printed in the output, so both are refused, naming the
+single-quoted spelling. The layout is Go's reference time because the
+library renders with it and a Go caller already knows it; its names are English, and a
+locale's own month and weekday names are data.
+
+## A title is a table under `sex`
+
+2026-09-18, Lilleman auf Larv.
+
+A prefix drawn apart would put `Mr` on a record whose `sex` column says `female`, which
+is the disagreement the record exists to prevent; the tables this set already has are
+what a title needs, so `en_US.title` links to `sex` as `first-name` does. Swedish has no
+everyday sexed honorific, so `sv_SE.title` is a table as well but carries no `parent`.
+Its weight column is `share`, not the `count` a name table carries, because the values
+are a curated proportion rather than bearers anyone counted.
+
+## A table owns the spelling of a selector on it
+
+2026-09-18, Lilleman auf Larv.
+
+A reference reaches a table by a path that carries no selector — `sv_SE.address` reads
+`geo.SE.locality` through its template — so the walk that reached it cannot say where
+a reader would type a selector. The table's own location can, which is why it keeps its
+path, and why an error names `geo.SE.municipality[1281]` rather than `municipality[1281]`.
+
+## No builtin reads the clock, so a date is bounded by days, never by an age
+
+2026-09-18, Lilleman auf Larv.
+
+An `age(min,max)` would make a seeded fixture change with the day it runs on, which is
+what a seed exists to prevent; a birthdate for someone 20 to 60 is
+`date(1966-01-01,2006-12-31,…)`, re-pinned as any fixture is.
+
+## `misc` is what every locale shares
+
+2026-09-18, Lilleman auf Larv.
+
+A category whose facts differ by country belongs in that country's locale, read from the
+register that country's own records use; `misc` takes only sources that are
+international. NHTSA vPIC and Mobility Sweden's registrations are national, so they are
+for `en_US.car` and `sv_SE.car`, and `misc.car` waits for an international source.
+
+## A register's canonical spelling loses to the one its domain writes
+
+2026-09-18, Lilleman auf Larv.
+
+Where a source offers several spellings of one fact, the shipped one is what records in
+that domain carry. `misc.timezone` reads `zone.tab` and not the `zone1970.tab` that
+supersedes it, because the latter keeps one zone per set of countries that have agreed
+since 1970: it spells Sweden `Europe/Berlin`, and no Swedish system writes that. For the
+same reason `misc.language` takes the ISO 639-2 register's first synonym over CLDR,
+which says "Chinese, Mandarin" for `zh`.
+
+## `misc.territory` is the spine, and a `misc` table naming a territory links to it
+
+2026-09-24, Lilleman auf Larv.
+
+Where every territory row has a child the column is a `parent`, and the import drops the
+child rows whose territory the set does not ship — 17 of `misc.timezone`'s, Antarctica's
+ten among them. Agreement across a record is worth more than the last rows of a table
+(goal 4.1), as the maintainer confirmed on 2026-10-03.
+Where no such link can hold the fact stays a column. Layer your own `misc.territory`
+over the shipped one and you must layer `misc.timezone` too, or the link fails at load
+naming the row.
+
+## A table whose register publishes no frequency draws evenly
+
+2026-09-20, Lilleman auf Larv.
+
+`misc.httpmethod`, `misc.port`, `misc.httpstatus`, `misc.mimetype` and `misc.tld` weigh
+every row alike, so GET is a ninth of the methods drawn. Where a register publishes a
+frequency it is read, as `misc.timezone` reads GeoNames populations and
+`sv_SE.first-name` SCB bearers.
+
+## `misc.timezone` weighs a zone by the people living in it
+
+2026-09-24, Lilleman auf Larv.
+
+The weight is the population GeoNames records in the zone's cities of 15,000 or more,
+floored at 15,000, which goal 16 asks for: over 300 seeded draws of
+`misc.territory[US].timezone`, the four zones most Americans live in took 278 where an
+even weight gave them 44, and `America/Indiana/Petersburg`, a town of 2,400, fell from 17
+to 0. Valid while a draw weighted this way lands where people live.
+
+## `misc.car` is one flat table, not a make linked to its models
+
+2026-09-18, Lilleman auf Larv.
+
+A row is a make and a model drawn together, so no render pairs a Volvo with a RAV4. Two
+linked tables would reach the same pairs and add a selector nothing asks for; a make
+alone is `misc.car.make`.
+
+## `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+
+2026-09-24, Lilleman auf Larv.
+
+ISO 3166-1 codes territories, so `territory` is the honest name, and `is_independent` in
+the register gives each one's state. A second table of the 195 sovereigns would hold a
+`DK` row beside the territory `DK` row, both carrying Denmark's capital, currency, flag
+and TLD — two owners for one fact, drifting at the next import. Splitting the columns to
+avoid that is worse: put `capital` on the territory alone and a country row can no
+longer name Copenhagen. The column cannot be a `parent`, since a table is never its own
+parent; a test proves every value names a row instead. A territory the register
+records no state for stands alone, which is `EH` alone, and naming one for it would be a
+claim fejkdata has no business making.
+
+## `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+
+2026-09-27, Lilleman auf Larv.
+
+A flat list of names carries neither the code a PRI encodes nor a selector reaching it,
+and goal 4 draws the two as one fact. The canonical spelling losing to the one its
+domain writes, above, settles the rest: a configuration writes `info`, so RFC 5424's
+`Informational` stays the `severity` column.
+
+The data still keys it on the code; `todo.md` item 41 brings it to this decision.
+
+## `misc.tld` keys carry the leading dot, where other tables key on a bare code
+
+2026-09-20, Lilleman auf Larv.
+
+The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a
+bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a host
+with no separator. `misc.tld[se]` misses for it, which `todo.md` item 44 ends.
+
+## `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+
+2026-09-20, Lilleman auf Larv.
+
+A `parent` demands a child for every parent row, so linking them would drop every root
+zone row naming no territory, which is most of them, and goal 2 wants them shipped. The
+loader refuses the link outright anyway: `tld` is a column of `misc.territory`, and a table
+may not be named like a column of its ancestor.
+
+## `misc.territory` carries a currency code, it does not link to `misc.currency`
+
+2026-09-18, Lilleman auf Larv.
+
+A `parent` demands a child for every parent row, and ISO 4217 registers codes no
+country's row can name: the funds codes (Mvdol, WIR Euro, US Dollar (Next day)), and VED
+beside VES, both Venezuela's, of which a country row names one. Linking would trade the
+register for the link, and `misc.territory.currency` already pairs a country with its
+currency in one draw.
+
+## An extension may name two media types
+
+2026-09-18, Lilleman auf Larv.
+
+`.xml`, `.rtf`, `.sub`, `.mpp` and `.ac` each name two rows of `misc.mimetype`.
+Separating them would mean dropping a registered type, or naming one by an extension
+that is not its own — `.mpt` is Project's template, not its document. Selecting such a
+name is an error listing both keys; select by type instead.
+
+## The Swedish ids draw Skatteverket's test series
+
+2026-09-18, Lilleman auf Larv.
+
+A Luhn-valid personnummer over a random birth number may be a living person's; 238 and
+239 after any date are blocked from assignment, so the shipped `personnummer` and
+`samordningsnummer` use those. They sit in a `birth-number` table under `sex`, not in a
+column of it, so a pick of a sex can read the number, and `sex` keeps one shape across
+locales. `todo.md` item 28 has `personnummer` read it that way, so the number agrees
+with the person's sex. A samordningsnummer's day, the birthday plus 60, is
+drawn from 61 to 88, valid in every month, rather than computed from the date drawn.
+
+## The US given names come from a mirror of the SSA file
+
+2026-09-18, Lilleman auf Larv.
+
+ssa.gov refuses a client outside the US, so `names-us.py` reads a GitHub copy that ends
+at 2020, which a count over the births since 1930 barely feels; `--names` takes the
+official zip. The SSA's placeholder rows are top-1000 entries that name nobody, so the
+import drops them by name rather than by a rank a regeneration would move.
+
+## `List` advertises direct descents only
+
+2026-09-17, Lilleman auf Larv.
+
+`region.municipality.locality` is listed, and `region.locality` resolves too but is not:
+the set of every descent through a chain of five tables is every subsequence of it, and
+the direct chain is the one a reader can predict from the tables' parents.
+
+## A path draws through its compiled steps
+
+2026-09-30, larv-review on the comprehension round the maintainer approved on 2026-09-29;
+replaces "The path walks are separate loops".
+Goals 3 and 13: `drawSteps` draws every path from its steps, compiled when templates resolve, by
+`compiledPath` or, for a caller's path, by `callerPathSteps` into a stack buffer. Both take each
+step through `compileStep`, and differ only at a choice: `compiledPath` walks every variant,
+`callerPathSteps` the first. On 2026-09-29 one loop over all three walks, switching on a mode field,
+leaked `compiledPath`'s leaves and errors with the draw's pins, arm and memo, since Go tracks a
+struct's fields as one, and a repeat of a reference path rose from 66 to 106 allocations.
+`drawSteps` and `compileStep` take their state as parameters and a step holds no node, so no
+benchmark gained an allocation. Valid while Go's escape analysis tracks a struct's fields
+as one.
+
+## A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+
+2026-10-03, larv-review in PR #145; approved by Lilleman auf Larv. Goals 5.3 and 5.6.
+`bindNames` scopes a name at compile, and a render keeps one `pickFrame` per scope. One
+field binds a name where another reads it, `"first": "{/person as p}{p.first}"` beside
+`"last": "{p.last}"`, so the scope cannot be the template holding the binding; the category
+is the smallest unit holding every field and record column. A choice's item binds a name
+for its own reads, picked anew each time the item is drawn; a read outside the item is
+refused, since it would see a binding another item leaves out. A pick is drawn on
+its first read, so a record's columns, rendered in name order, read one pick whichever binds
+it.
+
+The scope is lexical. A read entering a category, by a reference, through a name or from
+`Fake`, sees no frame (one render of a scope's picks) opened above it, so a category renders
+the same whoever references it. Such a read opens the frames around where it lands: a read through a
+name from its pick's memo, so `{n}` and `{n.path}` read one pick of each name inside, and a
+reference or `Fake` fresh ones.
+
+A name is a closure. It reaches every field, choice item and repeat nested inside the
+scope binding it, bar the field bound to it, and keeps one pick through the draw. A name bound inside a repeat picks
+anew on each iteration. A name an enclosing scope binds may not be bound again, so no name
+shadows another.
+The maintainer ruled so on 2026-10-09.
+
+A pick keeps only the levels a read of the name addresses, so a template draws the same way
+under a name as anywhere else (goal 5.2). A read of a name bound to a field stays in the
+category's frames, since the field sits inside it.
+
+Valid while names are read only inside the category binding them.
+
+## comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+
+2026-10-04, Lilleman auf Larv. Serves goal 3 and applies KISS.
+
+By the maintainer's decision of 2026-10-09, every 0.1.0 chunk ships under the floor, including the chunk that holds the release's scoring run, so that 0.1.0 can ship features. Items 170 to 175 and 177 move to 0.2.0. Item 178 plans first how item 107 regroups the engine. A paired ruling that rules worse still blocks a merge. Valid until 0.1.0 is cut.
+
+Shipped under that decision: item 89, the last of 0.1.0, at 52e0304. Its paired ruling against 90ffcff was better. The release's scoring run read Navigation 7.00, Locality 5.75, Shape 6.38, Self-sufficiency 6.50 and overall 6.25; its seats named places items 170 to 172 and 178 hold, and `isPath`, filed as item 251.
+
+Shipped under the floor as 0.1.0's distribution, each by the maintainer's decision: item 58, prebuilt binaries and packages, which the plan had under 0.2.0, on 2026-10-13; and the image on Docker Hub, on 2026-10-14. Neither changes Go code.
+
+Shipped by the maintainer's decision of 2026-10-11 with its ratchet waived: item 182, the shipped data as Go packages, at a632a92. Its last paired ruling against bf1becb was same overall, Mid A better and Junior A worse, with Shape fallen, three seats against the namespace mount in `internal/datafiles/mount.go` and the `at` field of `datafiles.Source`; the scoring run that would rule it worse under the floor was not run. `todo.md` item 197 replaces that mount.
+
+Shipped under the floor before that by the maintainer's decisions of 2026-10-07, each with the scoring run and its ratchet waived: the CLI reading its template from stdin; and goal 5.7 loading whatever is well-formed and means one thing, dropping every refusal that guessed at a mistake.
+
+The scoring runs are in [the comprehension history](comprehension-history.md). The nine-seat panel at depth 1, run beside the scoring run of 2026-10-09 on commit b9d1af6: Navigation 7.33, Locality 6.17, Shape 6.67, Self-sufficiency 6.83, overall 6.67. Against the panel on ad59967, its overall rose from 6.56, its Navigation from 7.17, its Locality from 6.06 and its Self-sufficiency from 6.56, and its Shape held at 6.67. Locality is lowest again, as at every panel since 2026-09-22. All 13 seats of the two runs found the named-pick keep rule hardest to follow: `keptInPick` and `readUnder`, where a key `pickKey.under` builds at render must equal one `addressedKeys` built at load, with `readField` and `readName` around them. What the two runs found is filed in `todo.md`, and `todo.md` item 177 runs both again. Valid while `AGENTS.md` gates goal 3 at 7.0.
+
+## The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+
+2026-10-05, Lilleman auf Larv. Serves goals 3.2 and 3.4. Two architect reviews found the
+engine's types, `template`, `table`, `arm`, `op` and `nameBinding`, bound in one cycle
+because several passes each write part of the same structs; the template language's
+recursion reads few of those types. Moving the engine while several resolve passes still
+wrote each value would have spread that knot across packages, behind forwarding calls from
+every public method. The draw state, the grammar, `DataType`, what a proof knows of a
+value, the builtins, the reading of data files and a table's rows read no engine type, so
+each sits in a package under `internal/`, behind the import list `imports_test.go` checks.
+A rows table holds the engine table owning it as a value of a type parameter, so
+`internal/rows` imports no engine type. Valid until `todo.md` item 107 moves the engine.
+
+## Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+
+2026-10-05, Lilleman auf Larv. Serves goal 5.1: everything outside `{…}` prints exactly as
+written. Yet `{{` prints `{`, `}}` prints `}`, and a lone `}` is a load error naming `}}`, as
+the README's Format string section states. The maintainer ruled the escape implicit in the
+goal: doubled braces and a lone `}` belong to the `{…}` syntax, so the goal names no escape and
+a lone `}` stays an error. Valid while a template escapes a brace by doubling it.
+
+## Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+
+2026-10-05, Lilleman auf Larv. Serves goal 7.2. Goals are aims: it is enough that a first
+template can be written with no escape and no options, as `echo 'name: {/sv_SE.person.last}' | fejkdata`
+is. A template that prints a literal brace writes `{{` or `}}`, and the goal allows that. Valid
+while `{{` and `}}` are the template syntax's only escape.
+
+## The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+
+2026-10-11, Lilleman auf Larv. Serves goal 2. [The Ethnologue
+200](https://www.ethnologue.com/insights/ethnologue200/) for 2026 ranks languages by first-
+and second-language speakers together: English, Mandarin Chinese, Hindi, Spanish, Standard
+Arabic, French, Bengali, Portuguese, Indonesian and Urdu. Each ships in the country with the
+most speakers of it: `en_US`, `zh_CN`, `hi_IN`, `es_MX`, `ar_EG`, `fr_FR`, `bn_BD`, `pt_BR`,
+`id_ID` and `ur_PK`. The Nordic countries are `sv_SE`, `nb_NO`, `da_DK`, `fi_FI` and
+`is_IS`, and the maintainer adds Germany, `de_DE`. Each locale carries goal 2.1's
+categories, and its `address` reads its country's `geo/` tree by reference. Egypt and
+Pakistan are not yet confirmed: `todo.md` item 209 and `todo.md` item 214 check each against
+Ethnologue's country pages before it ships, and record the result here. Valid until
+Ethnologue's ranking changes its ten.
+
+## A country with no open address register covering it whole ships its places and postal codes without streets
+
+2026-10-11, Lilleman auf Larv. Serves goal 2.2. Where no open register covers a country's
+streets, the only source is OpenStreetMap. ODbL binds a database derived from it to
+share-alike and attribution, and every program linking that country's geo package would
+inherit both terms. So that country's `address` prints no street, and the README's Data
+section says why beside each such country. This works against goal 2.1 until
+`todo.md` item 218 gives those countries streets. Valid until the maintainer accepts ODbL
+data, or the country opens a register.
+
+## The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+
+2026-10-13, decided by the maintainer. Serves goal 1.1; technical principle: prefer the
+vendor-neutral option (`~/.claude/principles/technical-principles.md`). The repositories
+are built here, as static files any host can serve: `release-tooling/package-repos.Dockerfile`
+signs them with a GPG key (apt, dnf) and an RSA key (apk). A hosted package service would
+only save that tooling. GitHub already hosts the code (see "GitHub is canonical, and the
+module path names it"), so Pages adds no new vendor. A Pages site holds at
+most 1 GB and a release's packages take about 35 MB, so the repositories hold the latest
+release only; each GitHub release still carries its own packages, and the image keeps a tag
+per version. Leaving Pages costs
+each Linux user an edit: their apt list, `.repo` file or apk line names `larvit.github.io`.
+Valid while GitHub hosts the code.
+
+## The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+
+2026-10-14, decided by the maintainer. Serves goal 1.1; technical principle: prefer the
+vendor-neutral option (`~/.claude/principles/technical-principles.md`). Docker Hub is
+Docker's default registry, so `docker run larvit/fejkdata` works with no host in the name;
+that is a capability only Docker Hub has. GHCR stays the image's source, and adds no
+vendor while GitHub hosts the code; being OCI, the image moves to another registry by a
+change of path. GoReleaser pushes there, and a job of its own copies the image to Docker
+Hub, so a Docker Hub failure holds up no other channel. GHCR also serves CI on shared
+runners, which Docker Hub's per-IP limit on anonymous pulls blocks. Leaving Docker Hub
+costs each user an edit of every `docker run` line and pinned script that names the bare
+`larvit/fejkdata`.

docs/reading-cost.md

@@ -0,0 +1,637 @@
+# Reading cost
+
+## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
+
+68314 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `compose.yaml` 3
+- `data-import/territory.py` 2
+- `data_test.go` 2
+- `shipped_data_test.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `data-import/tsv.py` 1
+- `data/en_US/phone.json` 1
+
+## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
+
+118930 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 8
+- `README.md` 6
+- `table_test.go` 4
+- `render.go` 3
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `fejkdata.go` 1
+
+## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
+
+93128 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `data_test.go` 4
+- `todo.md` 3
+- `format_stability_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data/en_US/email.json` 1
+- `data/en_US/first-name.tsv` 1
+- `data/en_US/sex.tsv` 1
+- `data/en_US/title.tsv` 1
+
+## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
+
+79628 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `todo.md` 5
+- `format.go` 4
+- `valueproof.go` 4
+- `calc_test.go` 3
+- `datatype_test.go` 2
+- `name.go` 2
+- `name_test.go` 2
+- `record_test.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
+
+366332 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `table_test.go` 17
+- `table.go` 9
+- `struct.go` 8
+- `README.md` 7
+- `docs/decisions.md` 7
+- `record.go` 5
+- `calc.go` 4
+- `data_test.go` 4
+- `reference.go` 4
+- `render.go` 4
+
+## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
+
+705270 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 16
+- `todo.md` 16
+- `README.md` 10
+- `render.go` 5
+- `table.go` 5
+- `format.go` 4
+- `AGENTS.md` 3
+- `calc.go` 3
+- `data.go` 3
+- `fejkdata.go` 3
+
+## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
+
+59846 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `CHANGELOG.md` 2
+- `README.md` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `fejkdata_test.go` 2
+- `go.mod` 2
+- `.dockerignore` 1
+- `builtins_test.go` 1
+- `cmd/fejkdata/main.go` 1
+
+## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
+
+98933 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `format_test.go` 5
+- `todo.md` 5
+- `calc.go` 4
+- `calc_test.go` 4
+- `format.go` 4
+- `path.go` 3
+- `reference.go` 3
+- `README.md` 2
+- `builtins_test.go` 2
+- `inline.go` 2
+
+## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
+
+129910 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `builtins_test.go` 7
+- `todo.md` 5
+- `format.go` 4
+- `calc_test.go` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `reference.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `calc.go` 1
+
+## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
+
+91864 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `table_test.go` 4
+- `data.go` 3
+- `todo.md` 3
+- `graph.go` 2
+- `node.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+
+## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
+
+98023 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `path_test.go` 4
+- `README.md` 3
+- `docs/decisions.md` 3
+- `imports_test.go` 2
+- `path.go` 2
+- `reference_test.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
+
+115125 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `name_test.go` 5
+- `todo.md` 5
+- `shape_test.go` 3
+- `README.md` 2
+- `graph.go` 2
+- `inline.go` 2
+- `node.go` 2
+- `record_test.go` 2
+- `struct.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
+
+53085 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `AGENTS.md` 3
+- `README.md` 3
+- `todo.md` 3
+- `.gitignore` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
+
+67696 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 3
+- `pick_test.go` 2
+- `render.go` 2
+- `table_test.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `data/geo/SE/address.json` 1
+- `data/sv_SE/address.json` 1
+- `docs/decisions.md` 1
+- `pick.go` 1
+
+## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
+
+75959 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 5
+- `README.md` 2
+- `inline.go` 2
+- `namefence.go` 2
+- `path.go` 2
+- `pick_test.go` 2
+- `record.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
+
+129293 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `README.md` 6
+- `format.go` 6
+- `docs/decisions.md` 5
+- `record.go` 5
+- `inline.go` 4
+- `render.go` 4
+- `shipped.go` 4
+- `CHANGELOG.md` 3
+- `data.go` 3
+
+## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
+
+79101 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `format.go` 5
+- `todo.md` 5
+- `path.go` 3
+- `docs/decisions.md` 2
+- `env.go` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
+
+2986167 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 24
+- `AGENTS.md` 18
+- `CHANGELOG.md` 18
+- `data-import/geo-se.py` 17
+- `record.go` 17
+- `cmd/fejkdata/main.go` 16
+- `struct.go` 16
+- `testdata/shipped_shape.txt` 16
+- `.github/workflows/test.yml` 15
+- `arm.go` 15
+
+## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
+
+96842 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `CHANGELOG.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 2
+- `doc.go` 2
+- `docs/decisions.md` 2
+- `path.go` 2
+- `resolve.go` 2
+- `AGENTS.md` 1
+- `arm.go` 1
+
+## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
+
+152553 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 8
+- `data-import/geo-us.py` 5
+- `AGENTS.md` 4
+- `fejkdata.go` 4
+- `graph.go` 4
+- `docs/decisions.md` 3
+- `name.go` 3
+- `pick.go` 3
+- `resolve.go` 3
+
+## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
+
+2852445 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 23
+- `cmd/fejkdata/main.go` 23
+- `pick.go` 18
+- `AGENTS.md` 17
+- `CHANGELOG.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+- `arm.go` 15
+- `fejkdata.go` 15
+- `path.go` 15
+
+## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
+
+93957 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `name.go` 6
+- `README.md` 5
+- `graph.go` 4
+- `env.go` 3
+- `path.go` 3
+- `resolve.go` 3
+- `arm.go` 2
+- `compose.yaml` 2
+- `docs/decisions.md` 2
+
+## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
+
+68500 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `render_test.go` 6
+- `todo.md` 3
+- `README.md` 2
+- `docs/decisions.md` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `doc.go` 1
+- `doc_test.go` 1
+- `node.go` 1
+- `render.go` 1
+
+## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
+
+69756 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 3
+- `docs/decisions.md` 3
+- `CHANGELOG.md` 2
+- `AGENTS.md` 1
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/tsv.py` 1
+
+## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
+
+60588 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 2
+- `docs/decisions.md` 2
+- `record.go` 2
+- `render.go` 2
+- `CHANGELOG.md` 1
+- `docs/comprehension-history.md` 1
+- `inline.go` 1
+- `reference.go` 1
+- `shipped.go` 1
+
+## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
+
+2855972 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 25
+- `CHANGELOG.md` 22
+- `cmd/fejkdata/main.go` 20
+- `node.go` 19
+- `testdata/shipped_shape.txt` 19
+- `AGENTS.md` 18
+- `arm.go` 17
+- `DATA-LICENSES.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+
+## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
+
+340530 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 15
+- `docs/decisions.md` 14
+- `README.md` 4
+- `.github/workflows/test.yml` 2
+- `compose.yaml` 2
+- `fejkdata.go` 2
+- `go.mod` 2
+- `release-tooling/publish_release.py` 2
+- `shipped.go` 2
+- `shippedindex.go` 2
+
+## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
+
+91452 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `todo.md` 5
+- `cmd/fejkdata/main_test.go` 3
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `imports_test.go` 2
+- `.dockerignore` 1
+- `.github/workflows/test.yml` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
+
+112728 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `docs/decisions.md` 5
+- `shipped_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data.go` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+- `internal/datafiles/datafiles.go` 1
+
+## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
+
+200600 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 13
+- `todo.md` 8
+- `CHANGELOG.md` 6
+- `cmd/fejkdata/main.go` 6
+- `docs/decisions.md` 6
+- `shipped_test.go` 6
+- `DATA-LICENSES.md` 5
+- `bench_test.go` 5
+- `data_test.go` 5
+- `data.go` 4
+
+## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
+
+87725 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 4
+- `data_test.go` 4
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/source.py` 1
+
+## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
+
+90566 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `data-import/geo-us.py` 6
+- `README.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 3
+- `compose.yaml` 2
+- `data/geo/US/locality.tsv` 2
+- `data_test.go` 2
+- `.gitignore` 1
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
+
+76079 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 10
+- `docs/decisions.md` 6
+- `README.md` 4
+- `AGENTS.md` 2
+- `data-import/geo-se.py` 2
+- `CHANGELOG.md` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+
+## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
+
+102267 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `CHANGELOG.md` 6
+- `todo.md` 6
+- `cli/cli_test.go` 4
+- `docs/decisions.md` 3
+- `cmd/fejkdata/main_test.go` 2
+- `AGENTS.md` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+- `data/modules.go` 1
+
+## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
+
+372546 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 30
+- `todo.md` 10
+- `cli/cli.go` 5
+- `cli/cli_test.go` 3
+- `CHANGELOG.md` 2
+- `data/sv_SE/.fejkdata.json` 2
+- `docs/decisions.md` 2
+- `internal/datafiles/datafiles.go` 2
+- `record.go` 2
+- `compose.yaml` 1
+
+## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
+
+78327 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 8
+- `README.md` 5
+- `CHANGELOG.md` 1
+- `DATA-LICENSES.md` 1
+- `data/misc/car.tsv` 1
+- `docs/decisions.md` 1
+- `docs/research/research-sources-se.md` 1
+- `docs/research/research-sources-world.md` 1
+
+## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
+
+60817 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `shipped_data_test.go` 2
+- `.dockerignore` 1
+- `.gitignore` 1
+- `data_test.go` 1
+- `docs/decisions.md` 1
+- `release-tooling/publish_release.py` 1
+- `shipped_test.go` 1
+
+## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
+
+78241 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 10
+- `todo.md` 7
+- `CHANGELOG.md` 3
+- `docs/decisions.md` 3
+- `compose.yaml` 1
+- `data/en_US/itin.json` 1
+- `data/misc/timezone.json` 1
+- `data/misc/timezone.tsv` 1
+- `go.mod` 1

docs/research/research-age-bands.md

@@ -0,0 +1,10 @@
+# Population by age band — research 2026-10-04
+
+Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
+
+| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
+|---|---|---|---|---|---|---|---|
+| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
+| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
+
+At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.

docs/research/research-sources-se.md

@@ -1,7 +1,7 @@
 # Swedish (sv_SE) fake-data sources — research 2026-09-17
 
 Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
-"Verified" = fetched and read this session. Local copies of inspected files: `/home/lilleman/.claude/jobs/e377c8d4/tmp/`.
+"Verified" = fetched and read for this research.
 
 ## Licence summary (embed in MIT repo?)
 
@@ -15,7 +15,7 @@ Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other
 | Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
 | Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
 | Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
-| PTS numbering plan PDF | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
+| PTS numbering plan PDF, pts.se number pages | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
 | Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
 | SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
 | Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
@@ -90,6 +90,7 @@ Recommendation: embed SCB 2022 xlsx-derived lists (top-N per sheet with counts)
 ## 3. Phone
 
 - Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
+- Reserved for fiction, PTS "Telefonnummer till böcker och filmer" (verified 2026-09-21; curl and Playwright are Radware-blocked, a rendering fetcher gets through): https://www.pts.se/internet-och-telefoni/telefonnummer-och-adressering/telefonnummer-till-bocker-och-filmer/ — mobile 070-1740605–1740699, geographic 08-46500400–46500499 (Stockholm), 031-3900600–3900699 (Göteborg), 040-6280400–6280499 (Malmö), 0980-319200–319299 (Kiruna). 495 numbers PTS holds back so they reach nobody, which is goal 16's never-reaching path for `sv_SE.phone`. Same licence status as the numbering plan: none stated, facts only, cite PTS. The mobile block fits the shipped `{a} {b} {c}` grouping (`174 06 05`–`99`), as do 031 and 040; Stockholm's 8 subscriber digits and Kiruna's 6 need a format of their own, and 0980 is not among the 20 shipped area codes.
 - 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
 - Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
 - Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).

draw.go

@@ -1,461 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// drawSet is one render's reference draws: the unnamed draw group's, and each named one's.
-type drawSet struct {
-	unnamed draws
-	named   map[string]*draws
-}
-
-// drawScope is where a render reads its reference paths: a draw set, in the draw group of the
-// template rendering, and the row a table's format is rendering, which its columns read.
-type drawScope struct {
-	set   *drawSet
-	group string
-	t     *table
-	row   int
-}
-
-// newDrawSet makes the unnamed group's maps where the set is declared, keeping them on that frame's
-// stack for a render that reads through them; a zero drawSet makes them on its first read instead.
-func newDrawSet(s *session) drawSet {
-	return drawSet{unnamed: draws{variant: map[string]node{}, value: map[string]draw{}, s: s}}
-}
-
-// renderOnce renders n as one render, over draws of its own.
-func renderOnce(s *session, n node) string {
-	set := drawSet{unnamed: draws{s: s}}
-	return render(s, n, drawScope{set: &set})
-}
-
-// in is the scope t renders in: its draw group where it names one, else its caller's.
-func (sc drawScope) in(t *template) drawScope {
-	if t.drawGroupKey != "" {
-		sc.group = t.drawGroupKey
-	}
-	return sc
-}
-
-// draws is the set's draws for sc's draw group. The session is passed in rather than
-// read out of the set: copying it from there would leak the set's maps to the heap.
-func (sc drawScope) draws(s *session) *draws {
-	if sc.group == "" {
-		return &sc.set.unnamed
-	}
-	d, drew := sc.set.named[sc.group]
-	if !drew {
-		if sc.set.named == nil {
-			sc.set.named = map[string]*draws{}
-		}
-		d = &draws{variant: map[string]node{}, value: map[string]draw{}, s: s}
-		sc.set.named[strings.Clone(sc.group)] = d // a key from sc would leak sc, and with it every render's draw set, to the heap
-	}
-	return d
-}
-
-// drawGroupOf reads a template's "drawGroup" (default ""), which a repeat cannot carry: each
-// iteration renders in no draw group.
-func drawGroupOf(m map[string]any, repeat int) (string, error) {
-	v, ok := m["drawGroup"]
-	if !ok {
-		return "", nil
-	}
-	name, ok := v.(string)
-	switch {
-	case !ok:
-		return "", fmt.Errorf("drawGroup must be a string, got %T", v)
-	case name == "":
-		return "", fmt.Errorf(`drawGroup "" is the default, so it has no effect; drop it`)
-	case repeat > 1:
-		return "", fmt.Errorf("drawGroup %q on a repeat names nothing, since each iteration is a render of its own; drop it", name)
-	}
-	return name, nil
-}
-
-// keyDrawGroup keys t's draw group by the category t sits in, "" for an inline template, so a name
-// is local to its category.
-func (t *template) keyDrawGroup(category string) {
-	if t.drawGroup != "" {
-		t.drawGroupKey = category + "/" + t.drawGroup
-	}
-}
-
-// checkNestedDrawGroup refuses a template beneath one drawing in group that names group again,
-// short of a repeat or another draw group.
-func checkNestedDrawGroup(fields map[string]node, group string) error {
-	if group == "" {
-		return nil
-	}
-	var walk func(path string, n node) error
-	walk = func(path string, n node) error {
-		t, isTemplate := n.(*template)
-		switch {
-		case isTemplate && t.drawGroup == group:
-			return fmt.Errorf("%q names drawGroup %q, the draw group this template draws in already; drop it", path, group)
-		case isTemplate && (t.drawGroup != "" || t.repeat > 1):
-			return nil
-		}
-		for _, c := range contained(n) {
-			if err := walk(join(path, c.name), c.node); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	for _, name := range sortedNames(fields) {
-		if err := walk(name, fields[name]); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// drawCheck fences each template of a scope as a render of its own, remembering which nodes read a
-// reference path.
-type drawCheck struct {
-	memo map[readsMemo]bool
-}
-
-type readsMemo struct {
-	n           node
-	stopAtGroup bool
-}
-
-// checkDrawGroup refuses a draw group that splits nothing: one whose render reads every reference
-// path inside a repeat or a nested draw group, which draw apart from it whatever it names.
-func (c *drawCheck) checkDrawGroup(path string, n node) error {
-	if t, ok := n.(*template); ok && t.drawGroup != "" && !c.splitsDraws(t) {
-		return fmt.Errorf("%s: drawGroup %q splits nothing, since nothing it renders reads a reference path outside a repeat or a nested drawGroup; drop it", path, t.drawGroup)
-	}
-	return nil
-}
-
-func (c *drawCheck) checkDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok {
-		return nil
-	}
-	if err := checkOwnFamily(t); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	if !c.readsPath(t) {
-		return nil
-	}
-	w := newDrawWalk()
-	for _, e := range renderEdges(t) {
-		w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
-	}
-	if err := w.check(); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// checkOwnFamily refuses a table's format or cell that reads, however many templates
-// away and through a repeat or a draw group too, a table of its own family: a row
-// rendered whole draws its row without pinning it, so the family would draw apart
-// from the row being rendered, whichever draws the reaching template holds.
-func checkOwnFamily(t *template) error {
-	own := t.table
-	if own == nil {
-		own = t.cellOf
-	}
-	if own == nil {
-		return nil
-	}
-	seen := map[node]bool{}
-	var find func(n node) (renderEdge, *table, bool)
-	find = func(n node) (renderEdge, *table, bool) {
-		if seen[n] {
-			return renderEdge{}, nil, false
-		}
-		seen[n] = true
-		for _, e := range renderEdges(n) {
-			if a, isRef := refRead(n, e.label); isRef {
-				if head, isTable := n.(*template).fields[a.key].(*table); isTable && head.family() == own.family() {
-					return e, head, true
-				}
-			}
-			if e, head, found := find(e.to); found {
-				return e, head, true
-			}
-		}
-		return renderEdge{}, nil, false
-	}
-	if e, head, found := find(t); found {
-		return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", e.reached(), head.category, own.category)
-	}
-	return nil
-}
-
-// checkRecordDraws fences the columns of a record — a template compiled at the top without a
-// repeat — so a load proves the record view of it as well as the string view.
-func (c *drawCheck) checkRecordDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || !t.record {
-		return nil
-	}
-	// A record-only template's format renders nothing, so weigh the columns, not the format.
-	columns := recordColumns(t)
-	reads := false
-	for _, name := range columns {
-		reads = reads || c.readsPath(t.fields[name])
-	}
-	if !reads {
-		return nil
-	}
-	if err := checkColumnDraws(t, columns); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// readsPath reports whether rendering n reads a reference path, short of a repeat, which renders
-// over draws of its own.
-func (c *drawCheck) readsPath(n node) bool { return c.reads(n, false) }
-
-// splitsDraws reports whether rendering n reads a reference path that n's own draw group answers
-// for: one outside a repeat and outside a nested draw group, which hold their own draws.
-func (c *drawCheck) splitsDraws(n node) bool { return c.reads(n, true) }
-
-// reads walks what rendering n renders for a reference path, stopping at a repeat — and at a nested
-// draw group when stopAtGroup — since each holds draws of its own.
-func (c *drawCheck) reads(n node, stopAtGroup bool) bool {
-	k := readsMemo{n, stopAtGroup}
-	if r, done := c.memo[k]; done {
-		return r
-	}
-	r := false
-	for _, e := range renderEdges(n) {
-		if a, isRef := refRead(n, e.label); (isRef && (len(a.tail) > 0 || readsTable(n, a))) || (!repeats(e.to) && !(stopAtGroup && grouped(e.to)) && c.reads(e.to, stopAtGroup)) {
-			r = true
-			break
-		}
-	}
-	if c.memo == nil {
-		c.memo = map[readsMemo]bool{}
-	}
-	c.memo[k] = r
-	return r
-}
-
-func repeats(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.repeat > 1
-}
-
-func grouped(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.drawGroupKey != ""
-}
-
-// readsTable reports whether a reference of n names a table, whose draw the render's
-// group answers for even when read whole.
-func readsTable(n node, a arm) bool {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return false
-	}
-	_, isTable := t.fields[a.key].(*table)
-	return isTable
-}
-
-// refRead is the reference an edge of n reads; false when the edge reads none.
-func refRead(n node, label string) (arm, bool) {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return arm{}, false
-	}
-	a := splitArm(label, t.refs)
-	return a, isRef(a.key)
-}
-
-// checkColumnDraws fences a record's columns as one render.
-func checkColumnDraws(t *template, columns []string) error {
-	w := newDrawWalk()
-	for _, name := range columns {
-		w.walk(t.fields[name], drawAt{group: t.drawGroupKey, route: drawRoute{spelling: fmt.Sprintf("column %q", name)}})
-	}
-	return w.check()
-}
-
-// drawWalk gathers the references one render reads, by draw group, for check to compare.
-type drawWalk struct {
-	reads []pathRead
-	read  map[drawKey]bool
-	seen  map[drawVisit]bool
-}
-
-// drawAt is where a walk stands: the draw group it draws in, how the render's root reached it, and
-// the table rows it entered.
-type drawAt struct {
-	group string
-	route drawRoute
-	alt   rowSet
-}
-
-// rowSet is the rows a walk entered, one per table: only one row of a table renders, so reads in
-// two rows of one table never meet, while reads in one row, across its columns and whatever they
-// reach, do.
-type rowSet []tablePin
-
-func (s rowSet) rowOf(t *table) (int, bool) {
-	for _, p := range s {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	return 0, false
-}
-
-// enter is s with row r of t, where t is not in it yet; the set is copied, since walks branch.
-func (s rowSet) enter(t *table, r int) rowSet {
-	if _, in := s.rowOf(t); in {
-		return s
-	}
-	out := append(append(make(rowSet, 0, len(s)+1), s...), tablePin{t, r})
-	sort.Slice(out, func(i, j int) bool { return out[i].t.category < out[j].t.category })
-	return out
-}
-
-// key spells the set for a map, by the tables' identities.
-func (s rowSet) key() string {
-	var b strings.Builder
-	for _, p := range s {
-		fmt.Fprintf(&b, "%p[%d]", p.t, p.row)
-	}
-	return b.String()
-}
-
-// alternatives reports whether two reads sit in different rows of one table.
-func alternatives(a, b drawAt) bool {
-	for _, p := range a.alt {
-		if r, in := b.alt.rowOf(p.t); in && r != p.row {
-			return true
-		}
-	}
-	return false
-}
-
-// drawRoute is how a render reaches a draw: as its author spells it, and the root edge's label.
-type drawRoute struct{ spelling, label string }
-
-// pathRead is one reference a render reads: a path, or a bare reference with no tail.
-type pathRead struct {
-	at drawAt
-	a  arm
-	tr *tableRead // set where the reference names a table
-}
-
-type drawVisit struct {
-	n     node
-	group string
-	alt   string
-}
-
-type drawKey struct {
-	group string
-	path  string
-	alt   string
-}
-
-func newDrawWalk() *drawWalk {
-	return &drawWalk{read: map[drawKey]bool{}, seen: map[drawVisit]bool{}}
-}
-
-// walk follows what rendering n renders. A repeat renders over draws of its own, so the walk stops
-// there.
-func (w *drawWalk) walk(n node, at drawAt) {
-	v := drawVisit{n, at.group, at.alt.key()}
-	if w.seen[v] {
-		return
-	}
-	w.seen[v] = true
-	if repeats(n) {
-		return
-	}
-	if t, isTemplate := n.(*template); isTemplate && t.drawGroupKey != "" {
-		at.group = t.drawGroupKey
-	}
-	for _, e := range renderEdges(n) {
-		if cell, isCell := e.to.(*template); isCell && cell.cellOf != nil {
-			if at.alt.excludes(cell) {
-				continue
-			}
-			w.edge(n, e, drawAt{at.group, at.route, at.alt.enter(cell.cellOf, cell.cellRow)})
-			continue
-		}
-		w.edge(n, e, at)
-	}
-}
-
-// excludes reports whether a cell's row cannot render with the rows entered: another
-// row of its table, or a row outside an entered ancestor's.
-func (s rowSet) excludes(cell *template) bool {
-	for _, p := range s {
-		if p.t == cell.cellOf {
-			return p.row != cell.cellRow
-		}
-		if cell.cellOf.descends(p.t) && !cell.cellOf.under(cell.cellRow, p.t, p.row) {
-			return true
-		}
-	}
-	return false
-}
-
-// edge records the reference an edge reads, then walks on with every row the read
-// pins entered, so a selected row renders only its own cells.
-func (w *drawWalk) edge(from node, e renderEdge, at drawAt) {
-	if a, reads := refRead(from, e.label); reads {
-		tr := tableReadOf(from.(*template).fields[a.key], a, e.to)
-		if k := (drawKey{at.group, a.path, at.alt.key()}); !w.read[k] {
-			w.read[k] = true
-			w.reads = append(w.reads, pathRead{at, a, tr})
-		}
-		if tr != nil {
-			tr.pins.each(func(t *table, r int) { at.alt = at.alt.enter(t, r) })
-		}
-	}
-	w.walk(e.to, at)
-}
-
-// check refuses what one draw per reference path cannot answer for: a read of a level beside a path
-// another read takes into it, and two reads of one table family that select different rows. Reads
-// are compared in path order, so which pair is reported does not vary.
-func (w *drawWalk) check() error {
-	sort.SliceStable(w.reads, func(i, j int) bool {
-		if w.reads[i].at.group != w.reads[j].at.group {
-			return w.reads[i].at.group < w.reads[j].at.group
-		}
-		return w.reads[i].a.path < w.reads[j].a.path
-	})
-	for i, level := range w.reads {
-		for _, into := range w.reads[i+1:] {
-			if into.at.group != level.at.group || alternatives(level.at, into.at) {
-				continue
-			}
-			if strings.HasPrefix(into.a.path, level.a.path+".") && !(level.tr != nil && level.tr.whole) {
-				return overlapError(level.at.route, level.a.name, into)
-			}
-		}
-	}
-	return checkFamilies(w.reads)
-}
-
-func overlapError(route drawRoute, ref string, into pathRead) error {
-	return fmt.Errorf("%s renders a level that %s reads a path into; name the fields you want instead, or draw them apart with a drawGroup", route.spelled(ref), into.at.route.spelled(into.a.name))
-}
-
-// spelled names the route, and the reference it reaches a draw by where its root edge is not that
-// reference.
-func (r drawRoute) spelled(ref string) string {
-	if ref == "" || ref == r.label {
-		return r.spelling
-	}
-	return fmt.Sprintf("%s with {%s}", r.spelling, ref)
-}

env.go

@@ -0,0 +1,113 @@
+package fejkdata
+
+import (
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// readValue is what one read drew: its text, and whether it landed on a null.
+type readValue struct {
+	text string
+	null bool
+}
+
+// frameStack is the frames of the name scopes rendering, innermost last. A render's env points
+// at it, and a repeat iteration's shares it: holding the frames in a renderEnv would move every
+// render's env to the heap.
+type frameStack struct {
+	frames []*pickFrame
+}
+
+func (st *frameStack) push(f *pickFrame) int {
+	mark := len(st.frames)
+	st.frames = append(st.frames, f)
+	return mark
+}
+
+func (st *frameStack) pop(mark int) { st.frames = st.frames[:mark] }
+
+// renderEnv is where a render reads: the frames of the name scopes rendering; and row, the row
+// of the table rendering, which its columns read. It passes by value, and what is read from it
+// reaches a map key, an interface or a func value only as a copy; else env, and with it every
+// render's frames, moves to the heap.
+type renderEnv struct {
+	frames *frameStack
+	row    renderedRow
+	// base is the depth of the frame stack when the read rendering the category entered it,
+	// below which no frame is the category's.
+	base int
+	// pick is the named pick being rendered, nil outside one; pickAt is the key in it of what renders.
+	pick   *namedPick
+	pickAt pickKey
+}
+
+type renderedRow struct {
+	t     *table
+	index int
+}
+
+// at returns the env n renders in, n being the leaf of a path whose rows sit in pins; where n
+// is a table's row or column, the env carries the row pins holds for that table.
+func (env renderEnv) at(n node, pins *pinSet) renderEnv {
+	switch n := n.(type) {
+	case *tableRow:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	case *tableColumn:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	}
+	return env
+}
+
+func (env renderEnv) rowOf(t *table) int {
+	if env.row.t != t {
+		panic(invariant.Broken("a column of %s renders in an env holding no row of it", t.rows.Segment()))
+	}
+	return env.row.index
+}
+
+// enter starts a read landing on n: the read sees no frame opened before it, and gets a frame for
+// each name scope around n, from memo where there is one, so reads sharing memo read one pick of
+// each name. It returns the mark that closes those frames.
+func (env renderEnv) enter(n node, memo *drawMemo) (renderEnv, int) {
+	env = env.entering()
+	for scope := scopeAround(n); scope != nil; scope = scope.up {
+		if len(scope.order) > 0 {
+			env.frames.push(memo.enteredFrame(scope))
+		}
+	}
+	return env, env.base
+}
+
+// entering is env as a read entering a category sees it: no frame opened before it, and no pick.
+func (env renderEnv) entering() renderEnv {
+	env.base, env.pick = len(env.frames.frames), nil
+	return env
+}
+
+// renderFrame opens a fresh frame of t's scope, where t binds names and no read entering the
+// category opened one, returning the mark that closes it, or -1.
+func (env renderEnv) renderFrame(t *template) int {
+	own := t.ownScope()
+	if own == nil || env.frameOf(own) != nil {
+		return -1
+	}
+	return env.frames.push(newPickFrame(own))
+}
+
+// frameOf is the frame of scope rendering since the read entering the category, nil where none is.
+func (env renderEnv) frameOf(scope *nameScope) *pickFrame {
+	for i := len(env.frames.frames) - 1; i >= env.base; i-- {
+		if f := env.frames.frames[i]; f.scope == scope {
+			return f
+		}
+	}
+	return nil
+}
+
+// drawRowOf draws the row a render of t reads: inside the pick's rows where t renders as part of one.
+func (env renderEnv) drawRowOf(s *drawstate.State, t *table) int {
+	if env.pick != nil {
+		return t.rows.DrawIn(s, &env.pick.pins)
+	}
+	return t.rows.Draw(s)
+}

family.go

@@ -1,319 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// tablePin is one table's pinned row.
-type tablePin struct {
-	t   *table
-	row int
-}
-
-// pinned is the row the render pinned for t, if any.
-func (d *draws) pinned(t *table) (int, bool) {
-	for _, p := range d.pins[:d.npins] {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	r, ok := d.more[t]
-	return r, ok
-}
-
-// mustRow is the row pinned for t, which the walk reaching a column pinned.
-func (d *draws) mustRow(t *table) int {
-	r, ok := d.pinned(t)
-	if !ok {
-		panic(fmt.Sprintf("fejkdata: a column of %s is rendered with no row pinned", t.category))
-	}
-	return r
-}
-
-// pin pins row r of t, and the rows of t's ancestors it links to.
-func (d *draws) pin(t *table, r int) {
-	for {
-		if _, done := d.pinned(t); done {
-			return
-		}
-		if d.npins < len(d.pins) {
-			d.pins[d.npins] = tablePin{t, r}
-			d.npins++
-		} else {
-			if d.more == nil {
-				d.more = map[*table]int{}
-			}
-			d.more[t] = r
-		}
-		if t.parentT == nil {
-			return
-		}
-		t, r = t.parentT, t.parentRow(r)
-	}
-}
-
-// each calls fn for every pinned row, in pin order, the spilled ones by table name.
-func (d *draws) each(fn func(t *table, r int)) {
-	for _, p := range d.pins[:d.npins] {
-		fn(p.t, p.row)
-	}
-	spilled := make([]*table, 0, len(d.more))
-	for t := range d.more {
-		spilled = append(spilled, t)
-	}
-	sort.Slice(spilled, func(i, j int) bool { return spilled[i].category < spilled[j].category })
-	for _, t := range spilled {
-		fn(t, d.more[t])
-	}
-}
-
-// rowOf is the render's row of t: the one pinned, else one drawn inside the
-// nearest pinned ancestor — its parent drawn inside that first where the ancestor
-// is further up — or over the whole table, and pinned with its ancestors.
-func (d *draws) rowOf(t *table) int {
-	if r, ok := d.pinned(t); ok {
-		return r
-	}
-	r := -1
-	for a := t.parentT; a != nil && r < 0; a = a.parentT {
-		if _, ok := d.pinned(a); !ok {
-			continue
-		}
-		if t.parentT != a {
-			d.rowOf(t.parentT)
-		}
-		pr, _ := d.pinned(t.parentT)
-		r = t.drawUnder(d.s, pr)
-	}
-	if r < 0 {
-		r = t.draw(d.s)
-	}
-	d.pin(t, r)
-	return r
-}
-
-// pinRow pins row r of t where it agrees with the rows pinned before it.
-func (d *draws) pinRow(t *table, r int) error {
-	if pr, ok := d.pinned(t); ok && pr != r {
-		return fmt.Errorf("%s and %s are two rows of %s", t.selectorSpelling(pr), t.selectorSpelling(r), t.path)
-	}
-	for a := t.parentT; a != nil; a = a.parentT {
-		if pa, ok := d.pinned(a); ok && !t.under(r, a, pa) {
-			return fmt.Errorf("%s is not inside %s", t.selectorSpelling(r), a.selectorSpelling(pa))
-		}
-	}
-	d.pin(t, r)
-	return nil
-}
-
-// selectRow pins the row a selector names.
-func (d *draws) selectRow(t *table, sel string) error {
-	r, err := t.find(sel, d)
-	if err != nil {
-		return err
-	}
-	return d.pinRow(t, r)
-}
-
-// inside keeps the rows of t that sit inside every pinned ancestor.
-func (d *draws) inside(t *table, rows []int) []int {
-	for a := t.parentT; a != nil; a = a.parentT {
-		pa, ok := d.pinned(a)
-		if !ok {
-			continue
-		}
-		var kept []int
-		for _, r := range rows {
-			if t.under(r, a, pa) {
-				kept = append(kept, r)
-			}
-		}
-		rows = kept
-	}
-	return rows
-}
-
-// tableRead is what a reference path reads of a table family: the table its head
-// names, the rows its selectors pin, the tables it draws — those it walks with no
-// row pinned, and their unpinned ancestors — each selector's spelling, and whether
-// it lands on a row rendered whole.
-type tableRead struct {
-	head  *table
-	pins  draws
-	drawn map[*table]bool
-	sels  []tableSel
-	whole bool
-}
-
-// tableSel is one selector on the way: the table it selects a row of, and the path
-// as written up to and including it.
-type tableSel struct {
-	t        *table
-	spelling string
-}
-
-// tableReadOf replays a reference's selectors through the walk a render uses, so
-// two paths pinning one row by different routes compare equal. checkPath proved
-// each selector names a row.
-func tableReadOf(head node, a arm, leaf node) *tableRead {
-	t, isTable := head.(*table)
-	if !isTable {
-		return nil
-	}
-	tr := &tableRead{head: t, drawn: map[*table]bool{}}
-	_, _ = walkPath(t, a.tail, pathWalk{pins: &tr.pins})
-	written := a.name[:len(a.name)-len(joinSegments(a.tail))]
-	cur := t
-	tr.draws(cur)
-	for i, seg := range a.tail {
-		switch d := cur.descendant(seg); {
-		case isSelector(seg):
-			tr.sels = append(tr.sels, tableSel{cur, written + joinSegments(a.tail[:i+1])})
-		case d != nil:
-			cur = d
-			tr.draws(cur)
-		}
-	}
-	_, tr.whole = leaf.(*row)
-	return tr
-}
-
-// draws marks t and its ancestors drawn, up to the first the read pins.
-func (r *tableRead) draws(t *table) {
-	for ; t != nil; t = t.parentT {
-		if _, pinned := r.pins.pinned(t); pinned {
-			return
-		}
-		r.drawn[t] = true
-	}
-}
-
-// joinSegments spells segments as a path: a selector attaches to the name before it.
-func joinSegments(segs []string) string {
-	var b strings.Builder
-	for _, s := range segs {
-		if b.Len() > 0 && !isSelector(s) {
-			b.WriteByte('.')
-		}
-		b.WriteString(s)
-	}
-	return b.String()
-}
-
-// family is the table a chain of parents ends at.
-func (t *table) family() *table {
-	for t.parentT != nil {
-		t = t.parentT
-	}
-	return t
-}
-
-// selected is the selector in r on t, or on the nearest ancestor of t it selects.
-func (r *tableRead) selected(t *table) (tableSel, bool) {
-	for ; t != nil; t = t.parentT {
-		for _, s := range r.sels {
-			if s.t == t {
-				return s, true
-			}
-		}
-	}
-	return tableSel{}, false
-}
-
-// checkFamilies refuses reads of one table family in one draw group that cannot
-// read one consistent draw: a table rendered whole beside a path into the family,
-// a table one read draws that another pins, and two reads pinning different rows.
-// The reads come sorted by group and path, so which pair is reported does not vary.
-func checkFamilies(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[:i] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(o.at, r.at) || o.tr.head.family() != r.tr.head.family() {
-				continue
-			}
-			if err := checkFamilyPair(o, r); err != nil {
-				return err
-			}
-		}
-	}
-	return replayPairs(reads)
-}
-
-// replayPairs replays every two reads of one draw group that can render together into
-// one draws, the earlier read first. A pin conflicts with one earlier pin, never with a
-// combination, so pairs find every conflict a full replay would.
-func replayPairs(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[i+1:] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(r.at, o.at) {
-				continue
-			}
-			var d draws
-			if err := r.tr.replay(&d); err != nil {
-				return conflict(r, err)
-			}
-			if err := o.tr.replay(&d); err != nil {
-				return conflict(o, err)
-			}
-		}
-	}
-	return nil
-}
-
-func conflict(r pathRead, err error) error {
-	return fmt.Errorf("%s: %w; select the same rows in every path into the family, or draw them apart with a drawGroup", r.at.route.spelled(r.a.name), err)
-}
-
-// replay pins the read's rows into d, where they agree with the rows pinned before.
-func (r *tableRead) replay(d *draws) error {
-	var err error
-	r.pins.each(func(t *table, row int) {
-		if err == nil {
-			err = d.pinRow(t, row)
-		}
-	})
-	return err
-}
-
-// checkFamilyPair refuses a table read whole beside a path into its family, and a
-// table one read draws that the other pins, since which token renders first would
-// then decide the row.
-func checkFamilyPair(a, b pathRead) error {
-	for _, pair := range [][2]pathRead{{a, b}, {b, a}} {
-		x, y := pair[0], pair[1]
-		if len(x.a.tail) == 0 && len(y.a.tail) > 0 {
-			return overlapError(x.at.route, x.a.name, y)
-		}
-		if drawn := x.tr.drawnOf(&y.tr.pins); drawn != nil {
-			if s, ok := y.tr.selected(x.tr.head); ok && len(x.tr.sels) == 0 {
-				tail := x.a.tail
-				if s.t != x.tr.head {
-					tail = append([]string{x.tr.head.category}, tail...)
-				}
-				return fmt.Errorf("%s draws %s, which %s selects a row of; write {%s.%s}, or draw them apart with a drawGroup",
-					x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name), s.spelling, joinSegments(tail))
-			}
-			return fmt.Errorf("%s draws %s, which %s selects a row of; select that row in both, or draw them apart with a drawGroup",
-				x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name))
-		}
-	}
-	return nil
-}
-
-// drawnOf is a table the read draws that pins holds a row of, if any.
-func (r *tableRead) drawnOf(pins *draws) *table {
-	var found *table
-	pins.each(func(t *table, _ int) {
-		if found == nil && r.drawn[t] {
-			found = t
-		}
-	})
-	return found
-}

fejkdata.go

@@ -1,81 +1,51 @@
-// Package fejkdata generates fake data from recursive JSON templates.
-//
-// Data lives in JSON, not in Go. A generator starts from the shipped data set and
-// layers any directories you add; folders and files become a dot-path namespace,
-// then generate values by path:
-//
-//	f, _ := fejkdata.New(fejkdata.WithSeed(42))
-//	f.Fake("sv_SE.address")          // "Järvedsvägen 43\n891 77 Järved"
-//	f.Fake("sv_SE.address.locality") // "Sundbyberg": a second call draws afresh
-//
-// Several sources merge in order, the last winning a name clash, so custom data
-// layers over the built-ins. The JSON template format is documented in the README.
 package fejkdata
 
 import (
-	crand "crypto/rand"
-	"embed"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io/fs"
-	"math/rand/v2"
-	"os"
 	"reflect"
 	"sort"
 	"sync"
-)
 
-//go:embed data
+	"github.com/larvit/fejkdata/internal/datafiles"
-var shippedFS embed.FS
+	"github.com/larvit/fejkdata/internal/drawstate"
+)
 
 // MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
 // any one path; the CLI's --repeat shares it.
 const MaxRepeat = 1 << 20
 
-var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions
+// Fails to compile where uint is narrower than 64 bits: docs/decisions.md#64-bit-targets-only
+var _ [^uint(0)>>63 - 1]struct{}
 
-// ErrNoData is returned by New when no source is loaded at all.
+// Generator generates fake data from a namespace tree. Create one with [New].
-var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS")
+// It is safe for concurrent use; a sequence is reproducible only when drawn
-
+// from one goroutine.
-// Generator generates fake data from a loaded namespace tree. Create one with [New].
-// It is safe for concurrent use; a seeded sequence is reproducible only when drawn
-// from one goroutine. The compiled tree is immutable after [New], so List reads it
-// without a lock; mu guards the rest, which every other entry point takes.
 type Generator struct {
-	mu         sync.Mutex
+	// mu guards drawState, structs and root, which gains an indexed category on the
-	rand       *session
+	// first call reaching it.
-	categories map[string]node
+	mu        sync.Mutex
-	root       folder  // the categories as the node a path walks from, owned here so a walk allocates none
+	drawState *drawstate.State
-	set        drawSet // one Fake's draws, owned here so a walk pinning rows keeps them off the heap
+	root      folder // the categories as the node a path walks from, owned here so a walk allocates none
-	records    map[node]recordShape
+	structs   map[reflect.Type]structResult
-	structs    map[reflect.Type]structResult
-}
-
-// session is one generator's mutable render state: the seeded rng plus the {seq()}
-// counters.
-type session struct {
-	*rand.Rand
-	counters map[string]uint64
-}
-
-func (s *session) next(key string) uint64 {
-	s.counters[key]++
-	return s.counters[key]
 }
 
 type config struct {
-	seed    uint64
+	seed       uint64
-	seeded  bool
+	seeded     bool
-	shipped bool
+	sources    []datafiles.Source
-	sources []dataSource
+	nilModules error
+	// dataGiven is whether a data option was passed, even WithDataFS(), which loads none.
+	dataGiven bool
 }
 
 // Option configures a [Generator].
 type Option func(*config)
 
-// WithSeed makes output reproducible: two generators with the same seed and data
+// WithSeed seeds the generator, as [New] requires: two generators with the same seed,
-// emit identical sequences.
+// version and data emit identical sequences. For a new seed each run, set
+// seed := rand.Uint64() (math/rand/v2), and log it to replay the run.
 func WithSeed(seed uint64) Option {
 	return func(c *config) { c.seed, c.seeded = seed, true }
 }
@@ -84,175 +54,85 @@ func WithSeed(seed uint64) Option {
 // layer several; the last wins a name clash.
 func WithDataPath(dir string) Option {
 	return func(c *config) {
-		c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir})
+		c.sources, c.dataGiven = append(c.sources, datafiles.Dir(dir)), true
 	}
 }
 
-// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own.
+// WithDataFS layers modules, each a data tree held in an [fs.FS], over what is loaded
-func WithDataFS(fsys fs.FS) Option {
+// before them, in order. A module may be a data package's FS, such as one of
-	return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) }
+// github.com/larvit/fejkdata/data's Modules, or an embed.FS of your own. WithDataFS()
+// loads no data.
+func WithDataFS(modules ...fs.FS) Option {
+	return func(c *config) {
+		for i, fsys := range modules {
+			if fsys == nil {
+				c.nilModules = errors.Join(c.nilModules, fmt.Errorf("fejkdata: WithDataFS module %d is nil", i+1))
+				continue
+			}
+			c.sources = append(c.sources, datafiles.FS(fsys))
+		}
+		c.dataGiven = true
+	}
 }
 
-// WithoutShippedData leaves the shipped data set out, so only the sources given
+var errNoSeed = errors.New(`fejkdata: no seed: add fejkdata.WithSeed(n), where the same n repeats a run's output; for a new seed each run, set n := rand.Uint64() (import "math/rand/v2") and log n to replay the run`)
-// with [WithDataPath] and [WithDataFS] load.
-func WithoutShippedData() Option {
-	return func(c *config) { c.shipped = false }
-}
 
-// New builds a generator from the shipped data set and the options' sources, merged
+var errNoData = errors.New(`fejkdata: no data: add fejkdata.WithDataFS(data.Modules()...) (import "github.com/larvit/fejkdata/data") for all shipped data, WithDataFS/WithDataPath for your own, or WithDataFS() for none`)
-// in order with the last winning a name clash. Each JSON file becomes a category
+
-// named after the file (address.json -> "address") and each subdirectory a
+// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
-// namespace segment. It errors on a missing directory, invalid JSON, invalid data,
+// of an indexed source.
-// or no data at all.
+var ErrLoad = errors.New("data fails to load")
+
+// loadError is a load's error, unchanged in its text, matching ErrLoad.
+type loadError struct{ error }
+
+func (e loadError) Unwrap() error      { return e.error }
+func (loadError) Is(target error) bool { return target == ErrLoad }
+
+// New builds a generator from the options' sources, merged in order with the last
+// winning a name clash; it fails where no [WithSeed] gives the seed, or no [WithDataFS]
+// or [WithDataPath] names the data to load. Each JSON file becomes a category named
+// after the file (address.json -> "address") and each subdirectory a namespace segment.
+// It errors on a missing directory, invalid JSON or invalid data. Given only
+// WithDataFS() with no modules, New loads no data, and the generator renders templates
+// that read none. A source whose .fejkdata.json carries an index, as each shipped
+// module's does, loads each category on the first call reaching it. Data failing to
+// load, in New or on that first call, gives an error matching [ErrLoad].
 func New(opts ...Option) (*Generator, error) {
-	c := config{shipped: true}
+	var c config
 	for _, opt := range opts {
 		opt(&c)
 	}
-	var sources []dataSource
+	if err := c.misuse(); err != nil {
-	if c.shipped {
+		return nil, err
-		sources = append(sources, dataSource{fsys: shippedFS, root: "data"})
 	}
-	sources = append(sources, c.sources...)
+	root, err := loadSources(c.sources)
-	if len(sources) == 0 {
-		return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
-	}
-	cats, err := loadData(sources)
 	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
+		return nil, fmt.Errorf("fejkdata: %w", loadError{err})
 	}
-	rng, err := newRand(c.seed, c.seeded)
+	return &Generator{drawState: drawstate.New(c.seed), root: root}, nil
-	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
-	}
-	return &Generator{rand: rng, categories: cats}, nil
 }
 
-// List returns the sorted dotted paths Fake can render: every category, the dotted
+// List returns the sorted dotted paths Fake renders: each category and every field,
-// fields within a template, and folder segments. A choice consumes no segment, so a
+// column and linked table below it, a direct descent at a time, listing a level
-// path continues through one only where every variant carries it, which is the
+// carrying a repeat but nothing below it. A choice consumes no segment, so a path
-// rule Fake applies too: List is the set of paths Fake accepts.
+// continues through one only where every variant carries it.
 func (f *Generator) List() []string {
-	var out []string
+	f.mu.Lock()
-	for _, name := range sortedNames(f.categories) {
+	defer f.mu.Unlock()
-		for _, p := range paths(f.categories[name]) {
+	out := paths(&f.root, false)
-			out = append(out, join(name, p))
-		}
-	}
 	sort.Strings(out)
 	return out
 }
 
-// paths lists the dot paths addressable from n, relative to it, where "" is n
+// misuse names every mistake in the options, before any data loads.
-// itself. A folder has no value of its own, so it contributes only its children's.
+func (c config) misuse() error {
-func paths(n node) []string {
+	var err error
-	switch n := n.(type) {
+	if !c.dataGiven {
-	case *folder:
+		err = errNoData
-		var out []string
-		for _, name := range sortedNames(n.children) {
-			for _, p := range paths(n.children[name]) {
-				out = append(out, join(name, p))
-			}
-		}
-		return out
-	case *template:
-		out := []string{""}
-		for _, name := range sortedNames(n.fields) {
-			if isRef(name) { // a binding, not a path segment
-				continue
-			}
-			for _, p := range paths(n.fields[name]) {
-				out = append(out, join(name, p))
-			}
-		}
-		return out
-	case *null:
-		return []string{""}
-	case *table:
-		return tablePaths(n)
-	case *column, *row:
-		return []string{""}
-	case *choice:
-		out := []string{""}
-		for p := range n.shared {
-			out = append(out, p)
-		}
-		return out
 	}
-	return nil
+	if !c.seeded {
-}
+		err = errors.Join(err, errNoSeed)
-
+	}
-// tablePaths is a table's columns, then each table linked to it under its name: the
+	return errors.Join(err, c.nilModules)
-// direct descents, a step at a time.
-func tablePaths(t *table) []string {
-	out := append([]string{""}, t.columns...)
-	sort.Strings(out[1:])
-	children := make([]string, 0, len(t.children))
-	for name := range t.children {
-		children = append(children, name)
-	}
-	sort.Strings(children)
-	for _, name := range children {
-		for _, p := range paths(t.children[name]) {
-			out = append(out, join(name, p))
-		}
-	}
-	return out
-}
-
-// sharedPaths is the sub-paths every item carries — the only ones a path may step
-// through a choice to reach. It intersects, bailing as soon as the set
-// is empty, which is immediate for a choice of plain strings.
-func sharedPaths(items []node) map[string]bool {
-	shared := subPaths(items[0])
-	for _, it := range items[1:] {
-		if len(shared) == 0 {
-			return nil
-		}
-		next := subPaths(it)
-		for p := range shared {
-			if !next[p] {
-				delete(shared, p)
-			}
-		}
-	}
-	return shared
-}
-
-// subPaths is paths(n) as a set, without the empty path that means n itself.
-func subPaths(n node) map[string]bool {
-	out := map[string]bool{}
-	for _, p := range paths(n) {
-		if p != "" {
-			out[p] = true
-		}
-	}
-	return out
-}
-
-func join(prefix, name string) string {
-	switch {
-	case prefix == "":
-		return name
-	case name == "":
-		return prefix
-	}
-	return prefix + "." + name
-}
-
-// randomBytes seeds an unseeded generator.
-var randomBytes = crand.Read
-
-func newRand(seed uint64, seeded bool) (*session, error) {
-	var r *rand.Rand
-	if seeded {
-		r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
-	} else {
-		var b [16]byte
-		if _, err := randomBytes(b[:]); err != nil {
-			return nil, fmt.Errorf("seeding from crypto/rand: %w", err)
-		}
-		r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:])))
-	}
-	return &session{Rand: r, counters: map[string]uint64{}}, nil
 }

fejkdata_test.go

@@ -3,11 +3,18 @@ package fejkdata
 import (
 	"errors"
 	"io/fs"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
-	"testing/fstest"
+
+	"github.com/larvit/fejkdata/data"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
+func withShipped() Option { return WithDataFS(data.Modules()...) }
+
 // newGenerator creates a generator over a single data directory, failing on
 // error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
 // on conflicts).
@@ -18,17 +25,28 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator {
 
 func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
 	t.Helper()
-	all := []Option{WithoutShippedData()}
+	var all []Option
 	for _, dir := range dirs {
 		all = append(all, WithDataPath(dir))
 	}
-	f, err := New(append(all, opts...)...)
+	f, err := New(append(append(all, WithSeed(1)), opts...)...)
 	if err != nil {
 		t.Fatalf("New(%q): %v", dirs, err)
 	}
 	return f
 }
 
+// newBareModule loads the shipped module in data/<module> without its manifest, so its
+// categories sit at the root, failing on error.
+func newBareModule(t *testing.T, module string, opts ...Option) *Generator {
+	t.Helper()
+	f, err := New(append([]Option{WithDataFS(withoutManifest{os.DirFS(filepath.Join("data", module))}), WithSeed(1)}, opts...)...)
+	if err != nil {
+		t.Fatalf("New(%s without its manifest): %v", module, err)
+	}
+	return f
+}
+
 // fake generates a value, failing the test on error.
 func fake(t *testing.T, f *Generator, path string) string {
 	t.Helper()
@@ -40,31 +58,32 @@ func fake(t *testing.T, f *Generator, path string) string {
 }
 
 func TestNewMissingDirectory(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath("data/de_DE"))
+	_, err := New(WithSeed(1), WithDataPath("data/de_DE"))
 	if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
 		t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
 	}
 }
 
 func TestNewRejectsAnEmptyDataPath(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath(""))
+	_, err := New(WithSeed(1), WithDataPath(""))
 	if err == nil || !strings.Contains(err.Error(), "empty") {
 		t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
 	}
 }
 
-func TestNewReportsAnEntropyFailure(t *testing.T) {
+func TestNewWithoutASeedFailsNamingWithSeed(t *testing.T) {
-	saved := randomBytes
+	for name, opts := range map[string][]Option{"no option": nil, "only data": {WithDataFS()}} {
-	randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") }
+		if _, err := New(opts...); err == nil || !strings.Contains(err.Error(), "WithSeed(") {
-	defer func() { randomBytes = saved }()
+			t.Errorf("New(%s) = %v, want it refused, naming WithSeed", name, err)
-	_, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}}))
+		}
-	if err == nil || !strings.Contains(err.Error(), "no entropy") {
+	}
-		t.Fatalf("New() without entropy = %v, want the failure reported", err)
+	if _, err := New(WithDataFS(nil)); err == nil || !strings.Contains(err.Error(), "WithSeed(") || !strings.Contains(err.Error(), "module 1 is nil") {
+		t.Errorf("New(WithDataFS(nil)) = %v, want both the missing seed and the nil module named", err)
 	}
 }
 
 func TestWithSeedIsDeterministic(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(42)), newGenerator(t, "data", WithSeed(42))
+	a, b := shipped(t, WithSeed(42)), shipped(t, WithSeed(42))
 	for i := 0; i < 50; i++ {
 		if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
 			t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
@@ -73,7 +92,7 @@ func TestWithSeedIsDeterministic(t *testing.T) {
 }
 
 func TestDifferentSeedsDiffer(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(1)), newGenerator(t, "data", WithSeed(2))
+	a, b := shipped(t, WithSeed(1)), shipped(t, WithSeed(2))
 	for i := 0; i < 50; i++ {
 		if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
 			return
@@ -92,3 +111,43 @@ func fakeTemplate(t *testing.T, f *Generator, s string) string {
 	}
 	return got
 }
+
+// engine builds a seeded generator with no loaded categories, for rendering tests.
+func engine(seed uint64) *Generator {
+	return &Generator{drawState: drawstate.New(seed)}
+}
+
+// parse unmarshals a JSON template fragment into its dynamic form.
+func parse(t *testing.T, s string) jsonvalue.Value {
+	t.Helper()
+	v, err := jsonvalue.Decode([]byte(s))
+	if err != nil {
+		t.Fatalf("parse %q: %v", s, err)
+	}
+	return v
+}
+
+// resolved compiles a JSON fragment and resolves it against no data.
+func resolved(t *testing.T, s string) (node, error) {
+	t.Helper()
+	n, err := compile(parse(t, s))
+	if err != nil {
+		return nil, err
+	}
+	return n, resolveInlineTemplates(inlineNodes(n, "template"), nil)
+}
+
+// compiled parses, compiles and resolves a JSON fragment into a node.
+func compiled(t *testing.T, s string) node {
+	t.Helper()
+	n, err := resolved(t, s)
+	if err != nil {
+		t.Fatalf("compile %q: %v", s, err)
+	}
+	return n
+}
+
+func mustRender(t *testing.T, f *Generator, s string) string {
+	t.Helper()
+	return renderOnce(f.drawState, compiled(t, s))
+}

format.go

@@ -0,0 +1,205 @@
+package fejkdata
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// checkFunc validates a call at compile time: a known builtin, its args, and, for calc, the
+// operands held in fields.
+func checkFunc(tok grammar.Token, fields map[string]node) error {
+	err := builtinfunc.Check(tok.Fn, tok.Args)
+	if err == nil && tok.Fn == builtinfunc.CalcName {
+		err = checkCalcFields(tok.Args, fields)
+	}
+	if err != nil {
+		return fmt.Errorf("token {%s}: %w", tok.Body, err)
+	}
+	return nil
+}
+
+func parseChecked(format string, fields map[string]node) ([]grammar.Token, []unboundRead, error) {
+	toks, err := grammar.ParseFormat(format)
+	if err != nil {
+		return nil, nil, err
+	}
+	unbound, err := checkTokens(toks, fields)
+	return toks, unbound, err
+}
+
+// unboundRead is a token reading a head no field holds, which only a name bound around the
+// template can answer; err is the refusal where none does, and body is the token's.
+type unboundRead struct {
+	head, body string
+	err        error
+}
+
+// checkTokens proves every token's grammar, functions and field paths, and returns the reads
+// whose head no field holds, for bindNames to look up among the names.
+func checkTokens(toks []grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	var unbound []unboundRead
+	for _, t := range toks {
+		u, err := checkToken(t, fields)
+		if err != nil {
+			return nil, err
+		}
+		unbound = append(unbound, u...)
+	}
+	return unbound, nil
+}
+
+func checkToken(t grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	switch t.Kind {
+	case grammar.BuiltinCall:
+		if err := checkFunc(t, fields); err != nil {
+			return nil, err
+		}
+		return checkReads(t, fields, true)
+	case grammar.NameBind:
+		return nil, checkBind(t, fields)
+	case grammar.PathRead:
+		return checkReads(t, fields, false)
+	}
+	return nil, nil
+}
+
+// checkReads proves each name t reads, an arm or an operand, is a reference or a path into a
+// field, returning those whose head no field holds.
+func checkReads(t grammar.Token, fields map[string]node, operands bool) ([]unboundRead, error) {
+	var unbound []unboundRead
+	names := tokenReads(t)
+	for _, name := range names {
+		if grammar.IsRef(name) {
+			if _, _, err := grammar.RefShape(name); err != nil {
+				return nil, fmt.Errorf("token {%s}: %w", t.Body, err)
+			}
+			continue // resolveRefs checks its target once the tree is assembled
+		}
+		missing, err := checkArm(name, fields, !operands && len(names) == 1)
+		if err == nil {
+			continue
+		}
+		err = fmt.Errorf("token {%s}: %w", t.Body, err)
+		if !missing {
+			return nil, err
+		}
+		unbound = append(unbound, unboundRead{head: splitArm(name, nil).head, body: t.Body, err: err})
+	}
+	return unbound, nil
+}
+
+// checkBind proves a {x as name} token is spelled once, binds a reference or a path into a
+// field, and names a valid name that is no option.
+func checkBind(t grammar.Token, fields map[string]node) error {
+	ref, name := t.BoundRef, t.Bound
+	if trimmed := strings.TrimSpace(ref) + grammar.AsWord + strings.TrimSpace(name); trimmed != t.Body {
+		return fmt.Errorf("token {%s}: write {%s}", t.Body, trimmed)
+	}
+	if err := checkBound(ref, fields); err != nil {
+		return fmt.Errorf("token {%s}: %w", t.Body, err)
+	}
+	if name == "" {
+		return fmt.Errorf("token {%s}: a binding names nothing; write {%s as n}", t.Body, ref)
+	}
+	if isOption(name) {
+		return fmt.Errorf("token {%s}: %q is an option and can never be a name; rename it", t.Body, name)
+	}
+	if err := grammar.CheckIdentifier(name); err != nil {
+		return fmt.Errorf("token {%s}: name %w", t.Body, err)
+	}
+	return nil
+}
+
+// checkBound proves what a binding binds is a reference or a path into a field of its binder.
+func checkBound(ref string, fields map[string]node) error {
+	if grammar.IsRef(ref) {
+		_, _, err := grammar.RefShape(ref)
+		return err
+	}
+	missing, err := checkArm(ref, fields, false)
+	if missing {
+		return fmt.Errorf("%w; a binding names a field, or a reference, which starts with /, . or ..", err)
+	}
+	return err
+}
+
+// checkArm validates one sibling name or path against a template's fields.
+// wholeToken says the name is the token's entire body, so {/name} would render
+// the same value and can be offered as the reference spelling. missing says no
+// field holds the head, which a name may yet answer.
+func checkArm(name string, fields map[string]node, wholeToken bool) (missing bool, err error) {
+	a := splitArm(name, nil)
+	if err := checkSegments(a); err != nil {
+		return false, err
+	}
+	field, ok := fields[a.head]
+	if !ok {
+		if a.head == "" {
+			return false, fmt.Errorf("a name is never empty, so this token can name no field")
+		}
+		if isOption(a.head) {
+			return false, fmt.Errorf("%q is an option and can never be a field", a.head)
+		}
+		if len(fields) == 0 {
+			hint := ""
+			if wholeToken && hintableRef(name) {
+				hint = fmt.Sprintf(" — write {/%s} to reference the data", name)
+			}
+			return true, fmt.Errorf("no field %q; a token names a sibling field, and this template has none%s", a.head, hint)
+		}
+		return true, fmt.Errorf("no field %q", a.head)
+	}
+	if err := provePath(field, a.tail, a.head, 0); err != nil {
+		return false, fmt.Errorf("field %q: %w", a.head, err)
+	}
+	return false, nil
+}
+
+// hintableRef reports whether {/name} is a reference the grammar accepts, so the
+// hint never names a spelling that fails too.
+func hintableRef(name string) bool {
+	return grammar.CheckPathIdentifiers(name) == nil
+}
+
+// tokenReads lists the names t reads: a read's arms, or the operands a call's builtin reads.
+func tokenReads(t grammar.Token) []string {
+	if t.Kind == grammar.BuiltinCall {
+		return builtinfunc.Operands(t.Fn, t.Args)
+	}
+	return t.Arms
+}
+
+// formatOps is a compiled format: its ops, and the size of its literal text.
+type formatOps struct {
+	ops  []op
+	grow int
+}
+
+// compileFormat compiles t's parsed format against its resolved references and its names'
+// targets; a {ref as n} token compiles to no op. checkTokens proved every token valid.
+func compileFormat(t *template, targets map[*nameBinding]nameTarget) (formatOps, error) {
+	var c formatOps
+	for _, tok := range t.tokens {
+		o := op{Token: tok}
+		var err error
+		switch tok.Kind {
+		case grammar.LiteralRun:
+			c.grow += len(tok.Lit)
+		case grammar.BuiltinCall:
+			o.call = builtinfunc.Prep(tok.Fn, tok.Args)
+			o.operands, err = t.compileArms(builtinfunc.Operands(tok.Fn, tok.Args), targets)
+		case grammar.PathRead:
+			o.arms, err = t.compileArms(tok.Arms, targets)
+		case grammar.NameBind:
+			continue
+		}
+		if err != nil {
+			return c, fmt.Errorf("token {%s}: %w", tok.Body, err)
+		}
+		c.ops = append(c.ops, o)
+	}
+	return c, nil
+}

format_stability_test.go

@@ -0,0 +1,54 @@
+package fejkdata
+
+import "testing"
+
+// TestSeededOutputIsStable pins the exact bytes a seeded generator emits for each
+// piece of the token grammar, so a change to how a format is scanned or rendered
+// cannot quietly shift the rng stream that reproducibility depends on.
+func TestSeededOutputIsStable(t *testing.T) {
+	dir := writeFiles(t, map[string]string{
+		"alt.json":          `{"format":"{a|b}","a":"A","b":"B"}`,
+		"calc.json":         `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"classes.json":      `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`,
+		"escapes.json":      `{"format":"01Aa#{x}","x":"!"}`,
+		"funcs.json":        `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`,
+		"municipality.json": `{"format":"{name}","rows":"municipality.tsv","key":"code","name":"name","parent":"region"}`,
+		"municipality.tsv":  "code\tname\tregion\n0180\tStockholm\t01\n0184\tSolna\t01\n1280\tMalmö\t12\n1281\tLund\t12\n",
+		"named.json":        `{"format":"{p as q}{q.a}{q.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"nested.json":       `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`,
+		"paths.json":        `{"format":"{p.a}{p.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"ref.json":          `"see {/alt}"`,
+		"refpath.json":      `"{/region.municipality.name} in {/region.name}"`,
+		"region.json":       `{"format":"{name}","rows":"region.tsv","key":"code","name":"name","weight":"people"}`,
+		"region.tsv":        "code\tname\tpeople\n01\tStockholms län\t3\n12\tSkåne län\t1\n",
+		"repeat.json":       `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`,
+		"sums.json":         `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`,
+		"weights.json":      `[{"format":"big","weight":9},"tiny"]`,
+	})
+	want := map[string][]string{
+		"alt":                            {"A", "A", "B", "A"},
+		"calc":                           {"100.00 x 3 = 700.00", "5.00 x 7 = 35.00", "5.00 x 3 = 35.00", "100.00 x 2 = 139.93"},
+		"classes":                        {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"},
+		"escapes":                        {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"},
+		"funcs":                          {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"},
+		"named":                          {"ab", "ab", "AB", "AB"},
+		"nested":                         {"i-73", "i-23", "i-67", "i-85"},
+		"paths":                          {"aB", "aB", "ab", "AB"},
+		"paths.p.a":                      {"A", "a", "a", "A"},
+		"ref":                            {"see A", "see A", "see A", "see B"},
+		"refpath":                        {"Solna in Skåne län", "Solna in Stockholms län", "Solna in Stockholms län", "Stockholm in Stockholms län"},
+		"region.municipality[0184].name": {"Solna", "Solna", "Solna", "Solna"},
+		"region[12].municipality.name":   {"Malmö", "Malmö", "Lund", "Malmö"},
+		"repeat":                         {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"},
+		"sums":                           {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"},
+		"weights":                        {"big", "big", "big", "big"},
+	}
+	for path := range want {
+		f := newGenerator(t, dir, WithSeed(42))
+		for i, expect := range want[path] {
+			if got := fake(t, f, path); got != expect {
+				t.Errorf("%s draw %d = %q, want %q", path, i, got, expect)
+			}
+		}
+	}
+}

format_test.go

@@ -0,0 +1,132 @@
+package fejkdata
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func TestStringIsAFormat(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`"Malmö"`:                  "Malmö",
+		`"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0",
+		`"{{x}}"`:                  "{x}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) {
+		t.Errorf(`render("{digits(3)}") = %q, want three digits`, got)
+	}
+	if _, err := resolved(t, `"{x}"`); err == nil || !strings.Contains(err.Error(), `no field "x"`) {
+		t.Errorf(`compile("{x}") = %v, want a no-field error`, err)
+	}
+}
+
+func TestTextIsLiteral(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" {
+		t.Fatalf("text = %q, want it verbatim", got)
+	}
+}
+
+func TestBraceEscapes(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`{"format":"{{","x":"v"}`:         "{",
+		`{"format":"}}","x":"v"}`:         "}",
+		`{"format":"{{x}}","x":"v"}`:      "{x}",
+		`{"format":"{{{x}}}","x":"v"}`:    "{v}",
+		`{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	for src, want := range map[string]string{
+		`"x}y"`:                      "}}",
+		`"}"`:                        "}}",
+		`{"format":"{a{b}","a":"Q"}`: "'{'",
+		`"{x"`:                       "unterminated",
+	} {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want)
+		}
+	}
+}
+
+func TestTokenSubstitution(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" {
+		t.Fatalf("token = %q, want Eriksson", got)
+	}
+}
+
+func TestAlternationDrawsOneField(t *testing.T) {
+	f := engine(3)
+	seen := map[string]bool{}
+	for i := 0; i < 100; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || len(seen) != 2 {
+		t.Fatalf("alternation produced %v, want both A and B", seen)
+	}
+}
+
+func TestAlternationThreeWay(t *testing.T) {
+	f := engine(4)
+	seen := map[string]bool{}
+	for i := 0; i < 200; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 {
+		t.Fatalf("3-way alternation produced %v, want A, B and C", seen)
+	}
+}
+
+func TestHashIsLiteral(t *testing.T) {
+	f := engine(1)
+	cases := map[string]string{
+		`{"format":"","x":"v"}`:         "",
+		`{"format":"#","x":"v"}`:        "#",
+		`{"format":"##","x":"v"}`:       "##",
+		`{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a",
+		`{"format":"#{x}","x":"v"}`:     "#v",
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Errorf("render(%s) = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestMultibyteFormat(t *testing.T) {
+	// Scanning is rune-aware: multibyte literals coexist with class chars and
+	// tokens without corrupting indices.
+	got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`)
+	if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) {
+		t.Fatalf("multibyte format = %q", got)
+	}
+}
+
+func TestFormatCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{x"`,                       // unterminated brace
+		`"x}y"`,                      // a lone } must be written }}
+		`{"format":"{a{b}","a":"Q"}`, // a brace inside a token
+		`"{x}"`,                      // a bare string has no fields to name
+		`{"format":"{y}","x":"Q"}`,   // token names a missing field
+		`"{}"`,                       // empty token name
+		`{"format":"{a|}","a":"Q"}`,  // empty alternation segment
+		`"{luhn(}"`,                  // malformed function token
+		`"{lower()}"`,                // wrong arity
+		`"{nope()}"`,                 // unknown function
+		`"{luhn(x)}"`,                // function given args it takes none of
+		`"{int(1)}"`,                 // wrong arity
+		`"{float(1,2)}"`,             // wrong arity
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}

go.mod

@@ -1,3 +1,3 @@
-module gitea.larvit.se/larvit/fejkdata
+module github.com/larvit/fejkdata
 
 go 1.22

graph.go

@@ -2,128 +2,99 @@ package fejkdata
 
 import (
 	"fmt"
-	"sort"
+	"slices"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// walkNodes calls fn once per contained node, passing the dot path that reaches it,
+// eachNode visits n and every node inside it once, and passes fn each node with the label an
-// visiting keys in sorted order so which of several broken nodes gets reported does
+// error uses for it: the dot path reaching it, then ", line N" for a table cell. It never
-// not depend on map iteration.
+// crosses a reference edge — a {/path} reference is skipped — so a single inline node is
-func walkNodes(root map[string]node, fn func(path string, n node) error) error {
+// walked on its own.
-	for _, name := range sortedNames(root) {
+func eachNode(n node, label string, fn func(label string, n node) error) error {
-		if err := eachNode(root[name], name, fn); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// eachNode visits n and every node contained within it once, passing the dot path
-// that reaches each. It never crosses a reference edge — a bound {/path} field is
-// skipped — so a single inline node is walked on its own.
-func eachNode(n node, path string, fn func(path string, n node) error) error {
 	seen := map[node]bool{}
 	var visit func(string, node) error
-	visit = func(path string, m node) error {
+	visit = func(label string, m node) error {
 		if m == nil || seen[m] {
 			return nil
 		}
 		seen[m] = true
-		if err := fn(path, m); err != nil {
+		if err := fn(label, m); err != nil {
 			return err
 		}
 		for _, c := range contained(m) {
-			if err := visit(join(path, c.name), c.node); err != nil {
+			if err := visit(c.labelIn(label), c.node); err != nil {
 				return err
 			}
 		}
 		return nil
 	}
-	return visit(path, n)
+	return visit(label, n)
 }
 
-// namedNode is a contained child and the segment reaching it; a choice's items carry
+// containedNode is a contained child and the segment reaching it; a choice's items carry
 // no segment, matching how a dot path steps over a choice.
-type namedNode struct {
+type containedNode struct {
 	name string
 	node node
+	line int // a table cell's line in its rows file, else 0
 }
 
-func contained(n node) []namedNode {
+func (c containedNode) labelIn(label string) string {
+	if c.line > 0 {
+		return fmt.Sprintf("%s, line %d", label, c.line)
+	}
+	return join(label, c.name)
+}
+
+func contained(n node) []containedNode {
 	switch n := n.(type) {
 	case *folder:
-		return named(n.children)
+		return containedByName(n.children)
 	case *choice:
-		out := make([]namedNode, len(n.items))
+		out := make([]containedNode, len(n.items))
 		for i, it := range n.items {
-			out[i] = namedNode{node: it}
+			out[i] = containedNode{node: it}
 		}
 		return out
 	case *template:
-		return named(n.fields)
+		return containedByName(n.fields)
 	case *table:
-		return append([]namedNode{{node: n.format}}, named(n.fields)...)
+		return append([]containedNode{{node: n.formatTemplate}}, containedByName(n.formatTemplate.fields)...)
-	case *column:
+	case *tableColumn:
-		if len(n.t.tokens) == 0 {
+		if len(n.t.cellTemplates) == 0 {
 			return nil
 		}
-		var out []namedNode
+		var out []containedNode
-		for r := 0; r < n.t.rows(); r++ {
+		for r := 0; r < n.t.rows.Len(); r++ {
-			if cell := n.t.cellNode(r, n.i); cell != nil {
+			if cell := n.t.cellTemplate(r, n.i); cell != nil {
-				out = append(out, namedNode{node: cell})
+				out = append(out, containedNode{node: cell, line: r + 2})
 			}
 		}
 		return out
-	default:
+	case *nullItem, *tableRow:
 		return nil
+	default:
+		panic(invariant.Broken("contained has no case for node %T", n))
 	}
 }
 
-// named skips a bound {/path} key: it is a render edge, not containment, so using
+func containedByName(m map[string]node) []containedNode {
-// it as a path segment would report a node under a path that does not reach it. Only
+	out := make([]containedNode, 0, len(m))
-// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a
-// folder's children never carry the prefix — so this one skip serves both.
-func named(m map[string]node) []namedNode {
-	out := make([]namedNode, 0, len(m))
 	for _, name := range sortedNames(m) {
-		if isRef(name) {
+		out = append(out, containedNode{name: name, node: m[name]})
-			continue
-		}
-		out = append(out, namedNode{name: name, node: m[name]})
 	}
 	return out
 }
 
-func sortedNames(m map[string]node) []string {
-	names := make([]string, 0, len(m))
-	for name := range m {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return names
-}
-
 // renderEdge is a child a node renders into, labelled by what reaches it (a field
-// name, reference, or choice index) for a readable cycle report. operand names
+// name, reference, or choice index) for a readable cycle report.
-// the builtin when the label is its operand rather than a token, so an error can
-// name it the way the author wrote it.
 type renderEdge struct {
-	to      node
+	to    node
-	label   string
+	label string
-	operand string
 }
 
-// reached names an edge as the author spelled it, the vocabulary boundReaders uses
+// renderEdges lists the children rendering n recurses into, mirroring render.
-// for the sibling fence.
-func (e renderEdge) reached() string {
-	if e.operand != "" {
-		return fmt.Sprintf("%s operand %q", e.operand, e.label)
-	}
-	return "{" + e.label + "}"
-}
-
-// renderEdges lists the children rendering n recurses into, mirroring expand: a
-// choice's items, and a template's field/reference tokens plus its operands. A
-// folder renders nothing, so it has no edges.
 func renderEdges(n node) []renderEdge {
 	switch n := n.(type) {
 	case *choice:
@@ -134,120 +105,40 @@ func renderEdges(n node) []renderEdge {
 		return es
 	case *template:
 		var es []renderEdge
-		add := func(name, operand string) {
+		for _, o := range n.compiled.ops {
-			a := splitArm(name, n.refs)
+			for _, a := range slices.Concat(o.operands, o.arms) {
-			c, ok := n.fields[a.key]
+				for _, leaf := range a.leaves {
-			if !ok {
+					es = append(es, renderEdge{leaf, a.spelling})
-				return
+				}
-			}
-			for _, leaf := range pathLeaves(c, a.tail) {
-				es = append(es, renderEdge{leaf, name, operand})
 			}
 		}
-		_ = eachToken(n.format, func(t ftoken) error {
-			if t.kind != 'b' {
-				return nil
-			}
-			if fn, _, isFunc := funcCall(t.body); isFunc {
-				for _, operand := range tokenOperands(t.body) {
-					add(operand, fn)
-				}
-				return nil
-			}
-			for _, name := range splitOutside(t.body, '|') {
-				add(name, "")
-			}
-			return nil
-		})
 		return es
 	case *table:
-		return []renderEdge{{to: n.format, label: "format"}}
+		return []renderEdge{{to: n.formatTemplate, label: "format"}}
-	case *row:
+	case *tableRow:
-		return []renderEdge{{to: n.t.format, label: "format"}}
+		return []renderEdge{{to: n.t.formatTemplate, label: "format"}}
-	case *column:
+	case *tableColumn:
 		var es []renderEdge
 		for _, c := range contained(n) {
-			es = append(es, renderEdge{to: c.node, label: n.t.columns[n.i]})
+			es = append(es, renderEdge{to: c.node, label: n.name()})
 		}
 		return es
-	default:
+	case *folder, *nullItem:
 		return nil
+	default:
+		panic(invariant.Broken("renderEdges has no case for node %T", n))
 	}
 }
 
-// pathLeaves lists what a token's dotted tail renders: a choice on the way
+type renderCounts map[node]int
-// contributes every variant, since any of them may be the one drawn. checkPath has
-// already proved the tail resolves in every variant.
-func pathLeaves(n node, tail []string) []node {
-	var out []node
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil },
-		leaf:   func(n node) error { out = append(out, n); return nil },
-	})
-	return out
-}
 
-// nodeScope is the set of nodes one validation pass covers: a whole loaded tree,
+func (m renderCounts) renderCount(n node) int {
-// or a single inline node.
-type nodeScope func(fn func(path string, n node) error) error
-
-func treeScope(root map[string]node) nodeScope {
-	return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
-}
-
-func inlineScope(n node, label string) nodeScope {
-	return func(fn func(path string, m node) error) error { return eachNode(n, label, fn) }
-}
-
-func checkScope(s nodeScope) error {
-	if err := checkColumns(s); err != nil {
-		return err
-	}
-	return checkRenders(s)
-}
-
-// checkRenders runs the per-node fences over a scope, each over the whole scope
-// before the next, so which of several broken nodes is reported does not depend on
-// the walk. It runs after checkNoCycles, whose guarantee is what lets the walks
-// terminate.
-func checkRenders(s nodeScope) error {
-	mem := reachMemo{}
-	if err := s(func(path string, n node) error { return repeatCheck(path, n, mem) }); err != nil {
-		return err
-	}
-	if err := s(heldCheck); err != nil {
-		return err
-	}
-	fence := &drawCheck{}
-	refs := false
-	if err := s(func(path string, n node) error {
-		if t, ok := n.(*template); ok && len(t.refs) > 0 {
-			refs = true
-		}
-		return fence.checkDrawGroup(path, n)
-	}); err != nil {
-		return err
-	}
-	if refs {
-		if err := s(fence.checkDraws); err != nil {
-			return err
-		}
-		if err := s(fence.checkRecordDraws); err != nil {
-			return err
-		}
-	}
-	return s((&valueProof{}).checkDatatype)
-}
-
-type reachMemo map[node]int
-
-func (m reachMemo) of(n node) int {
 	if r, done := m[n]; done {
 		return r
 	}
 	r := 1
 	for _, e := range renderEdges(n) {
-		if c := m.of(e.to); c > r {
+		if c := m.renderCount(e.to); c > r {
 			r = c
 		}
 	}
@@ -260,9 +151,10 @@ func (m reachMemo) of(n node) int {
 
 // repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
 // path, so nested repeats cannot build what one repeat may not.
-func repeatCheck(path string, n node, mem reachMemo) error {
+// docs/decisions.md#the-repeat-cap-bounds-renders-not-bytes
-	if t, ok := n.(*template); ok && t.repeat > 1 && mem.of(n) > MaxRepeat {
+func repeatCheck(label string, n node, mem renderCounts) error {
-		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, mem.of(n), MaxRepeat)
+	if t, ok := n.(*template); ok && t.repeat > 1 && mem.renderCount(n) > MaxRepeat {
+		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", label, t.repeat, mem.renderCount(n), MaxRepeat)
 	}
 	return nil
 }
@@ -270,32 +162,42 @@ func repeatCheck(path string, n node, mem reachMemo) error {
 // checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
 // — directly, mutually, or through a chain — never terminates, so it must fail at
 // New rather than stack-overflow at render. It is a depth-first walk of the render
-// graph (renderEdges); grey marks nodes on the current path so a back-edge to one
+// graph (renderEdges). Every node is a root: a field its parent's format never
-// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped.
+// renders is still reachable by dot path, so a cycle in one would otherwise reach
-// Every node is a root: a field its parent's format never renders is still reachable
+// render and be fatal there.
-// by dot path, so a cycle in one would otherwise reach render and be fatal there.
+func checkNoCycles(s nodeSet) error {
-func checkNoCycles(root map[string]node) error {
 	const (
 		grey  = 1
 		black = 2
 	)
+	inSet := map[node]bool{}
+	_ = s(func(_ string, n node) error {
+		inSet[n] = true
+		if t, isTable := n.(*table); isTable {
+			inSet[t.rowNode] = true // a selector lands on it, and contained lists it nowhere
+		}
+		return nil
+	})
 	color := map[node]int{}
-	var visit func(n node, path string) error
+	var visit func(n node, label string) error
-	visit = func(n node, path string) error {
+	visit = func(n node, label string) error {
+		if !inSet[n] {
+			return nil // loaded before these nodes and proven then, so it never reaches back into it
+		}
 		switch color[n] {
 		case grey:
-			return fmt.Errorf("reference cycle: %s", path)
+			return fmt.Errorf("reference cycle: %s", label)
 		case black:
 			return nil
 		}
 		color[n] = grey
 		for _, e := range renderEdges(n) {
-			if err := visit(e.to, path+" -> "+e.label); err != nil {
+			if err := visit(e.to, label+" -> "+e.label); err != nil {
 				return err
 			}
 		}
 		color[n] = black
 		return nil
 	}
-	return walkNodes(root, func(path string, n node) error { return visit(n, path) })
+	return s(func(label string, n node) error { return visit(n, label) })
 }

hold.go

@@ -1,358 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// heldCheck rejects every route to a held sibling name except the ones that read its
-// draw. An expansion holds one draw of that name; anything else that renders it draws
-// again, and the two disagree. checkNoOverlap settles the spellings within one format
-// (a token, an operand); this settles the rest — a reference, whether it sits in that
-// format or in anything the format renders, however deep. A reference path is held
-// for the whole render instead, which drawCheck fences.
-func heldCheck(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || len(t.held) == 0 {
-		return nil
-	}
-	readers := boundReaders(t.format, t.bound, t.refs)
-	for _, head := range heldHeads(t) {
-		if _, isPath := t.bound[head]; isPath && isRef(head) {
-			continue
-		}
-		if err := checkHeadHeld(t, head, readers); err != nil {
-			return fmt.Errorf("%s: %w", path, err)
-		}
-	}
-	return nil
-}
-
-// heldHeads lists a template's held names, operand heads first, then paths, each
-// in name order: a level read both ways is reported by the operand's fence, and
-// which overlap is reported does not vary.
-func heldHeads(t *template) []string {
-	heads := make([]string, 0, len(t.held))
-	for head := range t.held {
-		heads = append(heads, head)
-	}
-	sort.Slice(heads, func(i, j int) bool {
-		_, pi := t.bound[heads[i]]
-		_, pj := t.bound[heads[j]]
-		if pi != pj {
-			return !pi
-		}
-		return heads[i] < heads[j]
-	})
-	return heads
-}
-
-// pinned collects what the hold of head answers for: a path pins the levels it
-// passes through and the leaf it lands on, an operand exactly the value its render
-// produces.
-func pinned(t *template, head string, readers []reader) map[node]bool {
-	held := map[node]bool{}
-	if _, isPath := t.bound[head]; !isPath {
-		operandDraw(t.fields[head], held)
-		return held
-	}
-	for _, r := range readers {
-		if a := splitArm(r.name, t.refs); a.key == head {
-			coverPath(t.fields[head], a.tail, held)
-		}
-	}
-	return held
-}
-
-// checkHeadHeld rejects every route to what head's hold pins except the readers
-// holding it. One seen set across the edges: a node that cannot reach the level
-// cannot reach it by another route either, so it is walked once.
-func checkHeadHeld(t *template, head string, readers []reader) error {
-	held := pinned(t, head, readers)
-	if len(held) == 0 {
-		return nil // a fixed head holds nothing to reach
-	}
-	reader, isPath := t.bound[head]
-	seen := map[node]bool{}
-	for _, e := range renderEdges(t) {
-		if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) {
-			continue
-		}
-		if isPath {
-			return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader)
-		}
-		return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head))
-	}
-	return nil
-}
-
-// operandReader names the builtin whose operand holds head.
-func operandReader(t *template, head string) string {
-	fn := ""
-	_ = eachToken(t.format, func(tok ftoken) error {
-		if tok.kind != 'b' || fn != "" {
-			return nil
-		}
-		if name, _, isFunc := funcCall(tok.body); isFunc {
-			for _, operand := range tokenOperands(tok.body) {
-				if splitArm(operand, t.refs).key == head {
-					fn = name
-				}
-			}
-		}
-		return nil
-	})
-	return fn
-}
-
-// coverPath collects what holding one path pins: every choice level the path
-// passes through, whole, and the leaf it renders.
-func coverPath(n node, tail []string, into map[node]bool) {
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil },
-		leaf:   func(n node) error { cover(n, into, false); return nil },
-	})
-}
-
-// cover collects a level and everything contained in it. A fixed string outside a
-// choice is left out — it cannot disagree with itself — but inside one each
-// variant carries its own, so there it counts.
-func cover(n node, into map[node]bool, inChoice bool) {
-	if isFixed(n) && !inChoice {
-		return
-	}
-	into[n] = true
-	_, isChoice := n.(*choice)
-	for _, c := range contained(n) {
-		cover(c.node, into, inChoice || isChoice)
-	}
-}
-
-// operandDraw collects what one held draw of an operand answers for: the operand
-// and what rendering it settles inside itself. The builtin renders its operand
-// whole, so that draw fixes every value the render produced, and a second route to
-// any of them disagrees with it.
-//
-// The walk stops at a reference edge, which is where the operand's own value ends
-// and a shared source begins: two names referencing one category are two draws, the
-// same rule {word} {word} follows.
-func operandDraw(n node, into map[node]bool) {
-	if isFixed(n) {
-		return
-	}
-	if into[n] {
-		return
-	}
-	into[n] = true
-	for _, e := range renderEdges(n) {
-		if isRef(e.label) {
-			continue
-		}
-		operandDraw(e.to, into)
-	}
-}
-
-// isFixed is a string that varies nothing: fixed text with no fields to read into.
-func isFixed(n node) bool {
-	t, ok := n.(*template)
-	return ok && t.fixed && len(t.fields) == 0
-}
-
-// renders reports whether rendering n can reach anything in want, following the
-// same edges expand does. seen keeps a node shared by several routes from being
-// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk
-// ends.
-func renders(n node, want, seen map[node]bool) bool {
-	if want[n] {
-		return true
-	}
-	if seen[n] {
-		return false
-	}
-	seen[n] = true
-	for _, e := range renderEdges(n) {
-		if renders(e.to, want, seen) {
-			return true
-		}
-	}
-	return false
-}
-
-// checkNoOverlap rejects a format that both renders a sibling level and reads a path
-// into it — {p} beside {p.first}, {p.addr} beside {p.addr.city}. The path reads the
-// level's held draw while rendering the level expands it afresh, so their values would
-// disagree. Reads are compared in sorted order, so which pair is reported does not
-// depend on where the tokens sit.
-func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
-	names := boundReaders(format, bound, refs)
-	// Stable over one format-order scan, so two readers of one name (a token and a
-	// calc operand both naming "p") are reported as the format writes them.
-	sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
-	for i, level := range names {
-		for _, path := range names[i+1:] {
-			if strings.HasPrefix(path.path, level.path+".") {
-				return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name)
-			}
-		}
-	}
-	return nil
-}
-
-// reader is one way a format reaches a bound field: as written, by its one
-// spelling, and how to name it.
-type reader struct{ name, path, label string }
-
-// boundReaders lists every way a format reaches a bound sibling field, in the order the
-// format writes them. An operand renders its field, so it names a level exactly
-// as a token does; one scan finds both, which is what puts them in one order.
-func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
-	var names []reader
-	_ = eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if fn, _, isFunc := funcCall(t.body); isFunc {
-			for _, operand := range tokenOperands(t.body) {
-				a := splitArm(operand, refs)
-				if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-					names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
-				}
-			}
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-				names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
-			}
-		}
-		return nil
-	})
-	return names
-}
-
-// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside
-// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
-// error names the single-token spelling.
-func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
-	count := map[string]int{}
-	return eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if _, _, isFunc := funcCall(t.body); isFunc {
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if len(a.tail) > 0 || !c.held[a.key] {
-				continue
-			}
-			if count[a.key]++; count[a.key] > 1 {
-				return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
-			}
-		}
-		return nil
-	})
-}
-
-// draws is what has already been drawn for held names: the variant each was drawn as,
-// so every path under it reads one row, and the draw each read made, by its one
-// spelling, so the same read written twice reads one draw. An expansion keeps one for
-// its sibling names, and a render one per group for its reference paths.
-type draws struct {
-	variant map[string]node
-	value   map[string]draw
-	pins    [8]tablePin // the rows pinned, inline so a render over a country's five-deep geo tree stays off the heap
-	npins   int
-	more    map[*table]int // the rows pinned past the inline eight
-	s       *session       // what draws a row; nil where a walk only proves selectors
-}
-
-// draw is what one read drew: its text, and whether it landed on a null.
-type draw struct {
-	text string
-	null bool
-}
-
-// readField renders one arm of a token. An arm's key is a sibling field or a
-// reference linkRefs bound into fields. A name the expansion holds — a level some
-// token addresses by dotted path, or a field an operand reads — is drawn once and
-// kept, so {place.postal-code} and {place.locality} read one row, either read twice
-// gives one value, and a shown operand is the operand computed. Every other name is
-// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
-// linkRefs prove every step, so the walk cannot fail.
-func readField(s *session, t *template, held *draws, sc drawScope, a arm) draw {
-	if !t.held[a.key] {
-		if len(a.tail) > 0 {
-			panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
-		}
-		return draw{text: render(s, t.fields[a.key], sc)}
-	}
-	d := readScope(s, held, sc, a)
-	if r, done := d.value[a.path]; done {
-		return r
-	}
-	leaf, err := walkPath(t.fields[a.key], a.tail, pathWalk{
-		// Hold the draw at every level passed through, so two paths sharing a
-		// prefix share it.
-		choice: func(c *choice, rest []string) ([]node, error) {
-			key := a.key
-			if consumed := len(a.tail) - len(rest); consumed > 0 {
-				key = a.steps[consumed-1]
-			}
-			n, drew := d.variant[key]
-			if !drew {
-				n = drawn(s, c)
-				if d.variant == nil {
-					d.variant = map[string]node{}
-				}
-				d.variant[key] = n
-			}
-			return []node{n}, nil
-		},
-		pins: d,
-	})
-	if err != nil {
-		panic(fmt.Sprintf("fejkdata: %q: %v; a fence should have refused this at New", a.name, err))
-	}
-	r := renderLeaf(s, leaf, sc)
-	if d.value == nil {
-		d.value = map[string]draw{}
-	}
-	d.value[a.path] = r
-	return r
-}
-
-// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
-// the render's draws for its group, so its draw spans the render; for a sibling, or a
-// reference read whole, held.
-func readScope(s *session, held *draws, sc drawScope, a arm) *draws {
-	if isRef(a.key) && len(a.tail) > 0 {
-		return sc.draws(s)
-	}
-	return held
-}
-
-// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
-// reference alone whose read drew null.
-func renderLeaf(s *session, n node, sc drawScope) draw {
-	n = drawn(s, n)
-	if _, isNull := n.(*null); isNull {
-		return draw{null: true}
-	}
-	r := draw{text: render(s, n, sc)}
-	if t, _ := n.(*template); t != nil && t.readsColumn != nil {
-		r.null = sc.in(t).draws(s).value[t.readsColumn.a.path].null
-	}
-	return r
-}
-
-// drawn resolves a choice to one variant, so a bound head is a concrete node the
-// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not
-// another set to pick from.
-func drawn(s *session, n node) node {
-	for c, ok := n.(*choice); ok; c, ok = n.(*choice) {
-		n = pick(s, c)
-	}
-	return n
-}

hold_test.go

@@ -1,771 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-)
-
-// This file pins the one-draw rule for dotted sibling tokens. A format that
-// addresses a sibling by path ({place.locality}) draws that sibling once per
-// expansion, so every {place.*} token in the same format reads the same variant.
-// That is what makes a correlated pair — a locality and a postal code that agree
-// — expressible in data, without the row having to be the whole node.
-
-// swedishPlaces is a two-variant sibling whose variants pair a locality with the
-// postal-code prefix that really belongs to it.
-const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`
-
-// agree reports whether a rendered "postcode locality" pair is a real pairing.
-var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`)
-
-func places(format string) string {
-	return strings.Replace(swedishPlaces, "%s", format, 1)
-}
-
-func TestDottedTokenCorrelatesSiblingFields(t *testing.T) {
-	f := engine(1)
-	seen := map[string]bool{}
-	for i := 0; i < 500; i++ {
-		got := mustRender(t, f, places("{place.postal-code} {place.locality}"))
-		if !agree.MatchString(got) {
-			t.Fatalf("draw %d = %q, want a postcode and locality that pair", i, got)
-		}
-		seen[got[len(got)-4:]] = true
-	}
-	if len(seen) < 2 {
-		t.Fatalf("500 draws produced only %v, want both localities", seen)
-	}
-}
-
-func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) {
-	// One draw, one spelling. A token that renders a level and one that reads a
-	// path into it name the same draw two ways, and only the path reads what the
-	// other rendered — so the pair is a load error rather than a shape whose two
-	// halves can disagree.
-	rejected := map[string]string{
-		"bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-		"prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` +
-			`{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`,
-		"either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-	}
-	for name, file := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), "reads a path into") {
-			t.Errorf("%s: New = %v, want it to name the overlap", name, err)
-		}
-	}
-	// The same path twice is one spelling, so it stays legal.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-	}))); err != nil {
-		t.Errorf("New = %v, want one path read twice accepted", err)
-	}
-}
-
-func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) {
-	// A calc operand renders its field, so naming a bound level in one is the same
-	// overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a
-	// path into, and the two disagree.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` +
-			`{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-		t.Fatalf("New = %v, want the calc operand rejected as an overlap", err)
-	}
-	// Arithmetic over fields no path names is untouched.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`,
-	}))); err != nil {
-		t.Errorf("New = %v, want plain arithmetic accepted", err)
-	}
-}
-
-func TestReferenceNamingABoundLevelIsRejected(t *testing.T) {
-	// A reference can name a bound level from the data root, which renders it
-	// afresh beside the path that reads its held draw — the same overlap by
-	// another spelling.
-	// A category never references itself, so each reference back into cat sits in a
-	// second category cat renders.
-	rejected := map[string]map[string]string{
-		"reference names the head": {
-			"cat": `{"format":"{p.first}|{/hop}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-			"hop": `"{/cat.p}"`,
-		},
-		"reference names the leaf": {
-			"cat": `{"format":"{p.addr}|{/hop}","p":{"format":"x","addr":["A","B","C","D"]}}`,
-			"hop": `"{/cat.p.addr}"`,
-		},
-		// The reference need not sit in the format that binds: any field it renders
-		// reaches the level just the same, however deep.
-		"reference from a sibling field": {
-			"cat": `{"format":"{p.first}|{inner}","p":[` +
-				`{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` +
-				`"inner":"{/hop}"}`,
-			"hop": `"{/cat.p}"`,
-		},
-	}
-	for name, files := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
-		if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-			t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err)
-		}
-	}
-	// A reference to anything this format does not bind is untouched.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":     `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-		"surname": `["Eriksson","Lindqvist"]`,
-	}))); err != nil {
-		t.Errorf("New = %v, want a reference outside the bound level accepted", err)
-	}
-}
-
-func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) {
-	// A path token renders what it lands on, not the level it started from, so the
-	// cycle walk has to follow it there. A head whose own format names nothing
-	// would otherwise hide the cycle until render, where it is fatal.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"a": `{"format":"{p.x}","p":{"format":"static","x":"{/b}"}}`,
-		"b": `"{/a}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reference cycle") {
-		t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
-	}
-}
-
-func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) {
-	// {p.a} renders q, so it is a route to the level {q.x} holds — even though p's
-	// own format names nothing.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/hop}"},` +
-			`"q":{"format":"{x}","x":["1","2"]}}`,
-		"hop": `"{/thing.q}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-		t.Fatalf("New = %v, want the second route to q rejected", err)
-	}
-}
-
-// TestACalcOperandIsHeldAgainstEveryRoute pins what a {calc()} operand's hold
-// fences. A calc renders its operand whole, so the draw it holds is that one node's
-// value: another route conflicts only by naming that same node, and then the value
-// shown is not the value computed. Two names that merely draw from one source are
-// two draws, as {word} {word} is — the rule everywhere else in the engine.
-func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) {
-	rejected := map[string]struct {
-		files map[string]string
-		want  string // the route the error names, spelled as the author wrote it
-	}{
-		// A category never references itself, so each route back into cat sits in a
-		// second category cat renders.
-		"a reference beside the operand": {
-			map[string]string{
-				"cat": `{"format":"{/hop} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
-				"hop": `"{/cat.net}"`,
-			},
-			`{/hop} renders "net"`,
-		},
-		"a reference one level down": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.net}"`,
-			},
-			`{q} renders "net"`,
-		},
-		// A reference ending at the operand binds the choice itself, not a variant,
-		// so wrapping the operand in one changes nothing.
-		"a reference to an operand wrapped in a choice": {
-			map[string]string{
-				"cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.n}"`,
-			},
-			`{q} renders "n"`,
-		},
-		// The reaching route can be another operand, which is not a token — so the
-		// error must name it as one rather than invent a {b} the format never wrote.
-		"an operand reaching another operand": {
-			map[string]string{
-				"cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` +
-					`"b":"{/hop}"}`,
-				"hop": `"{/cat.a}"`,
-			},
-			`calc operand "b" renders "a"`,
-		},
-		// Reaching *into* the operand is the same shape as naming it: the draw the
-		// calc holds settled that value too. The token spelling {net.v} is already
-		// rejected, so the reference spelling has to be.
-		"a reference into the operand": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)}|{/hop}",` +
-					`"net":{"format":"{v}","v":["10.00","20.00"]}}`,
-				"hop": `"{/cat.net.v}"`,
-			},
-			`{/hop} renders "net"`,
-		},
-		"a reference into the operand one level down": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.net.v}"`,
-			},
-			`{q} renders "net"`,
-		},
-		// Each head is walked against its own seen set. Sharing one across heads
-		// would let the first head's walk mark the only route to the second's draw,
-		// and this violation sits on the later head in sorted order.
-		"a violation on the second of two held heads": {
-			map[string]string{
-				"cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` +
-					`"b":{"format":"{y}","y":["3","4"]},"w":"{/hop}"}`,
-				"hop": `"{/cat.b}"`,
-			},
-			`{w} renders "b"`,
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
-		if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") {
-			t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err)
-			continue
-		}
-		if !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to name %q", name, err, c.want)
-		}
-	}
-
-	accepted := map[string]map[string]string{
-		// The spellings that read the one draw: the operand itself, and a bare token.
-		"only the operand and a bare token": {
-			"cat": `{"format":"{net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
-		},
-		// A sibling the operand's format never renders is no part of its value. A
-		// path head differs — a path may read into anything the level contains —
-		// which is why that half of the fence covers containment.
-		"a reference to a sibling the operand never renders": {
-			"cat": `{"format":"{calc(net * 2, 2)} {unit}",` +
-				`"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` +
-				`"unit":"{/hop}"}`,
-			"hop": `"{/cat.net.spare}"`,
-		},
-		// Two operands drawing from one source are two names, so two draws: each is
-		// held under its own name and shown once, and neither can disagree.
-		"two operands sharing one source": {
-			"die": `["1","2","3","4","5","6"]`,
-			"cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` +
-				`"d2":"{/die}"}`,
-		},
-		// Likewise a reference drawing from what the operand draws from: {/common}
-		// and net are two names, not two spellings of one field.
-		"a reference to what an operand renders through": {
-			"common": `["1","2"]`,
-			"cat":    `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`,
-		},
-		// A fixed string cannot disagree with itself, so it needs no fence.
-		"a literal operand named twice": {
-			"cat": `{"format":"{calc(n * 2, 0)} {/hop}","n":"5"}`,
-			"hop": `"{/cat.n}"`,
-		},
-		// One node reached twice while walking the operand: the walk must not
-		// revisit it, and the repeat is not a second route to anything.
-		"an operand that renders one field twice": {
-			"cat": `{"format":"{calc(v * 2, 0)}","v":{"format":"{a}{a}","a":["1","2"]}}`,
-		},
-	}
-	for name, files := range accepted {
-		if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-			t.Errorf("%s: New = %v, want it accepted", name, err)
-		}
-	}
-}
-
-func TestAPathReachesEveryVariantItMightDraw(t *testing.T) {
-	// A path through a choice may land in any variant, so both walks have to see
-	// all of them. Reaching only the first leaves whatever hides in a later
-	// variant to be found at render — a cycle there is fatal, and a second route
-	// to a held level disagrees silently.
-	rejected := map[string]struct {
-		files map[string]string
-		want  string
-	}{
-		"a cycle in a later variant": {
-			map[string]string{
-				"cat": `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}]}`,
-				"hop": `"{/cat}"`,
-			},
-			"reference cycle",
-		},
-		"a second route in a later variant": {
-			map[string]string{
-				"cat": `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}],` +
-					`"q":{"format":"{y}","y":["1","2"]}}`,
-				"hop": `"{/cat.q}"`,
-			},
-			"reads a path into",
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
-		if err == nil || !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
-		}
-	}
-}
-
-func TestALevelAPathNeverRendersIsAccepted(t *testing.T) {
-	// A path token does not expand its head's format, so a reference sitting in
-	// that format is not a second route to anything: it is never rendered by the
-	// path at all. Both orders must load. p and q sit one level down, so they are
-	// one column of thing rather than two that would read one another.
-	accepted := map[string]map[string]string{
-		"reference in the head's own format": {
-			"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
-				`"p":{"format":"{first} {/hop}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}}`,
-			"hop": `"{/thing.inner.q}"`,
-		},
-		"the mirror shape": {
-			"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
-				`"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/hop}","a":["1","2"]}}}`,
-			"hop": `"{/thing.inner.p}"`,
-		},
-	}
-	for name, files := range accepted {
-		if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-			t.Errorf("%s: New = %v, want it accepted", name, err)
-		}
-	}
-}
-
-func TestADeepDiamondChainLoads(t *testing.T) {
-	// A diamond chain is 2^n routes through n nodes. The search past a bound level
-	// must walk each node once, not once per route, or a deep chain never loads.
-	files := map[string]string{"l0": `"x"`}
-	for i := 1; i <= 30; i++ {
-		files[fmt.Sprintf("l%d", i)] = fmt.Sprintf(
-			`{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1)
-	}
-	files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}`
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-		t.Fatalf("New = %v, want a deep diamond chain to load", err)
-	}
-}
-
-func TestASharedNodeIsWalkedOnce(t *testing.T) {
-	// Two fields reaching one node make the render graph a diamond, not a tree.
-	// The search past a bound level must take that in its stride rather than walk
-	// the shared node once per route.
-	f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` +
-			`"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`,
-		"shared": `"x"`,
-	})), WithSeed(1))
-	if err != nil {
-		t.Fatalf("New = %v, want a shared node accepted", err)
-	}
-	if got, err := f.Fake("cat"); err != nil || !strings.HasSuffix(got, "|xx") {
-		t.Fatalf("Fake(cat) = %q, %v, want it to end in xx", got, err)
-	}
-}
-
-func TestTheEarlierReaderIsNamed(t *testing.T) {
-	// A token and a calc operand can name one level. The one the format writes
-	// first is the one reported, so the error points at the same place a reader
-	// looking at the format would start.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) {
-		t.Fatalf("New = %v, want the calc operand named, being written first", err)
-	}
-}
-
-func TestReferenceToAMatchingStringIsAccepted(t *testing.T) {
-	// A literal renders one fixed string, so no draw of it can disagree with a
-	// held one. Two unrelated literals that merely spell the same text must not
-	// read as the same node — the trap when comparing a value type.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":   `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`,
-		"other": `{"format":"x","tag":"Stockholm"}`,
-	}))); err != nil {
-		t.Fatalf("New = %v, want a reference to a matching string accepted", err)
-	}
-}
-
-func TestNestedChoiceDrawsOneVariant(t *testing.T) {
-	// A choice item may itself be a choice, so drawing a bound head unwraps until
-	// it reaches a value. Stopping at one level leaves a choice where the walk
-	// expects a template.
-	f := engine(21)
-	seen := map[string]bool{}
-	for i := 0; i < 200; i++ {
-		seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true
-	}
-	if !seen["1"] || !seen["2"] || len(seen) != 2 {
-		t.Fatalf("200 draws produced %v, want 1 and 2", seen)
-	}
-}
-
-func TestRepeatingLevelIsNamedInTheError(t *testing.T) {
-	// The level carrying the repeat is the one to fix, so the error names it
-	// rather than the head the path started from.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `"p.a"`) {
-		t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err)
-	}
-}
-
-func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) {
-	// A choice of rows is the shape this feature is for, so the repeat rule has to
-	// reach inside one — otherwise the direct spelling is a load error and the same
-	// mistake behind a choice silently drops the repeat.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "repeat") {
-		t.Fatalf("New = %v, want the repeat behind a choice rejected", err)
-	}
-}
-
-func TestPathIntoARepeatingLevelIsRejected(t *testing.T) {
-	// A path reads one level's draw, so it can never apply that level's repeat.
-	// The engine rejects options that cannot take effect, and this is one.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "repeat") {
-		t.Fatalf("New = %v, want a path into a repeating level rejected", err)
-	}
-	_, err = New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":  `{"format":"[{p.a}]","p":{"format":"{a}","a":"{/word.w}","drawGroup":"g"}}`,
-		"word": `{"format":"{w}","w":["x","y"]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `the level "p" carries a drawGroup`) {
-		t.Fatalf("New = %v, want a path into a level carrying a group rejected", err)
-	}
-}
-
-func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) {
-	// A head that is one template, not a choice, reports the missing segment
-	// directly — there are no variants to compare.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `no field "nope"`) {
-		t.Fatalf("New = %v, want it to name the missing field", err)
-	}
-}
-
-func TestFieldWithoutAPathTokenStillDrawsEachTime(t *testing.T) {
-	// A format with no dotted token keeps today's behaviour: every {word} is an
-	// independent draw.
-	f := engine(3)
-	varied := false
-	for i := 0; i < 200 && !varied; i++ {
-		got := mustRender(t, f, `{"format":"{w}{w}{w}","w":["a","b","c","d","e"]}`)
-		varied = got[0] != got[1] || got[1] != got[2]
-	}
-	if !varied {
-		t.Fatal("200 draws of {w}{w}{w} never varied, want independent draws")
-	}
-}
-
-func TestTwoBoundHeadsDrawIndependently(t *testing.T) {
-	// Binding is per head, so two correlated sets in one format compose instead
-	// of needing a cross product of variants.
-	f := engine(4)
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, `{"format":"{p.v}{q.v}",
-			"p":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
-			"q":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}`)] = true
-	}
-	for _, want := range []string{"A1", "A2", "B1", "B2"} {
-		if !seen[want] {
-			t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
-		}
-	}
-}
-
-func TestRepeatRedrawsTheBinding(t *testing.T) {
-	// The binding lives for one expansion, so each repetition is a fresh draw —
-	// and each repetition is internally consistent.
-	f := engine(5)
-	tmpl := strings.Replace(places("{place.postal-code} {place.locality}"), `"format":"{place.postal-code} {place.locality}"`,
-		`"format":"{place.postal-code} {place.locality}","repeat":6,"separator":"\n"`, 1)
-	varied := false
-	for i := 0; i < 40; i++ {
-		got := mustRender(t, f, tmpl)
-		lines := strings.Split(got, "\n")
-		if len(lines) != 6 {
-			t.Fatalf("repeat 6 produced %d lines", len(lines))
-		}
-		for _, line := range lines {
-			if !agree.MatchString(line) {
-				t.Fatalf("repetition %q does not pair", line)
-			}
-		}
-		if lines[0] != lines[1] || lines[1] != lines[2] {
-			varied = true
-		}
-	}
-	if !varied {
-		t.Fatal("40 renders of repeat 6 never varied within a render, want a redraw per repetition")
-	}
-}
-
-func TestNestedTemplateKeepsItsOwnBinding(t *testing.T) {
-	// A binding belongs to the expansion that made it: an inner template's own
-	// place is not the outer one's.
-	f := engine(6)
-	tmpl := `{"format":"{place.v}{inner}",
-		"place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
-		"inner":{"format":"{place.v}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}}`
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, tmpl)] = true
-	}
-	if !seen["AB"] && !seen["BA"] {
-		t.Fatalf("400 draws produced %v, want the inner binding to be independent", seen)
-	}
-}
-
-func TestBoundDrawIsSeedStable(t *testing.T) {
-	tmpl := places("{place.postal-code} {place.locality}")
-	a := mustRender(t, engine(42), tmpl)
-	b := mustRender(t, engine(42), tmpl)
-	if a != b {
-		t.Fatalf("same seed gave %q and %q, want an identical draw", a, b)
-	}
-}
-
-func TestDottedTokenErrors(t *testing.T) {
-	// A path token is validated where every other token is: at New, naming what
-	// is wrong, so a typo can never reach a render.
-	rejected := map[string]struct {
-		file string
-		want string
-	}{
-		"unknown head": {
-			`{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`,
-			`no field "nope"`,
-		},
-		"unknown tail": {
-			`{"format":"{place.nope}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}`,
-			`"nope"`,
-		},
-		"tail only some variants carry": {
-			`{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`,
-			`locality`,
-		},
-		"path into a literal": {
-			`{"format":"{x.y}","x":"plain"}`,
-			`"y"`,
-		},
-		"dotted field key": {
-			`{"format":"[{a.b}]","a.b":"V"}`,
-			`contains "."`,
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want it rejected at load", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
-		}
-	}
-}
-
-func TestSamePathReadTwiceReadsOneValue(t *testing.T) {
-	// A path is drawn once per expansion, so reading it twice reads one value —
-	// what a name shown in a display form and again in an address needs.
-	f := engine(7)
-	for i := 0; i < 300; i++ {
-		got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`)
-		parts := strings.Split(got, "|")
-		if parts[0] != parts[1] {
-			t.Fatalf("draw %d = %q, want one value read twice", i, got)
-		}
-	}
-}
-
-func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) {
-	// Two tails of one head stay in the same variant, each keeping its own value.
-	f := engine(8)
-	for i := 0; i < 300; i++ {
-		got := mustRender(t, f, `{"format":"{p.a}{p.b}",
-			"p":[{"format":"x","a":"A","b":"1"},{"format":"y","a":"B","b":"2"}]}`)
-		if got != "A1" && got != "B2" {
-			t.Fatalf("draw %d = %q, want a row's own pair", i, got)
-		}
-	}
-}
-
-// deepPlaces nests the correlated facts a level down: the row holds an addr, and
-// the addr holds the city and the zip that belongs to it.
-const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}`
-
-func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) {
-	// A choice below the head is drawn once too, so two paths sharing a prefix
-	// share every level of it — not just the row they started from.
-	f := engine(11)
-	seen := map[string]bool{}
-	for i := 0; i < 500; i++ {
-		got := mustRender(t, f, strings.Replace(deepPlaces, "%s", "{p.addr.city}/{p.addr.zip}", 1))
-		if got != "Stockholm/11111" && got != "Kiruna/98100" {
-			t.Fatalf("draw %d = %q, want a city and the zip that belongs to it", i, got)
-		}
-		seen[got] = true
-	}
-	if len(seen) < 2 {
-		t.Fatalf("500 draws produced only %v, want both", seen)
-	}
-}
-
-func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) {
-	// The rule does not run out at two: every level a path passes through is held.
-	f := engine(12)
-	tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}`
-	want := map[string]bool{
-		"Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true,
-		"Malmö/21100 Skåne": true, "Lund/22100 Skåne": true,
-	}
-	seen := map[string]bool{}
-	for i := 0; i < 600; i++ {
-		got := mustRender(t, f, tmpl)
-		if !want[got] {
-			t.Fatalf("draw %d = %q, want a town with its own zip and region", i, got)
-		}
-		seen[got] = true
-	}
-	if len(seen) != len(want) {
-		t.Fatalf("600 draws produced %v, want all four towns", seen)
-	}
-}
-
-func TestPathWalksVariantsOfDifferentShape(t *testing.T) {
-	// One variant holds addr as a plain template where another holds a choice.
-	// Both carry the path, so both must walk — the walk reads what it drew, never
-	// assuming a shape.
-	f := engine(13)
-	tmpl := `{"format":"{p.addr.city}/{p.addr.zip}","p":[
-		{"format":"{addr}","addr":{"format":"{city}","city":"Kiruna","zip":"98100"}},
-		{"format":"{addr}","addr":[
-			{"format":"{city}","city":"Malmö","zip":"21100"},
-			{"format":"{city}","city":"Lund","zip":"22100"}]}]}`
-	for i := 0; i < 400; i++ {
-		got := mustRender(t, f, tmpl)
-		if got != "Kiruna/98100" && got != "Malmö/21100" && got != "Lund/22100" {
-			t.Fatalf("draw %d = %q, want a city with its own zip", i, got)
-		}
-	}
-}
-
-func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) {
-	// Two paths share only what they actually share: a common prefix is one draw,
-	// and each keeps its own draw past the point they part.
-	f := engine(14)
-	tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}`
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, tmpl)] = true
-	}
-	for _, want := range []string{"11", "12", "21", "22"} {
-		if !seen[want] {
-			t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
-		}
-	}
-}
-
-func TestEmptyPathSegmentIsRejected(t *testing.T) {
-	// "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is
-	// reported as that rather than as a missing field. An empty name is rejected
-	// where it is authored, so no data can make these resolve.
-	rejected := map[string]string{
-		"trailing dot": `{"format":"[{a.}]","a":"x"}`,
-		"double dot":   `{"format":"[{a..b}]","a":"x"}`,
-	}
-	for name, file := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want the unfinished path rejected", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), "empty segment") {
-			t.Errorf("%s: New = %v, want it to name the empty segment", name, err)
-		}
-	}
-}
-
-func TestCycleThroughAPathTokenIsRejected(t *testing.T) {
-	// A path token is a render edge like any other, so a cycle routed through one
-	// must be caught at New. Reaching render would be fatal: the recursion never
-	// terminates, and a stack overflow cannot be recovered.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/b}"}}`,
-		"b": `"{/a}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reference cycle") {
-		t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
-	}
-}
-
-func TestBoundPathIsReachableByFake(t *testing.T) {
-	// Binding changes how a format reads a sibling, not what List and Fake offer:
-	// the sub-fields stay addressable on their own.
-	dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")})
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9))
-	if err != nil {
-		t.Fatalf("New = %v", err)
-	}
-	for _, path := range []string{"address", "address.place", "address.place.locality", "address.place.postal-code"} {
-		if _, err := f.Fake(path); err != nil {
-			t.Errorf("Fake(%q) = %v, want it to render", path, err)
-		}
-	}
-}
-
-func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) {
-	for src, want := range map[string]string{
-		`{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`:                                    "write {w} once",
-		`{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`:                    "write {w} once",
-		`{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`:                                 "write {n} once",
-		`{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once",
-	} {
-		_, err := compile(parse(t, src))
-		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") {
-			t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want)
-		}
-	}
-	for _, ok := range []string{
-		`{"format":"{w} {w}","w":["a","b"]}`,
-		`{"format":"{w} {uppercase(w)}","w":["a","b"]}`,
-		`{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`,
-		`{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`,
-	} {
-		if _, err := compile(parse(t, ok)); err != nil {
-			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
-		}
-	}
-}
-
-func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
-	tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template)
-	if !ok {
-		t.Fatal("not a template")
-	}
-	mustPanic(t, "unheld arm with a path", func() {
-		readField(engine(1).rand, tm, nil, drawScope{}, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
-	})
-}

imports_test.go

@@ -0,0 +1,120 @@
+package fejkdata
+
+import (
+	"errors"
+	"go/build"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// TestPackageImports holds each package in the module to the module packages it may import.
+func TestPackageImports(t *testing.T) {
+	mod, err := os.ReadFile("go.mod")
+	if err != nil {
+		t.Fatal(err)
+	}
+	module := ""
+	for _, line := range strings.Split(string(mod), "\n") {
+		if f := strings.Fields(line); len(f) > 1 && f[0] == "module" {
+			module = strings.Trim(f[1], `"`)
+			break
+		}
+	}
+	if module == "" {
+		t.Fatal("go.mod names no module")
+	}
+	checked := 0
+	allowed := map[string][]string{
+		".":                    {"internal/builtinfunc", "internal/datafiles", "internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/jsonvalue", "internal/proven", "internal/rows"},
+		"cli":                  {"."},
+		"cmd/fejkdata":         {"cli", "data"},
+		"data":                 {"data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
+		"data/en_US":           {"data/geo/US"},
+		"data/geo/SE":          nil,
+		"data/geo/US":          nil,
+		"data/misc":            nil,
+		"data/sv_SE":           {"data/geo/SE"},
+		"internal/builtinfunc": {"internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/proven"},
+		"internal/datafiles":   {"internal/grammar", "internal/jsonvalue"},
+		"internal/datatype":    nil,
+		"internal/drawstate":   nil,
+		"internal/grammar":     nil,
+		"internal/invariant":   nil,
+		"internal/jsonvalue":   nil,
+		"internal/proven":      {"internal/datatype", "internal/grammar", "internal/invariant"},
+		"internal/rows":        {"internal/drawstate", "internal/grammar", "internal/invariant"},
+	}
+	testsAlso := map[string][]string{
+		".":           {"data", "data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
+		"cli":         {"internal/grammar"},
+		"data/en_US":  {"."},
+		"data/geo/SE": {"."},
+		"data/geo/US": {"."},
+		"data/misc":   {"."},
+		"data/sv_SE":  {"."},
+	}
+	seen := map[string]bool{}
+	eachPackage(t, func(dir string, pkg *build.Package) {
+		seen[dir] = true
+		may, listed := allowed[dir]
+		if !listed {
+			t.Errorf("package %s is not listed: add the module packages it may import", dir)
+			return
+		}
+		for _, imp := range slices.Concat(pkg.Imports, pkg.TestImports, pkg.XTestImports) {
+			may := may
+			if !slices.Contains(pkg.Imports, imp) {
+				may = slices.Concat(may, testsAlso[dir])
+			}
+			if imp != module && !strings.HasPrefix(imp, module+"/") {
+				continue
+			}
+			checked++
+			rel := strings.TrimPrefix(strings.TrimPrefix(imp, module), "/")
+			if rel == "" {
+				rel = "."
+			}
+			if rel != dir && !slices.Contains(may, rel) {
+				t.Errorf("package %s imports %s, which its list does not allow", dir, rel)
+			}
+		}
+	})
+	if checked == 0 {
+		t.Fatalf("no package imports a package of module %s, which go.mod names", module)
+	}
+	for dir := range allowed {
+		if !seen[dir] {
+			t.Errorf("listed package %s does not exist", dir)
+		}
+	}
+}
+
+// eachPackage calls fn with every Go package in the module and its slash-separated directory.
+func eachPackage(t *testing.T, fn func(dir string, pkg *build.Package)) {
+	t.Helper()
+	err := filepath.WalkDir(".", func(dir string, d fs.DirEntry, err error) error {
+		if err != nil || !d.IsDir() {
+			return err
+		}
+		if dir != "." && (strings.HasPrefix(d.Name(), ".") || d.Name() == "testdata") {
+			return filepath.SkipDir
+		}
+		pkg, err := build.ImportDir(dir, 0)
+		var noGo *build.NoGoError
+		if errors.As(err, &noGo) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		fn(filepath.ToSlash(dir), pkg)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

index.go

@@ -0,0 +1,188 @@
+package fejkdata
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"slices"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// indexed is a category an index names and no call has reached yet, and the source holding it.
+type indexed struct {
+	entry datafiles.IndexEntry
+	src   datafiles.Source
+}
+
+// placeIndex sets every category the index names as unloaded in root, in place of what
+// root holds at its path.
+func placeIndex(root *folder, src datafiles.Source, index map[string]datafiles.IndexEntry) {
+	for _, p := range sortedNames(index) {
+		segs := strings.Split(p, ".")
+		madeFolder(root, segs[:len(segs)-1]).putUnloaded(segs[len(segs)-1], &indexed{entry: index[p], src: src})
+	}
+}
+
+// categoryAt is where a category, loaded or not, sits: the folder holding it, that
+// folder's path, and its name.
+type categoryAt struct {
+	dir  []string
+	in   *folder
+	name string
+}
+
+func (c categoryAt) unloaded() bool { return c.in.unloaded[c.name] != nil }
+
+// categoryOn is the category, loaded or not, that a path from root names or passes.
+func categoryOn(root *folder, segs []string) (categoryAt, bool) {
+	g, i := folderAt(root, segs)
+	if i == len(segs) {
+		return categoryAt{}, false
+	}
+	c := categoryAt{dir: segs[:i:i], in: g, name: segs[i]}
+	return c, c.unloaded() || g.children[c.name] != nil
+}
+
+// loadCallerPath strips the / a caller's path may start with, splits the path, and loads the
+// unloaded category it names or passes. The caller walks it next, and a walk loads nothing.
+func (f *Generator) loadCallerPath(path string) (string, []string, error) {
+	path, err := grammar.CallerPath(path)
+	if err != nil {
+		return "", nil, fmt.Errorf("fejkdata: %w", err)
+	}
+	segs, err := grammar.SplitPath(path)
+	if err != nil {
+		return "", nil, fmt.Errorf("fejkdata: %w", err)
+	}
+	if c, ok := categoryOn(&f.root, segs); ok && c.unloaded() {
+		if err := loadReached(&f.root, nil, []categoryAt{c}); err != nil {
+			return "", nil, fmt.Errorf("fejkdata: %w", loadError{err})
+		}
+	}
+	return path, segs, nil
+}
+
+// referenced is every category, loaded or not, that the templates of nodes reference, each
+// reference read from the folder dir.
+func referenced(root *folder, dir []string, nodes nodeSet) []categoryAt {
+	var out []categoryAt
+	_ = nodes(func(_ string, n node) error {
+		t, isTemplate := n.(*template)
+		if !isTemplate {
+			return nil
+		}
+		for _, name := range refTokens(t.tokens) {
+			segs, err := refSegments(name, dir)
+			if err != nil {
+				continue // resolveRefs reports it
+			}
+			if c, ok := categoryOn(root, segs); ok {
+				out = append(out, c)
+			}
+		}
+		return nil
+	})
+	return out
+}
+
+// needs is every category, loaded or not, that s references, the table beside s that its
+// parent column names, and every unloaded table beside s whose parent s is.
+func needs(root *folder, s categorySite) []categoryAt {
+	out := referenced(root, s.dir, siteNodes([]categorySite{s}))
+	if parent := parentOf(s.n); parent != "" {
+		out = append(out, categoryAt{dir: s.dir, in: s.in, name: parent})
+	}
+	for _, name := range sortedNames(s.in.unloaded) {
+		if s.in.unloaded[name].entry.Parent == s.name {
+			out = append(out, categoryAt{dir: s.dir, in: s.in, name: name})
+		}
+	}
+	return out
+}
+
+// loadReached loads the unloaded categories among wanted and every unloaded category they
+// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an
+// error it puts back what it loaded, so the next call fails the same way.
+func loadReached(root *folder, sites []categorySite, wanted []categoryAt) error {
+	queue := wanted
+	for _, s := range sites {
+		queue = append(queue, needs(root, s)...)
+	}
+	var b batch
+	err := b.load(root, queue)
+	if err == nil {
+		err = b.run(root, sites)
+	}
+	if err != nil {
+		b.putBack()
+	}
+	return err
+}
+
+// batch is the categories one loadReached loaded, each beside the entry it was loaded from.
+type batch []loaded
+
+type loaded struct {
+	site categorySite
+	was  *indexed
+}
+
+// load loads each unloaded category in queue and what it needs. It stops at a table whose
+// parent column disagrees with its entry. Otherwise a table could load without its parent,
+// or an earlier table could link to one that putBack then removes.
+func (b *batch) load(root *folder, queue []categoryAt) error {
+	for ; len(queue) > 0; queue = queue[1:] {
+		c := queue[0]
+		x := c.in.unloaded[c.name]
+		if x == nil {
+			continue
+		}
+		if err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })); err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				err = fmt.Errorf("%s indexes %s: %w", x.src.ManifestPath(), categoryPath(c.dir, c.name), err)
+			}
+			return err
+		}
+		s := siteIn(c.dir, c.in, c.name)
+		*b = append(*b, loaded{site: s, was: x})
+		if parent := parentOf(s.n); parent != x.entry.Parent {
+			return fmt.Errorf("%s: the index entry for %s %s, and its table %s", x.src.ManifestPath(), s.path, namesParent(x.entry.Parent), namesParent(parent))
+		}
+		queue = append(queue, needs(root, s)...)
+	}
+	return nil
+}
+
+func (b batch) run(root *folder, sites []categorySite) error {
+	sites = slices.Clip(sites)
+	for _, l := range b {
+		sites = append(sites, l.site)
+	}
+	if len(sites) == 0 {
+		return nil
+	}
+	return categoryPipeline(sites, root.children).run()
+}
+
+func (b batch) putBack() {
+	for _, l := range b {
+		l.site.in.putUnloaded(l.site.name, l.was)
+	}
+}
+
+func namesParent(parent string) string {
+	if parent == "" {
+		return "names no parent"
+	}
+	return fmt.Sprintf("names parent %q", parent)
+}
+
+func parentOf(n node) string {
+	if t, isTable := n.(*table); isTable {
+		return t.rows.Options().Parent
+	}
+	return ""
+}

inline.go

@@ -3,28 +3,30 @@ package fejkdata
 import (
 	"encoding/json"
 	"fmt"
-	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
 // Template is an inline template compiled, referenced and validated against a
 // generator's loaded data once, ready to render many times with [Template.Fake].
 // It is safe for concurrent use: Fake serializes on its generator's lock, so a
-// seeded sequence is reproducible only when a generator — and its templates — are
+// sequence is reproducible only when one goroutine makes every draw from a generator
-// drawn from one goroutine.
+// and its templates.
 type Template struct {
 	g *Generator
 	n node
 }
 
-// Fake renders the template with one draw.
+// Fake renders the template once; each call draws anew.
 func (t *Template) Fake() string {
 	t.g.mu.Lock()
 	defer t.g.mu.Unlock()
-	return renderOnce(t.g.rand, t.n)
+	return renderOnce(t.g.drawState, t.n)
 }
 
 // NewTemplate compiles an inline template — a format string or a JSON value — and
-// binds its references against the loaded tree, so repeated renders pay the
+// resolves its references against the loaded tree, so repeated renders pay the
 // compile and validation once. It shares [New]'s guarantees: a bad template errors
 // here, and rendering cannot fail.
 func (f *Generator) NewTemplate(input string) (*Template, error) {
@@ -32,8 +34,10 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
-	if err := bindInline(n, "template", f.categories, checkScope); err != nil {
+	f.mu.Lock()
-		return nil, fmt.Errorf("fejkdata: %w", err)
+	defer f.mu.Unlock()
+	if err := loadInline(&f.root, n, "template", false); err != nil {
+		return nil, f.noDataNote(fmt.Errorf("fejkdata: %w", err))
 	}
 	return &Template{g: f, n: n}, nil
 }
@@ -49,127 +53,70 @@ func (f *Generator) FakeTemplate(input string) (string, error) {
 	return t.Fake(), nil
 }
 
-// IsTemplate reports whether arg is an inline template rather than a path, by its shape: a {
+// IsTemplate reports whether arg is an inline template rather than a path, by its shape: an
-// token, or a JSON object, array or string, is a template, and anything else is a path. A
+// arg holding a {, or a valid JSON array or string, is a template, and anything else is a
-// name never holds a bracket, a brace or a quote, so an arg holding one that is not valid
+// path. A path errors where it holds a } or a quote, starts with [, or is no valid path. A path
-// JSON names neither, and errors; so do a template of one reference alone, which is a path
+// may start with /, as a reference does, and every call taking a path drops it.
-// written as a template, and a path written with a leading /.
 func IsTemplate(arg string) (bool, error) {
-	inline, err := isTemplate(arg)
+	inline, err := grammar.IsTemplate(arg)
 	if err != nil {
 		return false, fmt.Errorf("fejkdata: %w", err)
 	}
 	return inline, nil
 }
 
-func isTemplate(arg string) (bool, error) {
-	if strings.ContainsRune(arg, '{') || (isJSONStart(strings.TrimSpace(arg)) && json.Valid([]byte(arg))) {
-		if path, lone := loneReference(arg); lone {
-			return false, pathAdvice(path, fmt.Sprintf("%s is the path %s written as a template", arg, path))
-		}
-		return true, nil
-	}
-	if i := strings.IndexAny(arg, `}"`); i >= 0 {
-		return false, fmt.Errorf("%q holds a %q, which no path may, and it is not valid JSON, so it names no template either", arg, arg[i:i+1])
-	}
-	if strings.HasPrefix(strings.TrimSpace(arg), "[") {
-		return false, fmt.Errorf(`%q starts with "[" and is not valid JSON, so it names no template; a path starts with a name`, arg)
-	}
-	if _, err := splitPath(arg); err != nil {
-		return false, err
-	}
-	if path := strings.TrimLeft(arg, "/"); path != arg && path != "" {
-		return false, pathAdvice(path, fmt.Sprintf("path %s starts with /, and every path starts at the root already", arg))
-	}
-	return false, nil
-}
-
-func isJSONStart(arg string) bool {
-	return strings.HasPrefix(arg, "[") || strings.HasPrefix(arg, `"`)
-}
-
-// pathAdvice refuses a spelling of path, naming path to write, or why no name can spell it.
-func pathAdvice(path, refusal string) error {
-	if err := checkPathNames(path); err != nil {
-		return err
-	}
-	return fmt.Errorf("%s; write %s", refusal, path)
-}
-
-// loneReference is the path a template spells when the value it holds — a format string, a
-// JSON string, or an object holding only a format — is one reference token and nothing else.
-func loneReference(arg string) (string, bool) {
-	var raw any
-	if json.Unmarshal([]byte(arg), &raw) != nil {
-		raw = arg
-	}
-	if m, isObject := raw.(map[string]any); isObject && len(m) == 1 {
-		raw = m["format"]
-	}
-	format, isString := raw.(string)
-	body, lone := loneRef(format)
-	if !isString || !lone {
-		return "", false
-	}
-	if strings.HasPrefix(body, "/") {
-		body = "/" + strings.TrimLeft(body, "/")
-	}
-	_, path, err := refShape(body)
-	return path, err == nil
-}
-
 func compileInput(input string) (node, error) {
-	v, err := inputValue(input)
+	return compile(inputValue(input))
+}
+
+// inputValue reads an inline template as the value compile takes: the JSON object, array, string
+// or null it holds, the whitespace around it dropped, or else the input itself as a format string,
+// a bare number or boolean among them.
+func inputValue(input string) jsonvalue.Value {
+	raw, err := jsonvalue.Decode([]byte(input))
 	if err != nil {
-		return nil, err
+		return jsonvalue.String(input)
 	}
-	return compile(v)
+	switch raw.Any().(type) {
+	case json.Number, bool:
+		return jsonvalue.String(input)
+	}
+	return raw
 }
 
-// inputValue reads an inline template as the value compile takes: the JSON value it holds, or
+// loadInline loads the unloaded categories n reads, then runs n through the pipeline against root.
-// the input itself as a format string when it is not JSON.
+func loadInline(root *folder, n node, label string, typedByGo bool) error {
-func inputValue(input string) (any, error) {
+	nodes := inlineNodes(n, label)
-	var raw any
+	if err := refuseFolderRefs(nodes); err != nil {
-	if err := json.Unmarshal([]byte(input), &raw); err != nil {
-		return input, nil
-	}
-	if trimmed := strings.TrimSpace(input); trimmed != input {
-		return nil, fmt.Errorf("a JSON template may not be padded with spaces, which a format string would render; write %s", trimmed)
-	}
-	return raw, nil
-}
-
-// bindInline links an inline node's references against root and runs check over it, naming its
-// nodes from label.
-func bindInline(n node, label string, root map[string]node, check func(nodeScope) error) error {
-	if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
-		return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)
-	}
-	scope := inlineScope(n, label)
-	if err := linkNodeRefs(scope, root); err != nil {
 		return err
 	}
-	return check(scope)
+	if err := loadReached(root, nil, referenced(root, nil, nodes)); err != nil {
+		return loadError{err}
+	}
+	return pipeline{
+		nodes:     nodes,
+		resolve:   func() error { return resolveInlineTemplates(nodes, root.children) },
+		typedByGo: typedByGo,
+	}.run()
 }
 
-// linkNodeRefs binds the references in an inline node's templates against the
+// refuseFolderRefs refuses each reference in nodes not written from the root, {/x}: an
-// loaded tree.
+// inline node sits in no folder.
-func linkNodeRefs(scope nodeScope, root map[string]node) error {
+func refuseFolderRefs(nodes nodeSet) error {
-	return scope(func(path string, m node) error {
+	return nodes(func(label string, m node) error {
 		t, ok := m.(*template)
 		if !ok {
 			return nil
 		}
-		t.keyDrawGroup("")
+		for _, name := range refTokens(t.tokens) {
-		for _, name := range refTokens(t.format) {
+			sigil, rest, err := grammar.RefShape(name)
-			sigil, rest, err := refShape(name)
 			if err != nil {
-				return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
+				return fmt.Errorf("%s: reference {%s}: %w", label, name, err)
 			}
 			if sigil != "/" {
-				return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", path, name, rest)
+				return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", label, name, rest)
 			}
 		}
-		return linkTemplateRefs(nil, path, "", t, root)
+		return nil
 	})
 }

inline_test.go

@@ -16,7 +16,7 @@ func tmpl(t *testing.T, f *Generator, input string) string {
 
 func shipped(t *testing.T, opts ...Option) *Generator {
 	t.Helper()
-	f, err := New(append([]Option{WithSeed(1)}, opts...)...)
+	f, err := New(append([]Option{withShipped(), WithSeed(1)}, opts...)...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -53,13 +53,13 @@ func TestFakeTemplateJSONArray(t *testing.T) {
 	}
 }
 
-func TestFakeTemplateCorrelatedReferences(t *testing.T) {
+func TestFakeTemplateNamedReference(t *testing.T) {
 	dir := writeData(t, map[string]string{
 		"person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`,
 	})
 	f := newGenerator(t, dir, WithSeed(1))
 	for i := 0; i < 100; i++ {
-		got := tmpl(t, f, "{/person.first} {/person.last}")
+		got := tmpl(t, f, "{/person as p}{p.first} {p.last}")
 		if got != "Ada Lovelace" && got != "Bo Ek" {
 			t.Fatalf("correlated references = %q, want one person's first and last", got)
 		}
@@ -104,13 +104,9 @@ func TestFakeTemplateErrors(t *testing.T) {
 	}{
 		{`{"x":"Q"}`, "missing string \"format\""},
 		{`"{x}"`, `no field "x"`},
-		{`"{digits(0)}"`, "must be positive"},
+		{`"{digits(-1)}"`, "must not be negative"},
 		{`name: {/no.such.path}`, "no entry"},
 		{`name: {..nope}`, "write {/nope}"},
-		{`{"format":"x"}`, "is a string"},
-		{`{/misc.territory} {/misc.territory.alpha2}`, "renders a level that {/misc.territory.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"},
-		{`{"format":"{/misc.territory.alpha2} {x}","x":"{/misc.territory}"}`, "reads a path into"},
-		{`{"format":"{/misc.territory.alpha2}","drawGroup":"g"}`, "nothing can reference"},
 	} {
 		_, err := f.FakeTemplate(c.input)
 		if err == nil || !strings.Contains(err.Error(), c.want) {
@@ -127,12 +123,15 @@ func TestFakeTemplateJSONString(t *testing.T) {
 	}
 }
 
-func TestPaddedJSONIsRejected(t *testing.T) {
+func TestPaddedJSONIsTheJSON(t *testing.T) {
 	f := shipped(t)
 	in := `{"format":"{x}","x":["a","b"]}`
-	_, err := f.NewTemplate("  " + in + "  ")
+	padded, err := f.NewTemplate(" \t" + in + "\n ")
-	if err == nil || !strings.Contains(err.Error(), "write "+in) {
+	if err != nil {
-		t.Fatalf("NewTemplate(padded JSON) = %v, want an error naming the unpadded spelling", err)
+		t.Fatalf("NewTemplate(padded JSON) = %v, want the JSON it pads", err)
+	}
+	if got := padded.Fake(); got != "a" && got != "b" {
+		t.Fatalf("padded JSON rendered %q, want a or b with no padding", got)
 	}
 }
 
@@ -153,46 +152,45 @@ func TestNewTemplateReusable(t *testing.T) {
 
 func TestIsTemplate(t *testing.T) {
 	for arg, want := range map[string]bool{
-		"sv_SE.person":    false,
+		"sv_SE.person":                           false,
-		"person.last":     false,
+		"person.last":                            false,
-		"name: {x}":       true,
+		"name: {x}":                              true,
-		`{"format":"x"}`:  true,
+		`{"format":"x"}`:                         true,
-		`["a","b"]`:       true,
+		`["a","b"]`:                              true,
-		`[1, 2]`:          true,
+		`[1, 2]`:                                 true,
-		` ["a","b"]`:      true, // padding is the template's own error, not a shape verdict
+		` ["a","b"]`:                             true, // padding is the template's own error, not a shape verdict
-		`"hello"`:         true,
+		`"hello"`:                                true,
-		"{/a}{/b}":        true,
+		"{/a}{/b}":                               true,
-		"{/a|/b}":         true,
+		"{/a|/b}":                                true,
-		"{uppercase(/a)}": true,
+		"{uppercase(/a)}":                        true,
-		"{{/a}}":          true,
+		"{{/a}}":                                 true,
-		`"{/a} x"`:        true,
+		`"{/a} x"`:                               true,
-		"x[1]":            false, // a row selected by key or name
+		"x[1]":                                   false, // a row selected by key or name
-		"x[St. Louis].y":  false,
+		"x[St. Louis].y":                         false,
+		"{/sv_SE.person.last}":                   true,
+		`"{/sv_SE.person}"`:                      true,
+		"{.person.last}":                         true,
+		` "{/sv_SE.person}"`:                     true,
+		`{"format":"{/sv_SE.person}"}`:           true,
+		"{//sv_SE.person}":                       true,
+		"{/misc.currency[US Dollar (Next day)]}": true,
+		"/sv_SE.person":                          false,
 	} {
 		if got, err := IsTemplate(arg); err != nil || got != want {
 			t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want)
 		}
 	}
 	for arg, want := range map[string]string{
-		"[abc]":                        `starts with "["`,
+		"[abc]":          `starts with "["`,
-		"[abc].field":                  `starts with "["`,
+		"[abc].field":    `starts with "["`,
-		"x[1]y":                        `"]"`,
+		"x[1]y":          `"]"`,
-		"x[[1]]":                       `"["`,
+		"x[[1]]":         `"["`,
-		"x[]":                          "empty",
+		"x[]":            "empty",
-		"a]b":                          `holds a "]"`,
+		"//sv_SE.person": "write /sv_SE.person",
-		"a}b":                          `holds a "}"`,
+		"a]b":            `holds a "]"`,
-		`"abc`:                         `holds a "\""`,
+		"a}b":            `holds a "}"`,
-		`"a]b`:                         `holds a "\""`, // the opener the reader typed, not the bracket behind it
+		`"abc`:           `holds a "\""`,
-		"{/sv_SE.person.last}":         "{/sv_SE.person.last} is the path sv_SE.person.last written as a template; write sv_SE.person.last",
+		`"a]b`:           `holds a "\""`, // the opener the reader typed, not the bracket behind it
-		`"{/sv_SE.person}"`:            "write sv_SE.person",
-		"{.person.last}":               "write person.last",
-		"/sv_SE.person":                "write sv_SE.person",
-		` "{/sv_SE.person}"`:           "write sv_SE.person",
-		`{"format":"{/sv_SE.person}"}`: "write sv_SE.person",
-		"//sv_SE.person":               "write sv_SE.person",
-		"{//sv_SE.person}":             "{//sv_SE.person} is the path sv_SE.person written as a template; write sv_SE.person",
-		"/sv_SE/person":                `path "sv_SE/person" contains "/"`,
-		"{/-}":                         `path "-" is reserved`,
 	} {
 		if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want)

internal/builtinfunc/builtinfunc.go

@@ -0,0 +1,119 @@
+// Package builtinfunc holds the {name(args)} functions a format calls: their checks, the
+// draws they make, and the numbers their text is proven to read as.
+package builtinfunc
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/proven"
+)
+
+// Call is a builtin prepared for one call site: its args already parsed. It reads the
+// output emitted so far in the current expansion (a derivation's payload) and the
+// values of the operands it named, which the caller read for it.
+type Call func(s *drawstate.State, emitted string, operands []string) string
+
+// builtin is a format-string function invoked as {name(args)}. Its Call stays pure over its
+// inputs, so seeded output is reproducible. arity is the exact arg count, or -1 for variadic,
+// where checkArgs does all the validation.
+type builtin struct {
+	arity int
+	// prep parses validated args once, at compile time, into the closure a render calls.
+	prep      func(args []string) Call
+	checkArgs func(args []string) error
+	// operands names the fields the call reads, which a render reads for it; nil
+	// for a builtin that reads none.
+	operands func(args []string) []string
+	// proveNumber bounds the number a call's text reads as, token its body, and says which
+	// datatypes that text is not; set it where every render reads as a finite number, which
+	// makes the call a calc operand, and leave it nil otherwise.
+	proveNumber func(token string, prints datatype.DataType, args []string) proven.Facts
+	// prints is the datatype a call's text is, handed to proveNumber: datatype.String where it
+	// reads as a number no column should type, as digits' leading zeros; unset where
+	// proveNumber is nil.
+	prints datatype.DataType
+}
+
+// CalcName is the builtin whose operands package fejkdata checks and proves, since only it
+// holds the nodes they read.
+const CalcName = "calc"
+
+// rng is the randomness a builtin sample draws from, which *drawstate.State satisfies.
+type rng interface {
+	IntN(n int) int
+	Float64() float64
+}
+
+// Check validates a call at compile time: naming a known builtin, with the arg count that
+// builtin takes and args its check accepts.
+func Check(name string, args []string) error {
+	b, known := builtins[name]
+	if !known {
+		return fmt.Errorf("unknown function %q", name)
+	}
+	if b.arity >= 0 && len(args) != b.arity {
+		return fmt.Errorf("%s takes %d argument%s, got %d", name, b.arity, plural(b.arity), len(args))
+	}
+	if b.checkArgs != nil {
+		return b.checkArgs(args)
+	}
+	return nil
+}
+
+// Prep parses a call's args, which Check accepted, into the Call a render makes.
+func Prep(name string, args []string) Call { return builtins[name].prep(args) }
+
+// Operands lists the fields a call reads as operands, empty for a builtin that reads none
+// or is unknown.
+func Operands(name string, args []string) []string {
+	b, known := builtins[name]
+	if !known || b.operands == nil {
+		return nil
+	}
+	return b.operands(args)
+}
+
+// IsTransform reports a builtin that rewrites the text of its one operand.
+func IsTransform(name string) bool {
+	_, isTransform := transforms[name]
+	return isTransform
+}
+
+// ProveNumber proves a call whose every render reads as a finite number, token its body;
+// false for any other call, calc included: proven.OfCalc proves a calc from its operands,
+// which only the caller holds.
+func ProveNumber(name, token string, args []string) (proven.Facts, bool) {
+	b := builtins[name]
+	if b.proveNumber == nil {
+		return proven.Facts{}, false
+	}
+	return b.proveNumber(token, b.prints, args), true
+}
+
+// TypedCalls lists the calls whose text a typed column may hold, as an error names them.
+func TypedCalls() string { return numberCalls(false) }
+
+// OperandCalls lists the calls whose text a calc reads as a number, as an error names them.
+func OperandCalls() string { return numberCalls(true) }
+
+func numberCalls(text bool) string {
+	var calls []string
+	for name, b := range builtins {
+		if b.proveNumber != nil && (text || b.prints != datatype.String) {
+			calls = append(calls, "{"+name+"()}")
+		}
+	}
+	slices.Sort(calls)
+	return strings.Join(calls, ", ") + " or {" + CalcName + "()}"
+}
+
+func plural(n int) string {
+	if n == 1 {
+		return ""
+	}
+	return "s"
+}

internal/builtinfunc/builtinfunc_test.go

@@ -0,0 +1,68 @@
+package builtinfunc
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// TestRegistryShapes pins the builtin contract Prep relies on: every entry supplies prep,
+// and args parsed at compile only behind a check.
+func TestRegistryShapes(t *testing.T) {
+	for name, b := range builtins {
+		if b.prep == nil {
+			t.Errorf("builtin %q has no prep: Prep would call a nil func", name)
+		}
+		if b.arity != 0 && b.checkArgs == nil {
+			t.Errorf("builtin %q parses args in prep with no check", name)
+		}
+	}
+}
+
+func TestCheckNamesWhatACallGetsWrong(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		args []string
+		want string
+	}{
+		{"nope", nil, `unknown function "nope"`},
+		{"luhn", []string{"1"}, "luhn takes 0 arguments, got 1"},
+		{"hex", nil, "hex takes 1 argument, got 0"},
+		{"calc", []string{"a +"}, `calc("a +"): `},
+		{"calc", []string{"a", "-1"}, "calc decimals "},
+		{"calc", nil, "calc takes an expression and an optional decimals count, got 0 args"},
+	} {
+		if err := Check(c.name, c.args); err == nil || !strings.HasPrefix(err.Error(), c.want) {
+			t.Errorf("Check(%s, %q) = %v, want it to start %q", c.name, c.args, err, c.want)
+		}
+	}
+	if err := Check("calc", []string{"a / b", "2"}); err != nil {
+		t.Errorf("Check(calc) = %v, want an expression over any operand accepted", err)
+	}
+}
+
+// TestArgGuardsPanic pins the guards that report a builtin arg its check should have
+// rejected. No data reaches them, since Check runs before Prep, so they are called directly.
+func TestArgGuardsPanic(t *testing.T) {
+	for name, call := range map[string]func(){
+		"atoi on an unvalidated arg":             func() { atoi("nope") },
+		"atof on an unvalidated arg":             func() { atof("nope") },
+		"prep on an expression that won't parse": func() { calcPrep([]string{"1 +"}) },
+	} {
+		mustPanic(t, name, call)
+	}
+}
+
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
+// separates a reported invariant break from a silently wrong value.
+func mustPanic(t *testing.T, name string, call func()) {
+	t.Helper()
+	defer func() {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
+		}
+	}()
+	call()
+}

internal/builtinfunc/builtins.go

@@ -1,4 +1,4 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"encoding/base64"
@@ -7,6 +7,11 @@ import (
 	"math"
 	"strconv"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/proven"
 )
 
 // maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
@@ -22,88 +27,88 @@ const (
 // emitted so far in the current expansion (place them after their payload);
 // samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
 // from the rng, not the wall clock, so seeded output stays reproducible.
-var builtins = map[string]builtin{
+var builtins = withTransforms(map[string]builtin{
 	"luhn":   {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
 	"mod11":  {arity: 0, prep: derive(mod11Check)},
 	"ean":    {arity: 0, prep: derive(eanCheck)},
 	"uuid":   {arity: 0, prep: sample(uuidV7)},
 	"ulid":   {arity: 0, prep: sample(ulid)},
-	"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)},
+	"nanoid": {arity: 1, checkArgs: countArg, prep: chars(nanoidAlphabet)},
-	"hex":    {arity: 1, check: posIntArg, prep: chars(hexDigits)},
+	"hex":    {arity: 1, checkArgs: countArg, prep: chars(hexDigits)},
-	"digits": {arity: 1, check: posIntArg, prep: chars("0123456789"), number: func(token string, a []string) proven {
+	"digits": {arity: 1, checkArgs: countArg, prep: chars("0123456789"), prints: datatype.String, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
-		return printing(token, DataTypeString, bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
+		if atoi(a[0]) == 0 {
+			return proven.Unproven("{" + token + "} prints nothing, which is no number")
+		}
+		return proven.Printing(token, prints, proven.Bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
 	}},
-	"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
+	"upper": {arity: 1, checkArgs: countArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
-	"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
+	"lower": {arity: 1, checkArgs: countArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
-	"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
+	"base64": {arity: 1, checkArgs: countArg, prep: func(a []string) Call {
 		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
 			return base64.StdEncoding.EncodeToString(randBytes(s, n))
 		}
 	}},
-	"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
+	"int": {arity: 2, checkArgs: intRangeArgs, prep: func(a []string) Call {
 		lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
-		return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
+		return func(s *drawstate.State, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
-	}, number: func(token string, a []string) proven {
+	}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
-		return printing(token, DataTypeInteger, bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
+		return proven.Printing(token, prints, proven.Bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
 	}},
-	"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
+	"float": {arity: 3, checkArgs: floatArgs, prep: func(a []string) Call {
 		lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
 			return formatFloat(lo+s.Float64()*(hi-lo), dp)
 		}
-	}, number: func(token string, a []string) proven {
+	}, prints: datatype.Number, proveNumber: func(token string, _ datatype.DataType, a []string) proven.Facts {
-		return printedNumber(token, bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
+		return proven.PrintedNumber(token, proven.Bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
 	}},
-	"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn {
+	"iban": {arity: 1, checkArgs: ibanArg, prep: func(a []string) Call {
 		cc := a[0]
-		return func(s *session, _ string, _ []string) string { return iban(s, cc) }
+		return func(s *drawstate.State, _ string, _ []string) string { return iban(s, cc) }
 	}},
-	"date":      {arity: -1, check: dateArgs, prep: datePrep},
+	"date":   {arity: -1, checkArgs: dateArgs, prep: datePrep},
-	"time":      {arity: -1, check: timeArg, prep: timePrep},
+	"time":   {arity: -1, checkArgs: timeArg, prep: timePrep},
-	"calc":      {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands},
+	CalcName: {arity: -1, checkArgs: checkCalc, prep: calcPrep, operands: calcOperands},
-	"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand},
+	// seq is the one stateful builtin: a per-generator counter from 1, advancing on
-	"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
-	"ascii":     {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
-	// seq is the one stateful builtin: a per-session counter from 1, advancing on
 	// each call. An optional name selects an independent counter; no name uses the
 	// default one. Deterministic by construction, so seeded output stays stable.
-	"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
+	"seq": {arity: -1, checkArgs: seqArg, prep: func(a []string) Call {
 		key := ""
 		if len(a) == 1 {
 			key = a[0]
 		}
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
-			return strconv.FormatUint(s.next(key), 10)
+			return strconv.FormatUint(s.Seq(key), 10)
 		}
-	}, number: func(token string, _ []string) proven {
+	}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, _ []string) proven.Facts {
-		return printing(token, DataTypeInteger, bounded(1, math.MaxInt64, true))
+		return proven.Printing(token, prints, proven.Bounded(1, math.MaxInt64, true))
 	}},
-}
+})
 
 // derive and sample are the two argument-free builtin shapes: a derivation reads
 // the output emitted so far, a sample reads only the rng. chars is the shape of a
 // sample of n characters drawn from an alphabet.
-func derive(f func(emitted string) string) func([]string) callFn {
+func derive(f func(emitted string) string) func([]string) Call {
-	return func([]string) callFn {
+	return func([]string) Call {
-		return func(_ *session, emitted string, _ []string) string { return f(emitted) }
+		return func(_ *drawstate.State, emitted string, _ []string) string { return f(emitted) }
 	}
 }
-func sample(f func(rng) string) func([]string) callFn {
+func sample(f func(rng) string) func([]string) Call {
-	return func([]string) callFn {
+	return func([]string) Call {
-		return func(s *session, _ string, _ []string) string { return f(s) }
+		return func(s *drawstate.State, _ string, _ []string) string { return f(s) }
 	}
 }
-func chars(alphabet string) func([]string) callFn {
+func chars(alphabet string) func([]string) Call {
-	return func(a []string) callFn {
+	return func(a []string) Call {
 		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) }
+		return func(s *drawstate.State, _ string, _ []string) string { return randChars(s, n, alphabet) }
 	}
 }
 
 const hexDigits = "0123456789abcdef"
 
-// formatFloat prints v to dp decimals, -1 for the shortest form, and a zero unsigned.
+// formatFloat prints v to dp decimals, or proven.ShortestDecimals, and a zero unsigned.
 func formatFloat(v float64, dp int) string {
 	s := strconv.FormatFloat(v, 'f', dp, 64)
 	if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
@@ -118,7 +123,7 @@ func formatFloat(v float64, dp int) string {
 func atoi(s string) int {
 	n, err := strconv.Atoi(s)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
 	}
 	return n
 }
@@ -127,7 +132,7 @@ func atoi(s string) int {
 func atof(s string) float64 {
 	f, err := strconv.ParseFloat(s, 64)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
 	}
 	return f
 }
@@ -146,8 +151,7 @@ func randChars(r rng, n int, alphabet string) string {
 	return string(b)
 }
 
-// plainInt parses an integer arg written the one way: no sign, no leading zero.
+func intArg(s string) (int, error) {
-func plainInt(s string) (int, error) {
 	n, err := strconv.Atoi(s)
 	if errors.Is(err, strconv.ErrRange) {
 		return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
@@ -155,54 +159,48 @@ func plainInt(s string) (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("%q is not an integer", s)
 	}
-	if strconv.Itoa(n) != s {
-		return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n)
-	}
 	return n, nil
 }
-func posIntArg(_ map[string]node, a []string) error {
+func countArg(a []string) error {
-	n, err := plainInt(a[0])
+	n, err := intArg(a[0])
 	if errors.Is(err, strconv.ErrRange) {
 		return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
 	}
 	if err != nil {
 		return fmt.Errorf("count %w", err)
 	}
-	if n < 1 {
+	if n < 0 {
-		return fmt.Errorf("count %q must be positive", a[0])
+		return fmt.Errorf("count %q must not be negative", a[0])
 	}
 	if n > maxLen {
 		return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
 	}
 	return nil
 }
-func intRangeArgs(_ map[string]node, a []string) error {
+func intRangeArgs(a []string) error {
-	lo, err := plainInt(a[0])
+	lo, err := intArg(a[0])
 	if err != nil {
 		return fmt.Errorf("int(min,max): min %w", err)
 	}
-	hi, err := plainInt(a[1])
+	hi, err := intArg(a[1])
 	if err != nil {
 		return fmt.Errorf("int(min,max): max %w", err)
 	}
 	if lo > hi {
 		return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
 	}
-	if lo == hi {
-		return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo)
-	}
 	if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
 		return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
 	}
 	return nil
 }
-func floatArgs(_ map[string]node, a []string) error {
+func floatArgs(a []string) error {
 	lo, e1 := strconv.ParseFloat(a[0], 64)
 	hi, e2 := strconv.ParseFloat(a[1], 64)
 	if e1 != nil || e2 != nil {
 		return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
 	}
-	dp, err := plainInt(a[2])
+	dp, err := intArg(a[2])
 	if err != nil {
 		return fmt.Errorf("float(min,max,dp): decimals %w", err)
 	}
@@ -215,15 +213,12 @@ func floatArgs(_ map[string]node, a []string) error {
 	if dp < 0 || dp > maxDecimals {
 		return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
 	}
-	if lo == hi {
-		return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64))
-	}
 	if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
 		return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
 	}
 	return nil
 }
-func seqArg(_ map[string]node, a []string) error {
+func seqArg(a []string) error {
 	if len(a) > 1 {
 		return fmt.Errorf("seq takes at most one name, got %d", len(a))
 	}

internal/builtinfunc/calc.go

@@ -0,0 +1,109 @@
+package builtinfunc
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/proven"
+)
+
+// evalCalc evaluates an expression over the operand values a render already read, so it
+// draws nothing.
+func evalCalc(n grammar.CalcNode, operands []string) float64 {
+	switch n := n.(type) {
+	case grammar.CalcNum:
+		return float64(n)
+	case grammar.CalcVar:
+		v, err := strconv.ParseFloat(strings.TrimSpace(operands[n.At]), 64)
+		if err != nil {
+			return math.NaN() // a non-numeric operand stays visible, never an error
+		}
+		return v
+	case grammar.CalcNeg:
+		return -evalCalc(n.X, operands)
+	case grammar.CalcBin:
+		return Arith(n.Operator, evalCalc(n.L, operands), evalCalc(n.R, operands))
+	}
+	panic(invariant.Broken("calc node %#v has no evaluation", n))
+}
+
+// Arith applies a calc operator.
+func Arith(operator byte, l, r float64) float64 {
+	switch operator {
+	case '+':
+		return l + r
+	case '-':
+		return l - r
+	case '*':
+		return l * r
+	default: // '/'
+		return l / r
+	}
+}
+
+// checkCalc validates a calc token's own args: a parseable expression and an optional
+// non-negative integer dp. What its operands hold is the caller's to check.
+func checkCalc(args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
+	}
+	if _, err := grammar.ParseCalc(args[0]); err != nil {
+		return fmt.Errorf("calc(%q): %w", args[0], err)
+	}
+	if len(args) == 2 {
+		dp, err := intArg(args[1])
+		if err != nil {
+			return fmt.Errorf("calc decimals %w", err)
+		}
+		if dp < 0 || dp > maxDecimals {
+			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
+		}
+	}
+	return nil
+}
+
+// calcPrep parses the expression and decimals once, at compile time. checkCalc proved
+// both args valid, so no step here can fail.
+func calcPrep(args []string) Call {
+	expr := ParsedCalc(args[0]).Expr
+	dp := CalcDecimals(args)
+	return func(_ *drawstate.State, _ string, operands []string) string {
+		return formatFloat(evalCalc(expr, operands), dp)
+	}
+}
+
+// ParsedCalc parses an expression Check accepted.
+func ParsedCalc(expr string) grammar.Calc {
+	c, err := grammar.ParseCalc(expr)
+	if err != nil {
+		panic(invariant.Broken("calc(%q) passed its check unparsed: %v", expr, err))
+	}
+	return c
+}
+
+// CalcDecimals is the decimals count of a calc Check accepted, or proven.ShortestDecimals
+// where it names none.
+func CalcDecimals(args []string) int {
+	if len(args) == 2 {
+		return atoi(args[1])
+	}
+	return proven.ShortestDecimals
+}
+
+// calcOperands lists the operands a calc's args read, fields or names. checkCalc reports
+// an expression that does not parse, so one that does not simply names nothing.
+func calcOperands(args []string) []string {
+	if len(args) == 0 {
+		return nil
+	}
+	c, err := grammar.ParseCalc(args[0])
+	if err != nil {
+		return nil
+	}
+	return c.Operands
+}

internal/builtinfunc/checksum.go

@@ -1,4 +1,4 @@
-package fejkdata
+package builtinfunc
 
 import "fmt"
 
@@ -65,16 +65,14 @@ func eanCheck(s string) string {
 // a left-to-right derivation; it generates the whole value instead.
 var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
 
-func ibanArg(_ map[string]node, a []string) error {
+func ibanArg(a []string) error {
 	if _, ok := ibanLen[a[0]]; !ok {
 		return fmt.Errorf("iban(%q): unsupported country code", a[0])
 	}
 	return nil
 }
 
-// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right
+// iban generates an IBAN for cc: a numeric BBAN of the right length, then mod-97 check digits.
-// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
-// the result passes length and checksum validation, which is what fake data needs.
 func iban(r rng, cc string) string {
 	bban := make([]byte, ibanLen[cc]-4)
 	for i := range bban {

internal/builtinfunc/layout.go

@@ -1,39 +1,23 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"fmt"
 	"strings"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 const dayLayout = "2006-01-02"
 
-// The instants a layout is proved against: layoutProbe2 is alike in no field, while
-// layoutDay differs from layoutProbe in its date fields alone and layoutClock in its
-// clock fields alone, so formatting two of them tells which kind a layout names.
-var (
-	layoutProbe  = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
-	layoutProbe2 = time.Date(2010, 11, 12, 13, 14, 15, 0, time.UTC)
-	layoutDay    = time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
-	layoutClock  = time.Date(2001, 2, 3, 13, 14, 15, 0, time.UTC)
-)
-
-func namesAField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutProbe2.Format(layout)
-}
-func namesADateField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutDay.Format(layout)
-}
-func namesAClockField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutClock.Format(layout)
-}
-
 // quotedLayout reports whether an arg carries the single quotes a layout is written in.
 func quotedLayout(a string) bool {
 	return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\''
 }
 
-// layoutArg is the Go layout a quoted arg holds, refused when unquoted or constant.
+// layoutArg is the Go layout a quoted arg holds, refused when unquoted.
+// docs/decisions.md#a-layout-is-always-quoted
 func layoutArg(a string) (string, error) {
 	if !quotedLayout(a) {
 		bare := strings.Trim(a, `'"`)
@@ -42,18 +26,14 @@ func layoutArg(a string) (string, error) {
 		}
 		return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare)
 	}
-	layout := a[1 : len(a)-1]
+	return a[1 : len(a)-1], nil
-	if !namesAField(layout) {
-		return "", fmt.Errorf("layout '%s' names no field, so it is the constant %q; write it as text", layout, layout)
-	}
-	return layout, nil
 }
 
 // layoutOf is layoutArg for an arg a check already validated.
 func layoutOf(a string) string {
 	layout, err := layoutArg(a)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a, err))
 	}
 	return layout
 }
@@ -81,7 +61,7 @@ func holdsQuotedLayout(a []string) bool {
 	}
 	return false
 }
-func dateArgs(_ map[string]node, a []string) error {
+func dateArgs(a []string) error {
 	if err := layoutArity("date", 3, a); err != nil {
 		return err
 	}
@@ -96,51 +76,45 @@ func dateArgs(_ map[string]node, a []string) error {
 	if to.Before(from) {
 		return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1])
 	}
-	layout, err := layoutArg(a[2])
+	_, err = layoutArg(a[2])
-	if err != nil {
+	return err
-		return err
-	}
-	if !namesADateField(layout) {
-		return fmt.Errorf("date(from,to,layout): '%s' names no date field; write time('%s')", layout, layout)
-	}
-	if from.Equal(to) && !namesAClockField(layout) {
-		return fmt.Errorf("date(%s,%s,'%s') is the constant %q; write it as text", a[0], a[1], layout, from.Format(layout))
-	}
-	return nil
 }
-func timeArg(_ map[string]node, a []string) error {
+func timeArg(a []string) error {
 	if err := layoutArity("time", 1, a); err != nil {
 		return err
 	}
 	layout, err := layoutArg(a[0])
-	if err != nil {
+	if err == nil && namesADateField(layout) {
-		return err
+		return fmt.Errorf("time(layout): '%s' names a date field, and time draws no date; write date(from,to,layout)", layout)
 	}
-	if namesADateField(layout) {
+	return err
-		return fmt.Errorf("time(layout): '%s' names a date field; write date(from,to,layout)", layout)
+}
-	}
+
-	return nil
+// namesADateField reports a layout that prints two instants differing in their date alone apart.
+func namesADateField(layout string) bool {
+	day, nextYear := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC), time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
+	return day.Format(layout) != nextYear.Format(layout)
 }
 
 // datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted
 // in seconds, since a Duration overflows past 292 years.
-func datePrep(a []string) callFn {
+func datePrep(a []string) Call {
 	from, err := time.Parse(dayLayout, a[0])
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[0], err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[0], err))
 	}
 	to, err := time.Parse(dayLayout, a[1])
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[1], err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[1], err))
 	}
 	layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400
-	return func(s *session, _ string, _ []string) string {
+	return func(s *drawstate.State, _ string, _ []string) string {
 		return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout)
 	}
 }
-func timePrep(a []string) callFn {
+func timePrep(a []string) Call {
 	layout := layoutOf(a[0])
-	return func(s *session, _ string, _ []string) string {
+	return func(s *drawstate.State, _ string, _ []string) string {
 		return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout)
 	}
 }

internal/builtinfunc/transform.go

@@ -1,9 +1,13 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"fmt"
 	"strings"
 	"unicode"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 // transforms are the builtins that rewrite one operand's value; they nest, so
@@ -14,11 +18,21 @@ var transforms = map[string]func(string) string{
 	"uppercase": strings.ToUpper,
 }
 
+func withTransforms(calls map[string]builtin) map[string]builtin {
+	for name, fn := range transforms {
+		if _, clash := calls[name]; clash {
+			panic(invariant.Broken("%s is registered as a builtin and a transform", name))
+		}
+		calls[name] = builtin{arity: 1, checkArgs: transformArg, prep: transformPrep(fn), operands: transformOperand}
+	}
+	return calls
+}
+
 // unwrapTransform peels nested transform calls off an operand arg, returning the
 // field it finally names and the transforms to apply, innermost last.
 func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
 	for {
-		name, args, isCall := funcCall(arg)
+		name, args, isCall := grammar.FuncCall(arg)
 		if !isCall {
 			return arg, chain, nil
 		}
@@ -33,31 +47,29 @@ func unwrapTransform(arg string) (leaf string, chain []func(string) string, err
 		arg = args[0]
 	}
 }
-func transformArg(fields map[string]node, a []string) error {
+
-	leaf, _, err := unwrapTransform(a[0])
+// transformArg checks that each call nested in the arg is a transform taking one arg.
-	if err != nil {
+func transformArg(a []string) error {
-		return err
+	_, _, err := unwrapTransform(a[0])
-	}
+	return err
-	if isRef(leaf) {
-		_, _, err := refShape(leaf)
-		return err
-	}
-	return checkArm(leaf, fields, false)
 }
 func transformOperand(a []string) []string {
+	if len(a) != 1 {
+		return nil
+	}
 	leaf, _, err := unwrapTransform(a[0])
 	if err != nil {
 		return nil
 	}
 	return []string{leaf}
 }
-func transformPrep(outer func(string) string) func([]string) callFn {
+func transformPrep(outer func(string) string) func([]string) Call {
-	return func(a []string) callFn {
+	return func(a []string) Call {
 		_, chain, err := unwrapTransform(a[0])
 		if err != nil {
-			panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err))
+			panic(invariant.Broken("transform arg %q reached prep unvalidated: %v", a[0], err))
 		}
-		return func(_ *session, _ string, operands []string) string {
+		return func(_ *drawstate.State, _ string, operands []string) string {
 			v := operands[0]
 			for i := len(chain) - 1; i >= 0; i-- {
 				v = chain[i](v)

internal/datafiles/datafiles.go

@@ -0,0 +1,215 @@
+// Package datafiles walks a data tree and reads its category files and their rows.
+package datafiles
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
+)
+
+// Source is one tree to load, from the root of an fs.FS. label prefixes file names in
+// errors; onDisk marks label as a directory that must exist; at is the folder fsys mounts the
+// tree at, "" for its root.
+type Source struct {
+	fsys   fs.FS
+	label  string
+	onDisk bool
+	at     string
+}
+
+// Dir is the tree in the directory path, its errors labelled with it.
+func Dir(path string) Source {
+	return Source{fsys: os.DirFS(path), label: path, onDisk: true}
+}
+
+// FS is the tree in fsys.
+func FS(fsys fs.FS) Source { return Source{fsys: fsys} }
+
+// Mounted is s with its tree under the folders the dot path namespace spells, "" for none.
+func (s Source) Mounted(namespace string) Source {
+	if namespace != "" && s.at == "" {
+		s.at = strings.ReplaceAll(namespace, ".", "/")
+		s.fsys = mounted{fsys: s.fsys, at: s.at}
+	}
+	return s
+}
+
+// Category is one category file: the folders above it from the source's root, its name,
+// and its parsed JSON.
+type Category struct {
+	Folders []string
+	Name    string
+	JSON    jsonvalue.Value
+	rows    *rowsFiles
+}
+
+// ReadRows reads a rows file beside the category.
+func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) }
+
+// labelled is the file p as errors name it: a labelled source's as its own tree spells it,
+// below the mount.
+func (s Source) labelled(p string) string {
+	if s.label == "" {
+		return p
+	}
+	if inner, below := underMount(s.at, p); below {
+		p = inner
+	}
+	return path.Join(s.label, p)
+}
+
+// dirPath is the folder that dir names, as fsys spells it.
+func (s Source) dirPath(dir []string) string {
+	if p := path.Join(dir...); p != "" {
+		return p
+	}
+	return "."
+}
+
+// Walk hands compile every category of the tree, folders and files in name order. A
+// folder holding no category anywhere below it is skipped; a hidden file or folder is no
+// category, though a category may name a hidden rows file beside it.
+func (s Source) Walk(compile func(Category) error) error {
+	if err := s.check(); err != nil {
+		return err
+	}
+	_, err := s.walkDir(nil, compile)
+	return err
+}
+
+// check refuses a data path that is empty or names no directory.
+func (s Source) check() error {
+	if !s.onDisk {
+		return nil
+	}
+	if s.label == "" {
+		return fmt.Errorf("a data path is empty")
+	}
+	info, err := os.Stat(s.label)
+	if errors.Is(err, fs.ErrNotExist) {
+		return noDirectory(s.label)
+	}
+	if err != nil {
+		return fmt.Errorf("data path %s: %w", s.label, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("data path %s: not a directory", s.label)
+	}
+	return nil
+}
+
+// noDirectory is a data path naming nothing.
+type noDirectory string
+
+func (d noDirectory) Error() string { return "data path " + string(d) + ": no such directory" }
+
+func (noDirectory) Unwrap() error { return fs.ErrNotExist }
+
+// walkDir walks the folder that dir names, and reports whether it handed over any category.
+func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) {
+	full := s.dirPath(dir)
+	entries, err := fs.ReadDir(s.fsys, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	rows := newRowsFiles(s, full, entries)
+	handed := false
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".") {
+			continue
+		}
+		var holds bool
+		if e.IsDir() {
+			holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile)
+		} else {
+			holds, err = s.compileFile(dir, e.Name(), rows, compile)
+		}
+		if err != nil {
+			return false, err
+		}
+		handed = handed || holds
+	}
+	return handed, nil
+}
+
+// walkFolder walks a subfolder, and refuses its name if it holds a category.
+func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) {
+	holds, err := s.walkDir(dir, compile)
+	if err != nil || !holds {
+		return false, err
+	}
+	if err := grammar.CheckIdentifier(dir[len(dir)-1]); err != nil {
+		return false, fmt.Errorf("%s: folder %w", s.labelled(s.dirPath(dir)), err)
+	}
+	return true, nil
+}
+
+// Load hands compile the category file name.json in the folder dir.
+func (s Source) Load(dir []string, name string, compile func(Category) error) error {
+	full := s.dirPath(dir)
+	entries, err := fs.ReadDir(s.fsys, full)
+	if err != nil {
+		return fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	_, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile)
+	return err
+}
+
+// compileFile hands compile a *.json file as a category named after it, and reports
+// whether it was one; any other file is skipped.
+func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) {
+	if !strings.HasSuffix(file, ".json") {
+		return false, nil
+	}
+	full := path.Join(rows.dir, file)
+	name := strings.TrimSuffix(file, ".json")
+	if err := grammar.CheckIdentifier(name); err != nil {
+		return false, fmt.Errorf("%s: category %w", s.labelled(full), err)
+	}
+	b, err := fs.ReadFile(s.fsys, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	raw, err := jsonvalue.Decode(b)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	if err := compile(Category{Folders: dir, Name: name, JSON: raw, rows: rows}); err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	return true, nil
+}
+
+// rowsFiles is what a category may name beside itself: the rows files of its folder.
+type rowsFiles struct {
+	src Source
+	dir string
+	tsv map[string]bool
+}
+
+func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles {
+	files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
+			files.tsv[e.Name()] = true
+		}
+	}
+	return files
+}
+
+func (r *rowsFiles) read(name string) (string, error) {
+	if !r.tsv[name] {
+		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir))
+	}
+	b, err := fs.ReadFile(r.src.fsys, path.Join(r.dir, name))
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err)
+	}
+	return string(b), nil
+}

internal/datafiles/datafiles_test.go

@@ -0,0 +1,217 @@
+package datafiles
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+type handed struct {
+	dir  []string
+	name string
+	json any
+}
+
+func walk(t *testing.T, src Source) ([]handed, error) {
+	t.Helper()
+	var got []handed
+	err := src.Walk(func(c Category) error {
+		got = append(got, handed{c.Folders, c.Name, c.JSON.Any()})
+		return nil
+	})
+	return got, err
+}
+
+func TestWalkHandsEachCategoryInOrder(t *testing.T) {
+	got, err := walk(t, Source{fsys: fstest.MapFS{
+		"a.json":          {Data: []byte(`{"x":"1"}`)},
+		"sub/b.json":      {Data: []byte(`"y"`)},
+		"sub/notes.txt":   {Data: []byte("not data")},
+		"empty/x.txt":     {Data: []byte("not data")},
+		".git/c.json":     {Data: []byte(`"hidden"`)},
+		"sub/.d.json":     {Data: []byte(`"hidden"`)},
+		"sub/deep/e.json": {Data: []byte(`"z"`)},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []handed{
+		{nil, "a", map[string]any{"x": "1"}},
+		{[]string{"sub"}, "b", "y"},
+		{[]string{"sub", "deep"}, "e", "z"},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Walk handed %#v, want %#v", got, want)
+	}
+}
+
+func TestReadRowsReadsOnlyAFileBesideTheCategory(t *testing.T) {
+	src := Source{fsys: fstest.MapFS{
+		"t.json":     {Data: []byte(`{"rows":"t.tsv"}`)},
+		"t.tsv":      {Data: []byte("key\nA\n")},
+		"sub/u.tsv":  {Data: []byte("key\nB\n")},
+		"sub/u.json": {Data: []byte(`{"rows":"u.tsv"}`)},
+	}, label: "lbl"}
+	rows := map[string]string{}
+	err := src.Walk(func(c Category) error {
+		data, err := c.ReadRows(c.Name + ".tsv")
+		rows[c.Name] = data
+		if err != nil {
+			return err
+		}
+		if _, err := c.ReadRows("x.tsv"); err == nil || err.Error() != "rows names x.tsv, which is not beside it in "+map[string]string{"t": "lbl", "u": "lbl/sub"}[c.Name] {
+			t.Errorf("ReadRows(x.tsv) = %v, want an error saying x.tsv is not beside %s", err, c.Name)
+		}
+		return nil
+	})
+	if err != nil || rows["t"] != "key\nA\n" || rows["u"] != "key\nB\n" {
+		t.Errorf("Walk = %v, read %q", err, rows)
+	}
+}
+
+func TestWalkIgnoresARowsFileNoCategoryNames(t *testing.T) {
+	if _, err := walk(t, Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "a.tsv": {Data: []byte("key\n")}}, label: "lbl"}); err != nil {
+		t.Errorf("Walk = %v, want a.tsv ignored", err)
+	}
+}
+
+func TestARowsFileMayBeHidden(t *testing.T) {
+	var rows string
+	err := Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, ".a.tsv": {Data: []byte("key\nA\n")}}}.Walk(func(c Category) error {
+		var err error
+		rows, err = c.ReadRows(".a.tsv")
+		return err
+	})
+	if err != nil || rows != "key\nA\n" {
+		t.Errorf("Walk = %v, read %q, want the hidden rows file read", err, rows)
+	}
+}
+
+func TestWalkLabelsWhatACategoryFileGetsWrong(t *testing.T) {
+	for _, c := range []struct {
+		name    string
+		fs      fstest.MapFS
+		compile error
+		want    string
+	}{
+		{"its compile", fstest.MapFS{"a.json": {Data: []byte(`"x"`)}}, errors.New("bad"), "lbl/a.json: bad"},
+		{"its JSON", fstest.MapFS{"a.json": {Data: []byte(`{`)}}, nil, "lbl/a.json: line 1, column 1: unexpected end of JSON input"},
+		{"its name", fstest.MapFS{"a.b.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b.json: category "},
+		{"its folder's name", fstest.MapFS{"a.b/c.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b: folder "},
+	} {
+		err := Source{fsys: c.fs, label: "lbl"}.Walk(func(Category) error { return c.compile })
+		if err == nil || !strings.HasPrefix(err.Error(), c.want) {
+			t.Errorf("%s: Walk = %v, want it to start %q", c.name, err, c.want)
+		}
+	}
+}
+
+func TestWalkChecksAFolderNameOnlyWhenItHoldsData(t *testing.T) {
+	if _, err := walk(t, Source{fsys: fstest.MapFS{"a.b/notes.txt": {Data: []byte("x")}, "c.json": {Data: []byte(`"x"`)}}}); err != nil {
+		t.Errorf("Walk = %v, want a folder holding no data skipped whatever its name", err)
+	}
+}
+
+func TestWalkRefusesADiskPathThatIsNoDirectory(t *testing.T) {
+	dir := t.TempDir()
+	for _, c := range []struct{ path, want string }{
+		{"", "a data path is empty"},
+		{filepath.Join(dir, "none"), "data path " + filepath.Join(dir, "none") + ": no such directory"},
+	} {
+		_, err := walk(t, Dir(c.path))
+		if err == nil || err.Error() != c.want {
+			t.Errorf("Walk(%q) = %v, want %q", c.path, err, c.want)
+		}
+	}
+}
+
+func TestLoadHandsOneCategoryAndItsRows(t *testing.T) {
+	src := Source{fsys: fstest.MapFS{
+		"sub/t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
+		"sub/t.tsv":  {Data: []byte("key\nA\n")},
+		"sub/u.json": {Data: []byte(`{`)},
+	}}
+	var got handed
+	var rows string
+	err := src.Load([]string{"sub"}, "t", func(c Category) error {
+		got = handed{c.Folders, c.Name, c.JSON.Any()}
+		var err error
+		rows, err = c.ReadRows("t.tsv")
+		return err
+	})
+	if err != nil || got.name != "t" || !reflect.DeepEqual(got.dir, []string{"sub"}) || rows != "key\nA\n" {
+		t.Errorf("Load = %v, handed %#v with rows %q", err, got, rows)
+	}
+}
+
+func TestDirReadsTheDirectoryAndLabelsItsErrorsWithIt(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "a.json"), []byte(`{`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := walk(t, Dir(dir))
+	if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "a.json"))+": ") {
+		t.Errorf("Walk = %v, want the error labelled with the directory", err)
+	}
+	if _, err := walk(t, Dir(filepath.Join(dir, "a.json"))); err == nil || err.Error() != "data path "+filepath.Join(dir, "a.json")+": not a directory" {
+		t.Errorf("Walk = %v, want a file refused as no directory", err)
+	}
+}
+
+func TestAMountedDirNamesItsFilesWhereTheyAreOnDisk(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "sub", "a.json"), []byte(`{`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := walk(t, Dir(dir).Mounted("my.data"))
+	if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "sub", "a.json"))+": ") {
+		t.Errorf("Walk = %v, want the error to name the file on disk", err)
+	}
+}
+
+func TestAMountedTreeListsAndReadsUnderItsMount(t *testing.T) {
+	m := mounted{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "sub/b.json": {Data: []byte(`"y"`)}}, at: "my/data"}
+	for dir, want := range map[string][]string{".": {"my"}, "my": {"data"}, "my/data": {"a.json", "sub"}, "my/data/sub": {"b.json"}} {
+		entries, err := fs.ReadDir(m, dir)
+		var got []string
+		for _, e := range entries {
+			got = append(got, e.Name())
+		}
+		if err != nil || !reflect.DeepEqual(got, want) {
+			t.Errorf("ReadDir(%q) = %v, %v, want %v", dir, got, err, want)
+		}
+	}
+	if b, err := fs.ReadFile(m, "my/data/sub/b.json"); err != nil || string(b) != `"y"` {
+		t.Errorf(`ReadFile("my/data/sub/b.json") = %q, %v, want "y"`, b, err)
+	}
+	if _, err := fs.ReadFile(m, "other/a.json"); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf(`ReadFile("other/a.json") = %v, want it not to exist`, err)
+	}
+}
+
+func TestAMountedSourceNamesItsManifestBesideItsFiles(t *testing.T) {
+	dir := t.TempDir()
+	for name, c := range map[string]struct {
+		src  Source
+		want string
+	}{
+		"an fs.FS":    {FS(fstest.MapFS{}).Mounted("my.data"), "my/data/.fejkdata.json"},
+		"a data path": {Dir(dir).Mounted("my.data"), filepath.ToSlash(filepath.Join(dir, ".fejkdata.json"))},
+	} {
+		if got := c.src.ManifestPath(); got != c.want {
+			t.Errorf("%s: ManifestPath() = %q, want %q", name, got, c.want)
+		}
+	}
+	m, err := FS(fstest.MapFS{ManifestFile: {Data: []byte(`{"namespace": "my.data"}`)}}).Mounted("my.data").Manifest()
+	if err != nil || m.Namespace != "my.data" {
+		t.Errorf("Manifest() after Mounted = %+v, %v, want the manifest read from beside the files", m, err)
+	}
+}

internal/datafiles/manifest.go

@@ -0,0 +1,246 @@
+package datafiles
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
+)
+
+// ManifestFile is the file at a source's root that describes the source.
+const ManifestFile = ".fejkdata.json"
+
+// Manifest is what a source's manifest says of it.
+type Manifest struct {
+	// Index is every category of the source by dot path; nil where the manifest has none.
+	Index map[string]IndexEntry
+	// Namespace is the dot path the source's root sits under, "" for the namespace's root.
+	Namespace string
+}
+
+// IndexEntry is what an index says of a category: its parent table, "" for none, and the
+// paths List advertises below it.
+type IndexEntry struct {
+	Parent string   `json:"parent,omitempty"`
+	Paths  []string `json:"paths"`
+}
+
+// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
+func (s Source) Manifest() (Manifest, error) {
+	if err := s.check(); err != nil {
+		return Manifest{}, err
+	}
+	b, err := fs.ReadFile(s.fsys, path.Join(s.at, ManifestFile))
+	if errors.Is(err, fs.ErrNotExist) {
+		return Manifest{}, nil
+	}
+	if err != nil {
+		return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
+	}
+	m, err := decodeManifest(b)
+	if err != nil {
+		return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
+	}
+	return m, nil
+}
+
+// ManifestPath is the manifest's path as errors name it.
+func (s Source) ManifestPath() string { return s.labelled(path.Join(s.at, ManifestFile)) }
+
+// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
+// reports every mistake: first each entry's own, in path order, then each parent naming
+// no entry.
+func decodeManifest(b []byte) (Manifest, error) {
+	raw, err := jsonvalue.Decode(b)
+	if err != nil {
+		return Manifest{}, err
+	}
+	top, isObject := raw.Any().(map[string]any)
+	if !isObject {
+		return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any()))
+	}
+	var m Manifest
+	var errs []error
+	for _, k := range sortedKeys(top) {
+		switch k {
+		case "index":
+			var indexErrs []error
+			m.Index, indexErrs = decodeIndex(top[k])
+			errs = append(errs, indexErrs...)
+		case "namespace":
+			var err error
+			m.Namespace, err = decodeNamespace(top[k])
+			errs = append(errs, err)
+		default:
+			errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index and namespace", k))
+		}
+	}
+	for _, p := range sortedKeys(m.Index) {
+		if m.Namespace != "" && !strings.HasPrefix(p, m.Namespace+".") {
+			errs = append(errs, fmt.Errorf("index entry %q lies outside the namespace %q", p, m.Namespace))
+		}
+	}
+	return m, errors.Join(errs...)
+}
+
+func decodeNamespace(v any) (string, error) {
+	namespace, isString := v.(string)
+	if !isString {
+		return "", fmt.Errorf("namespace must be a string, not %s", jsonvalue.Kind(v))
+	}
+	if namespace == "" {
+		return "", nil
+	}
+	if err := checkCategoryPath(namespace); err != nil {
+		return "", fmt.Errorf("namespace %w", err)
+	}
+	return namespace, nil
+}
+
+func decodeIndex(v any) (map[string]IndexEntry, []error) {
+	entries, isObject := v.(map[string]any)
+	if !isObject {
+		return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))}
+	}
+	index := map[string]IndexEntry{}
+	var errs []error
+	for _, p := range sortedKeys(entries) {
+		e, err := decodeEntry(p, entries[p])
+		if err != nil {
+			errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
+		}
+		index[p] = e
+	}
+	for _, p := range sortedKeys(index) {
+		for _, err := range []error{checkOutside(index, p), checkParent(index, p)} {
+			if err != nil {
+				errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
+			}
+		}
+	}
+	return index, errs
+}
+
+// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no
+// mistake twice.
+func decodeEntry(p string, v any) (IndexEntry, error) {
+	var e IndexEntry
+	errs := []error{checkCategoryPath(p)}
+	fields, isObject := v.(map[string]any)
+	if !isObject {
+		errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v)))
+		return e, errors.Join(errs...)
+	}
+	if _, has := fields["paths"]; !has {
+		errs = append(errs, fmt.Errorf("paths is missing"))
+	}
+	for _, k := range sortedKeys(fields) {
+		switch k {
+		case "parent":
+			parent, err := decodeParent(fields[k])
+			e.Parent = parent
+			errs = append(errs, err)
+		case "paths":
+			var err error
+			e.Paths, err = decodePaths(fields[k])
+			errs = append(errs, err)
+		default:
+			errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k))
+		}
+	}
+	return e, errors.Join(errs...)
+}
+
+func decodeParent(v any) (string, error) {
+	parent, isString := v.(string)
+	if !isString {
+		return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v))
+	}
+	if err := grammar.CheckIdentifier(parent); err != nil {
+		return "", fmt.Errorf("parent %w", err)
+	}
+	return parent, nil
+}
+
+// decodePaths decodes a list of paths below a category, "" being the category itself.
+func decodePaths(v any) ([]string, error) {
+	items, isList := v.([]any)
+	if !isList {
+		return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v))
+	}
+	out := make([]string, 0, len(items))
+	seen := map[string]bool{}
+	var errs []error
+	for i, item := range items {
+		q, isString := item.(string)
+		if !isString {
+			errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item)))
+			continue
+		}
+		if q != "" {
+			if err := checkCategoryPath(q); err != nil {
+				errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err))
+				continue
+			}
+		}
+		if seen[q] {
+			errs = append(errs, fmt.Errorf("paths item %d repeats %q", i+1, q))
+			continue
+		}
+		seen[q] = true
+		out = append(out, q)
+	}
+	return out, errors.Join(errs...)
+}
+
+// checkOutside refuses an entry whose path passes through another entry's category.
+func checkOutside(index map[string]IndexEntry, p string) error {
+	segs := strings.Split(p, ".")
+	for n := 1; n < len(segs); n++ {
+		outer := strings.Join(segs[:n], ".")
+		if _, isEntry := index[outer]; isEntry {
+			return fmt.Errorf("%q is a category, so it holds no other", outer)
+		}
+	}
+	return nil
+}
+
+// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
+// from one source.
+func checkParent(index map[string]IndexEntry, p string) error {
+	parent := index[p].Parent
+	if parent == "" {
+		return nil
+	}
+	sibling := parent
+	if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
+		sibling = p[:dot+1] + parent
+	}
+	if _, ok := index[sibling]; !ok {
+		return fmt.Errorf("parent %q names no entry in this entry's folder; a table in an indexed source links only to a parent in the same index, so to link under another source's table, drop this index, and the source loads whole at start", parent)
+	}
+	return nil
+}
+
+func checkCategoryPath(p string) error {
+	for _, seg := range strings.Split(p, ".") {
+		if err := grammar.CheckIdentifier(seg); err != nil {
+			return fmt.Errorf("%q: %w", p, err)
+		}
+	}
+	return nil
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

internal/datafiles/mount.go

@@ -0,0 +1,73 @@
+package datafiles
+
+import (
+	"io/fs"
+	"strings"
+)
+
+// mounted is fsys with its root moved down to the folder at, so a file f of fsys is at/f. It
+// serves what a walk asks of a tree, a folder's entries and a file's bytes: the folders above
+// at list only the next folder towards it, and opening one fails.
+type mounted struct {
+	fsys fs.FS
+	at   string
+}
+
+func (m mounted) Open(name string) (fs.File, error) {
+	if inner, below := m.inner(name); below {
+		return m.fsys.Open(inner)
+	}
+	return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+}
+
+func (m mounted) ReadFile(name string) ([]byte, error) {
+	if inner, below := m.inner(name); below {
+		return fs.ReadFile(m.fsys, inner)
+	}
+	return nil, &fs.PathError{Op: "read", Path: name, Err: fs.ErrNotExist}
+}
+
+func (m mounted) ReadDir(name string) ([]fs.DirEntry, error) {
+	if inner, below := m.inner(name); below {
+		return fs.ReadDir(m.fsys, inner)
+	}
+	rest, above := strings.CutPrefix(m.at, name+"/")
+	if name == "." {
+		rest, above = m.at, true
+	}
+	if !above {
+		return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
+	}
+	next, _, _ := strings.Cut(rest, "/")
+	return []fs.DirEntry{folderEntry(next)}, nil
+}
+
+// inner is name as fsys spells it, where name lies at or below at.
+func (m mounted) inner(name string) (string, bool) {
+	inner, below := underMount(m.at, name)
+	return inner, below && fs.ValidPath(name)
+}
+
+// underMount is name as the tree mounted at at spells it, where name lies at or below at.
+func underMount(at, name string) (string, bool) {
+	switch name {
+	case at:
+		return ".", true
+	case "":
+		return name, false
+	}
+	if at == "" {
+		return name, true
+	}
+	return strings.CutPrefix(name, at+"/")
+}
+
+// folderEntry is a folder above the mount, as a folder's entries list it.
+type folderEntry string
+
+func (e folderEntry) Name() string      { return string(e) }
+func (e folderEntry) IsDir() bool       { return true }
+func (e folderEntry) Type() fs.FileMode { return fs.ModeDir }
+func (e folderEntry) Info() (fs.FileInfo, error) {
+	return nil, &fs.PathError{Op: "stat", Path: string(e), Err: fs.ErrNotExist}
+}

internal/datatype/datatype.go

@@ -0,0 +1,39 @@
+// Package datatype names what a record column holds.
+package datatype
+
+import "fmt"
+
+// DataType is what a record column holds, which decides how a record writes its value.
+type DataType int
+
+// The datatypes a column declares with "datatype"; a column without one is a string.
+const (
+	String DataType = iota
+	Integer
+	Number
+	Boolean
+)
+
+// Count is the number of datatypes, for an array indexed by one.
+const Count = Boolean + 1
+
+var (
+	names = [Count]string{"string", "integer", "number", "boolean"}
+	nouns = [Count]string{"text", "an integer", "a number", "a boolean"}
+)
+
+// String is the datatype as data spells it.
+func (d DataType) String() string {
+	if d < 0 || d >= Count {
+		return fmt.Sprintf("DataType(%d)", int(d))
+	}
+	return names[d]
+}
+
+// Noun is the datatype as an error names what a value is.
+func Noun(d DataType) string {
+	if d < 0 || d >= Count {
+		return d.String()
+	}
+	return nouns[d]
+}

internal/datatype/datatype_test.go

@@ -0,0 +1,31 @@
+package datatype
+
+import "testing"
+
+func TestEachDatatypeIsSpelledAsDataWritesIt(t *testing.T) {
+	for d, want := range map[DataType][2]string{
+		String:  {"string", "text"},
+		Integer: {"integer", "an integer"},
+		Number:  {"number", "a number"},
+		Boolean: {"boolean", "a boolean"},
+	} {
+		if d.String() != want[0] || Noun(d) != want[1] {
+			t.Errorf("%d: String = %q, Noun = %q, want %q", int(d), d.String(), Noun(d), want)
+		}
+	}
+	if Count != 4 {
+		t.Errorf("Count = %d, want the four datatypes", Count)
+	}
+}
+
+func TestStringNamesADatatypeNoDataWrites(t *testing.T) {
+	if got := DataType(9).String(); got != "DataType(9)" {
+		t.Errorf("String = %q, want DataType(9)", got)
+	}
+}
+
+func TestNounNamesADatatypeNoDataWrites(t *testing.T) {
+	if got := Noun(DataType(9)); got != "DataType(9)" {
+		t.Errorf("Noun = %q, want DataType(9)", got)
+	}
+}

internal/drawstate/drawstate.go

@@ -0,0 +1,37 @@
+// Package drawstate holds a generator's seeded randomness and its {seq()} counters.
+package drawstate
+
+import (
+	"math/rand/v2"
+	"sort"
+)
+
+// State is one generator's draw state, kept across its renders.
+type State struct {
+	rand     *rand.Rand
+	counters map[string]uint64
+}
+
+// New builds the state a seed reproduces.
+func New(seed uint64) *State {
+	return &State{rand: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)), counters: map[string]uint64{}}
+}
+
+func (s *State) IntN(n int) int { return s.rand.IntN(n) }
+
+func (s *State) Float64() float64 { return s.rand.Float64() }
+
+// Weighted draws an index into cum, a running sum of weights of 0 or more with a positive total,
+// with odds by weight, so an index of weight 0 is never drawn.
+func (s *State) Weighted(cum []float64) int {
+	total := cum[len(cum)-1]
+	x := s.Float64() * total
+	// x stays below total: a product of a float64 under 1 and a positive float64 rounds below it.
+	return sort.Search(len(cum), func(i int) bool { return cum[i] > x })
+}
+
+// Seq advances the counter {seq(key)} reads, or {seq()} for an empty key, and returns it.
+func (s *State) Seq(key string) uint64 {
+	s.counters[key]++
+	return s.counters[key]
+}