Make great documentation goal 1.2, and renumber goals 13 to 16 as 12 to 15 wherever they are cited

.github/workflows/packages.yml

@@ -0,0 +1,162 @@
+name: Packages
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        required: true
+    secrets:
+      TAP_GITHUB_TOKEN:
+        required: true
+      PACKAGES_GPG_KEY:
+        required: true
+      PACKAGES_RSA_KEY:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The released tag to package, such as v0.1.0
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: packages
+  cancel-in-progress: false
+
+jobs:
+  packages:
+    name: Binaries, packages and image for ${{ inputs.tag }}
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+      packages: write
+    outputs:
+      latest: ${{ steps.latest.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.tag }}
+          path: src
+          fetch-depth: 0
+          persist-credentials: false
+      # Check out the packaging config at the workflow's commit, so a tag cut before the
+      # config existed can still be packaged.
+      - uses: actions/checkout@v7
+        with:
+          path: tooling
+          persist-credentials: false
+      - name: The Go version the gate tests
+        id: go
+        run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: false
+      - name: Channels holding one version
+        id: latest
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
+          if [ "$latest" = "$TAG" ]; then
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
+            echo "OLDER_TAG=1" >> "$GITHUB_ENV"
+          fi
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: v2.18.3
+          workdir: src
+          args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      - name: The built binary and the published image print the tag
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
+          test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  dockerhub:
+    name: Docker Hub image for ${{ inputs.tag }}
+    needs: packages
+    runs-on: ubuntu-26.04
+    permissions:
+      packages: read
+    steps:
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Copy the GHCR image to Docker Hub, and check it prints the tag
+        env:
+          LATEST: ${{ needs.packages.outputs.latest }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
+          if [ "$LATEST" = true ]; then
+            tags+=(-t docker.io/larvit/fejkdata:latest)
+          fi
+          docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
+          test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  repos:
+    name: apt, dnf and apk repositories for ${{ inputs.tag }}
+    needs: packages
+    if: needs.packages.outputs.latest == 'true'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: The release's Linux packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
+      - uses: docker/setup-buildx-action@v4
+      - name: Install fejkdata from each repository, then export them
+        env:
+          PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+          PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          build() {
+            docker buildx build -f release-tooling/package-repos.Dockerfile \
+              --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
+              --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
+          }
+          build --target checks --output type=local,dest=checks
+          build --target site --output type=local,dest=site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - id: deploy
+        uses: actions/deploy-pages@v5

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * *'
 
 permissions:
   contents: read
@@ -11,9 +14,10 @@ permissions:
 jobs:
   test:
     name: vet + fmt + tests
-    runs-on: ubuntu-24.04
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -26,11 +30,16 @@ jobs:
           else
             base='${{ github.event.before }}'
           fi
-          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
             echo "code=false" >> "$GITHUB_OUTPUT"
           else
             echo "code=true" >> "$GITHUB_OUTPUT"
           fi
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
+            echo "packaging=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "packaging=true" >> "$GITHUB_OUTPUT"
+          fi
           echo "base=$base" >> "$GITHUB_OUTPUT"
       - name: CHANGELOG entry for data changes
         run: |
@@ -48,22 +57,84 @@ jobs:
       # failure is version-specific.
       - name: Lowest supported Go
         if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
-        run: docker build --build-arg GO_VERSION=1.22.12 .
+        run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
+      - name: Package repositories, signed with throwaway keys
+        if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
+        run: |
+          docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
+          PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
+            docker build -f release-tooling/package-repos.Dockerfile --target checks \
+            --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
 
-  release:
+  vulnerabilities:
-    name: Gitea release from CHANGELOG.md
+    name: Known vulnerabilities in this Go version and the required modules
-    needs: test
+    runs-on: ubuntu-26.04
-    if: github.event_name == 'push'
+    timeout-minutes: 10
-    runs-on: ubuntu-24.04
     permissions:
+      actions: write
       contents: write
+      pull-requests: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - run: python3 release-tooling/publish_release.py
+      - name: Fetch govulncheck's report from vuln.go.dev
+        run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
+      - name: File each one in todo.md
+        id: file
+        run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
+      - name: Propose the filed items
+        if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
         env:
-          GITEA_API_URL: ${{ github.api_url }}
+          GH_TOKEN: ${{ github.token }}
-          GITEA_REPOSITORY: ${{ github.repository }}
+        run: |
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
+            exit 0
+          fi
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git switch -C vulnerabilities
+          git commit -qam "File the known vulnerabilities govulncheck reports"
+          git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
+          gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
+            --body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
+          # A pull request the workflow's token opens starts no workflow.
+          gh workflow run test.yml --ref vulnerabilities
+
+  release:
+    name: Release from CHANGELOG.md
+    needs: [test, vulnerabilities]
+    if: github.event_name == 'push'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    outputs:
+      tag: ${{ steps.publish.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - id: publish
+        run: python3 release-tooling/publish_release.py
+        env:
+          FORGE_API_URL: ${{ github.api_url }}
+          FORGE_REPOSITORY: ${{ github.repository }}
+          FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SHA: ${{ github.sha }}
+
+  packages:
+    needs: release
+    if: needs.release.outputs.tag != ''
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/packages.yml
+    with:
+      tag: ${{ needs.release.outputs.tag }}
+    secrets:
+      TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+      PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

AGENTS.md

@@ -1,27 +1,137 @@
 # Rules
 
-- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check.
+- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
 - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
 - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
 - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
-- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it
+- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
-  offers fails with "is it still open?", as does a merge whose required checks are
+  only, `gh pr merge --rebase` where the branch is already on top of main.
-  still pending, so read the checks before believing the style is the problem. Use
+- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
-  `tea api /repos/{owner}/{repo}/pulls/<n>/merge -f Do=fast-forward-only`.
+  its status once when a PR is marked ready, answer a review if one is there, and never
-- Gitea queues Actions runs that `/repos/{owner}/{repo}/actions/tasks` does not
+  wait for one.
-  list, so an empty task list says nothing about whether CI ran; read
+- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
-  `/repos/{owner}/{repo}/commits/<sha>/status`.
+  nothing else merges — no feature, no category, no data — bar a security fix, a
-- README goal 2 is gated at 7.0 on the `comprehension-panel` skill, and below it
+  dependency bump, a repair of behaviour the README documents, the infrastructure
-  nothing else merges — no feature, no category, no data — bar a security fix and a
+  the round itself runs on, and a chunk the comprehension-floor decision lists as
-  dependency bump. `todo.md` carries the round: the nine-seat findings run at depth 1,
+  shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
-  a PR per item it names, then the run again, until the score passes. At or above 7.0
+  findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
+  again, until the score passes. At or above 7.0
   every pull request scores with the four-seat run and answers it in one run. No merge
-  lowers the last score.
+  lowers the last score its own panel recorded: the nine-seat round and the four-seat
+  run are two ratchets, never compared with each other. A lower score blocks the merge
+  even where it may be panel noise: fix what the seats name until the score rises. A PR
+  that only records a run's scores and files what its seats name merges even when a
+  score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
 - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
-- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
-- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
 - Prose naming a source states what that source states: read the register's own field
   before paraphrasing it, and name the register you read before claiming none
   publishes a spelling.
 - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
-- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed.
+- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
+
+# Decisions
+
+In [docs/decisions.md](docs/decisions.md):
+
+- GitHub is canonical, and the module path names it
+- The Gitea copy stays, as a pull mirror of GitHub
+- The vocabulary sits below `doc.go`'s package clause, not in the package doc
+- Options and fields share one namespace
+- `{a|b}` stays beside nested choices
+- Flags follow getopt_long
+- A struct tag is a template by its shape
+- The CLI renders exactly the template on its stdin, and adds nothing
+- The CLI reads only the library's public API
+- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+- `FakeTemplate` and `NewTemplate` both stay
+- The shipped data is embedded, not discovered
+- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+- Every `{…}` draws afresh, and only a name keeps a pick
+- Reference sigils follow the filesystem
+- A change to what exists is a major; a minor only adds
+- An error is a contract by what it names, not its bytes
+- A load reports every mistake at once
+- Formats are checked after all data is loaded, so their errors name the category
+- Raising the lowest supported Go is a major
+- The format check runs on the latest Go only
+- The changelog heading is the one spelling of a release; CI cuts the tag
+- A `--data-path` override rebinds every reference to the category it replaces
+- The repeat cap bounds renders, not bytes
+- 64-bit targets only
+- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+- Samples say what they emit, transforms what they do
+- A record is a template seen as columns; a Go struct is the one second schema
+- A struct's records follow Go's field access, and compile on first use
+- A category may reference itself, and a record's fences run at load
+- A record's column set is fixed before the first draw
+- Null is a `null` item, not a rate
+- A typed column holds one value, not composed text
+- A column that only reads one reference or name is the column it reads
+- A typed column's calc is refused unless proven
+- `Column` carries text, not a Go value
+- A unit takes the stem of what it is, and a file the stem of the units it holds
+- One name, one meaning
+- The performance gate asserts allocations, not wall-clock time
+- Rows live in a TSV, the shape in JSON
+- A selector is bracketed, and a dot inside it is literal
+- `parent` names the link column and the table alike
+- After a row, a path names a column or a linked table
+- A selected row is fixed, not drawn
+- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+- A table selects by one key column, a code, never a free-form name
+- A parent row with no child row is a load error
+- A table is a record of string columns
+- The key map is built at load, the rest on first draw
+- Two categories may name one TSV
+- A cell may read a reference, and each row its own
+- A path is walked once without drawing before it is walked for real
+- A country's postal codes and streets are siblings under its locality
+- A locale package imports its country's geo package, and its `Modules()` returns both
+- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+- A locale's `address` binds its country's record once and prints it whole
+- A postort's kommun comes from its name, its tätort or its codes, never from distance
+- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+- `--list` stays a plain list of paths
+- A layout is always quoted
+- A title is a table under `sex`
+- A table owns the spelling of a selector on it
+- No builtin reads the clock, so a date is bounded by days, never by an age
+- `misc` is what every locale shares
+- A register's canonical spelling loses to the one its domain writes
+- `misc.territory` is the spine, and a `misc` table naming a territory links to it
+- A table whose register publishes no frequency draws evenly
+- `misc.timezone` weighs a zone by the people living in it
+- `misc.car` is one flat table, not a make linked to its models
+- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+- `misc.tld` keys carry the leading dot, where other tables key on a bare code
+- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+- `misc.territory` carries a currency code, it does not link to `misc.currency`
+- An extension may name two media types
+- The Swedish ids draw Skatteverket's test series
+- The US given names come from a mirror of the SSA file
+- `List` advertises direct descents only
+- A path draws through its compiled steps
+- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+- A country with no open address register covering it whole ships its places and postal codes without streets
+- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
+
+# Scoring run
+
+The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
+
+- `{language}`: Go
+- `{kind}`: a Go library and CLI that renders fake data from JSON templates
+- `{domain}`: fake test-data generation and template languages
+- `{domain docs}`: the documentation of any fake-data library, such as Faker
+- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another

CHANGELOG.md

@@ -8,85 +8,99 @@ replacement, and each removed path, column or flag.
 
 ### Added
 
-- First release: the CLI, the library and the shipped data set.
+- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
-- Table categories: a category JSON naming a `rows` TSV beside it, with the options
+  dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
-  `key`, `name`, `weight` and `parent`; a path selects a row by key or name,
+  the CLI installs without Go. 0.1.0 ships them too; the README's
-  `misc.territory[SE]`, and descends to a linked table by name; linked tables draw
+  [Install](https://github.com/larvit/fejkdata#install) lists each command.
-  consistently within one render and draw group. `rows` is an option, so no
+
-  template may carry a field of that name. A `name` without a `key` resolves
+## [0.1.0] - 2026-10-13
-  inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
+
-  `parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any
+The first release: a Go library and a CLI that render locale-aware fake data from JSON
-  table above it, a table whose format or cell references its own family, and,
+templates, and the data they ship with. The
-  within one render and draw group, a path drawing a table another path selects a
+[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
-  row of, or two paths pinning different rows of one table.
+Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
-- `New` refuses a root choice of templates sharing one format and one set of string
+library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
-  fields, naming the rows TSV to write instead.
+and any release may change what a seed draws, so pin one exact version
-- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and
+([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
-  `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has
+
-  a capital, a currency and a TLD, with the columns `calling-code`, `capital`,
+### Added
-  `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country`
+
-  holding the alpha-2 code of the sovereign state it belongs to and its own where it
+- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
-  is one; `misc.currency` the
+  `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
-  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
+  repeats a run's output while the version and data stay the same, and without it each
-  added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a
+  run takes a new seed; `-n` renders the template that many times, and `--list`
-  639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each
+  prints the paths the data offers.
-  code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each
+  `--data-path` layers a directory of your own data over the shipped set, and
-  type mime-db records as IANA-registered and gives a filename extension.
+  `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
-  `DATA-LICENSES.md` lists each source.
+  a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
-- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone`
+  error such as data that fails to load, and `2` on misuse.
-  is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the
+- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
-  zone's standard UTC offset and `territory` linking to `misc.territory`, so
+  `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
-  `misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory
+  from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
-  `misc.territory` does not ship, Antarctica's among them, and the constant `UTC`
+  without data, it fails and names `WithDataFS`. The same seed repeats the draws
-  are gone; spell that one as the text `"UTC"`. A `weight` column carries the
+  while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
-  population GeoNames records in each zone, so a draw favours the zones people live
+  and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
-  in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`,
+  `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
-  `device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a
+  `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
-  `model` column.
+  `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
-- `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries as tables.
+  and `IsTemplate` tells an inline template from a path. Data that fails to load gives
-  `misc.httpmethod` is the eight methods RFC 9110 defines and PATCH, carrying
+  an error matching `ErrLoad`, and a record with no fields gives one matching
-  `safe` and `idempotent` as `true` or `false`; `misc.protocol` the keyword, its
+  `ErrNoColumns`.
-  `number` and its `name`, selectable by either; `misc.port` renders a TCP port number,
+- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
-  with the IANA `service` name beside it, keeping the first service the registry
+  `cli.Main(modules...)` runs it over the modules given, so
-  describes for a port so a number selects one row.
+  `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
-- `misc.tld` is the IANA root zone as a table: every delegated TLD keyed with its
+  That CLI names itself by the file it runs from, and its `--version` prints its own
-  leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode`
+  module's version before fejkdata's.
-  column holding the form the register displays, so `misc.tld[.рф]` selects the row
+- The template language
-  `.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship.
+  ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
-- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes,
+  Every value in a data file is a string, a choice, a template, a table, a number or a
-  `err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by
+  boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
-  either, with the `severity` RFC 5424 spells beside it.
+  root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
-- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+  or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
-  `locality`, `postal-code` and `street`, weighted by population and address counts
+  pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
-  and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
+  digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
-  `address` record over one consistent draw of them. `sv_SE.address` and
+  `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
-  `en_US.address` read those records, so `en_US.address.street` no longer carries
+  transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
-  `name` and `suffix`, and a locale folder loads only beside `geo`.
+  grammar, could mean two things or cannot render a valid value.
-- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or
+- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
-  within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`.
+  a template seen as columns. A column is a string unless it declares a `datatype`,
-  `sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so
+  `integer`, `number` or `boolean`, and a `null` item draws it as null.
-  `date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`,
+- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
-  `time.sec` and `time.ampm` are no longer paths — a part of a date is now its own
+  category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
-  `{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant.
+  `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
-- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by
+  linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
-  bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a
+  `{/misc.timezone[Europe/Oslo]..territory.name}`.
-  name both sexes carry is a row under each. `person` reads them, so its columns are
+- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
-  `first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are
+  a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
-  no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead.
+  manifest at a source's root may carry a `namespace`, the dot path the source mounts
-  `en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms`
+  under, and an `index`, with which each category loads on the first call reaching it.
-  without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the
+- The shipped data, in Go packages that each export `FS` and `Modules()`:
-  same shape.
+  `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
-- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series
+  `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
-  from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose
+  `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
-  `ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the
+  [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
-  ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so
+  names each source.
-  `--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new
+  - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
-  and carries the same three. `person.prefix` is null where a person has no title,
+    `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
-  where it used to be an empty string, so `--format sql` writes `NULL` and a `string`
+    the Census Bureau. A person's first name and sex agree, and so does an `en_US`
-  struct field reading it becomes `*string`.
+    person's title; `person.prefix` is null where a person has no title.
-- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can
+    `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
-  answer from the rest.
+    `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
-- An error names a spelling that runs: a layout is named single-quoted and free of
+    columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
-  its own quotes, a row of a table with no key is named as the path that selects it,
+    assign.
-  `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record
+  - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
-  that gives it one.
+    HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
+    symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
+    zones weighted by the people living in each, user agents and cars, all as tables;
+    and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
+    ObjectIds and UUIDs.
+  - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+    `locality`, `postal-code` and `street`, weighted by population and address counts
+    and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
+    and an `address` record that draws a street and a postal code inside one
+    locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
+    (postal town) the import can place ships with all its streets. A US place ships
+    only if it is incorporated and holds most of some postal code's land. Of those
+    places, every state capital but Honolulu ships, as does every place of 100,000
+    people or more and a tenth of all of them drawn by population. Each US place
+    carries a quarter of its streets, at least ten.

DATA-LICENSES.md

@@ -9,12 +9,12 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
 | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
 | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
 | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
-| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
+| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
 | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
 | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
 | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
 | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
-| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 12) | — | — | — |
+| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
 | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
 | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
 | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |

Dockerfile

@@ -1,19 +1,43 @@
-# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
+ARG GO_VERSION=1.27.2
-# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
+FROM golang:${GO_VERSION} AS portable
-# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
-# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
-ARG GO_VERSION=1.26.4
-FROM golang:${GO_VERSION}
 
 WORKDIR /app
 
-# Module layer cached separately from source.
 COPY go.mod ./
 RUN go mod download
 
 COPY . .
 RUN go vet ./... && \
 	go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
-	{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
-	  { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
 	go test -race ./...
+
+FROM golang:${GO_VERSION} AS vulnerabilities
+WORKDIR /app
+COPY . .
+RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
+
+FROM scratch AS vulnerability-report
+COPY --from=vulnerabilities /report.json /
+
+FROM python:3.14.8-slim AS release-tooling
+COPY release-tooling/ /release-tooling/
+RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
+
+FROM python:3.14.8-slim AS vulnerability-filing
+COPY vulnerability-filing/ /vulnerability-filing/
+# The filer's test files an item into the plan itself.
+COPY todo.md /todo.md
+RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
+
+FROM goreleaser/goreleaser:v2.18.3 AS packaging
+WORKDIR /check
+COPY release-tooling/goreleaser.yaml ./
+RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
+
+FROM portable
+# BuildKit skips a stage nothing copies from.
+COPY --from=release-tooling /tested /tmp/release-tooling-tested
+COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
+COPY --from=packaging /tested /tmp/packaging-tested
+RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
+	{ echo "unformatted files:"; echo "$unformatted"; exit 1; }

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2026 Timewave
+Copyright (c) 2026 Larvit
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

arm.go

@@ -0,0 +1,117 @@
+package fejkdata
+
+import (
+	"fmt"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// arm is one alternative of a {a|b} token or one operand, split into the head,
+// whose node `template.startOf` finds, and the tail of a dotted path into it.
+type arm struct {
+	spelling string // as written, for messages
+	head     string
+	tail     []string
+	levels   []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
+	steps    []pathStep
+	leaves   []node // every node the path may land on, one per variant it passes
+	kind     armKind
+	named    *nameBinding // namedRead: the binding of the name it reads through
+}
+
+// armKind is how expand reads an arm, fixed at compile.
+type armKind uint8
+
+const (
+	freshRead armKind = iota // drawn afresh at every read
+	namedRead                // read through a name, kept in its pick
+)
+
+// splitArm splits one name into head and tail. refs maps a reference to what
+// resolveRefs resolved it to; before that, a reference is whole.
+func splitArm(name string, refs map[string]resolvedRef) arm {
+	if grammar.IsRef(name) {
+		if r, resolved := refs[name]; resolved {
+			return pathArm(name, r.head, r.tail)
+		}
+		return pathArm(name, name, nil)
+	}
+	segs, err := grammar.SplitPath(name)
+	if err != nil || len(segs) == 1 {
+		return pathArm(name, name, nil)
+	}
+	return pathArm(name, segs[0], segs[1:])
+}
+
+func pathArm(name, head string, segs []string) arm {
+	return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
+}
+
+// key is the key of the leaf a lands on, the one key every way of writing this read shares.
+func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
+
+func checkSegments(a arm) error {
+	if len(a.tail) == 0 {
+		return nil
+	}
+	return grammar.CheckSegments(append([]string{a.head}, a.tail...))
+}
+
+// op is one compiled unit of a format string: a literal run, a name read,
+// or a builtin already prepared with its args.
+type op struct {
+	grammar.Token
+	arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
+	call builtinfunc.Call
+	// operands are the fields the builtin reads, in the order its operands func
+	// fixed; expand reads them before the call. nil for a builtin that reads none.
+	operands []arm
+}
+
+func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+	arms := make([]arm, len(names))
+	for i, name := range names {
+		a, err := t.compileArm(name, targets)
+		if err != nil {
+			return nil, err
+		}
+		arms[i] = a
+	}
+	return arms, nil
+}
+
+// compileArm compiles one read into a path: from the head it names, or, for a read through a
+// name, from what that name binds.
+func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
+	a := splitArm(name, t.refs.byName)
+	if !t.isName(a.head) {
+		start := t.startOf(a.head)
+		if start == nil {
+			panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
+		}
+		w := compilePath(start, a.tail)
+		a.steps, a.leaves = w.steps, w.leaves
+		return a, nil
+	}
+	b := t.nameScope.lookup(a.head)
+	if grammar.HasSelector(a.tail) {
+		if b.bindsField() {
+			return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
+		}
+		return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
+	}
+	target := targets[b]
+	full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
+	if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
+		return a, err
+	}
+	w := compilePath(target.start, full)
+	a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
+	a.levels = levelKeys(full, 0)
+	return a, nil
+}

bench_test.go

@@ -6,17 +6,17 @@ import (
 	"testing"
 )
 
-func benchGenerator(b *testing.B, dir string) *Generator {
+func benchGenerator(b *testing.B, data Option) *Generator {
 	b.Helper()
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
+	f, err := New(data, WithSeed(1))
 	if err != nil {
 		b.Fatal(err)
 	}
 	return f
 }
 
-func benchPath(b *testing.B, dir, path string) {
+func benchPath(b *testing.B, data Option, path string) {
-	f := benchGenerator(b, dir)
+	f := benchGenerator(b, data)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
 	}
 }
 
-func BenchmarkPerson(b *testing.B)       { benchPath(b, "data", "sv_SE.person") }
+func BenchmarkPerson(b *testing.B)       { benchPath(b, withShipped(), "sv_SE.person") }
-func BenchmarkAddress(b *testing.B)      { benchPath(b, "data", "sv_SE.address") }
+func BenchmarkAddress(b *testing.B)      { benchPath(b, withShipped(), "sv_SE.address") }
-func BenchmarkWord(b *testing.B)         { benchPath(b, "data", "sv_SE.word") }
+func BenchmarkWord(b *testing.B)         { benchPath(b, withShipped(), "sv_SE.word") }
-func BenchmarkCreditcard(b *testing.B)   { benchPath(b, "data", "misc.creditcard") }
+func BenchmarkCreditcard(b *testing.B)   { benchPath(b, withShipped(), "misc.creditcard") }
-func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") }
+func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
-func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, "data", "misc.uuid") }
+func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, withShipped(), "misc.uuid") }
 
 func tmpData(b *testing.B, name, body string) string {
 	b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
 
 func BenchmarkCalc(b *testing.B) {
 	dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
-	benchPath(b, dir, "inv")
+	benchPath(b, WithDataPath(dir), "inv")
 }
 
 func BenchmarkLongLiteral(b *testing.B) {
 	dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "sql")
+	benchPath(b, WithDataPath(dir), "sql")
 }
 
 func BenchmarkRepeat(b *testing.B) {
 	dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "many")
+	benchPath(b, WithDataPath(dir), "many")
 }
 
-// BenchmarkBound measures a format that binds a head and reads two paths from it —
+// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
-// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
+// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
-// two independent fields.
+// independent fields.
-func BenchmarkBound(b *testing.B) {
+func BenchmarkNamed(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
+	dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-func BenchmarkUnbound(b *testing.B) {
+func BenchmarkUnnamed(b *testing.B) {
 	dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
+// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
-// depth question is about.
+func BenchmarkNamedDeep(b *testing.B) {
-func BenchmarkBoundDeep(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
-	dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
+	benchPath(b, WithDataPath(dir), "addr")
-	benchPath(b, dir, "addr")
 }
 
-// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
+// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
-// expansion holds outgrow a single bucket.
+// outgrow a single bucket.
-func BenchmarkBoundWide(b *testing.B) {
+func BenchmarkNamedWide(b *testing.B) {
-	dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
+	dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
 		{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
 		{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
-	benchPath(b, dir, "row")
+	benchPath(b, WithDataPath(dir), "row")
+}
+
+func BenchmarkFirstFake(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		f, err := New(withShipped(), WithSeed(1))
+		if err != nil {
+			b.Fatal(err)
+		}
+		if _, err := f.Fake("sv_SE.address"); err != nil {
+			b.Fatal(err)
+		}
+	}
 }
 
 func BenchmarkNew(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if _, err := New(WithSeed(1)); err != nil {
+		if _, err := New(withShipped(), WithSeed(1)); err != nil {
 			b.Fatal(err)
 		}
 	}

builtins_test.go

@@ -2,11 +2,14 @@ package fejkdata
 
 import (
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 func TestBuiltinIDGenerators(t *testing.T) {
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
 	}
 }
 
-// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
+// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
 // each call, named counters independent, and the whole thing scoped to one
-// session so a fresh Generator restarts at 1.
+// draw state so a fresh Generator restarts at 1.
-func TestBuiltinSeqPerSession(t *testing.T) {
+func TestBuiltinSeqPerGenerator(t *testing.T) {
 	f := engine(1)
 	for i := 1; i <= 5; i++ {
 		if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
 	if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
 		t.Fatalf("repeat seq = %q, want 1,2,3", got)
 	}
-	// ...and a fresh session restarts from 1.
+	// ...and a fresh generator restarts from 1.
 	if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
-		t.Fatalf("new session seq = %q, want 1", got)
+		t.Fatalf("new generator seq = %q, want 1", got)
 	}
 }
 
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
 		`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
 	}
 	for _, tmpl := range bad {
-		if _, err := compile(parse(t, tmpl)); err == nil {
+		if _, err := resolved(t, tmpl); err == nil {
 			t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
 		}
 	}
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
 	return rem == 1
 }
 
-// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
-// supplies prep, and args parsed at compile only behind a check.
+// separates a reported invariant break from a silently wrong value.
-func TestRegistryShapes(t *testing.T) {
-	for name, b := range builtins {
-		if b.prep == nil {
-			t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
-		}
-		if b.arity != 0 && b.check == nil {
-			t.Errorf("builtin %q parses args in prep with no check", name)
-		}
-	}
-}
-
-// TestArgGuardsPanic pins the guards that report a builtin arg its check should
-// have rejected. No data reaches them — checkFunc runs a builtin's check before
-// compileOps ever calls prep — so they are exercised directly.
-func TestArgGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"atoi on an unvalidated arg": func() { atoi("nope") },
-		"atof on an unvalidated arg": func() { atof("nope") },
-	} {
-		mustPanic(t, name, call)
-	}
-}
-
-// mustPanic fails unless call panics, which is what separates a reported
-// invariant break from a silently wrong value.
 func mustPanic(t *testing.T, name string, call func()) {
 	t.Helper()
 	defer func() {
-		if recover() == nil {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
-			t.Errorf("%s: no panic, want the invariant reported", name)
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
 		}
 	}()
 	call()
 }
 
 func TestClassBuiltinArgs(t *testing.T) {
-	for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
+	for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
-		if _, err := compile(parse(t, bad)); err == nil {
+		if _, err := resolved(t, bad); err == nil {
 			t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
 		}
 	}
 	for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v", ok, err)
 		}
 	}
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
 	}
 }
 
-// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order,
+// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
-// the layout is quoted, names a field, and for time names no date field.
+// layout is quoted.
 func TestBuiltinDateArgs(t *testing.T) {
 	for tmpl, want := range map[string]string{
 		`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`:        "1990-13-01",
+		`"{time('2006-01-02 15:04')}"`:                        "write date(from,to,layout)",
 		`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`:        "is after",
-		`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`:        "write it as text",
 		`"{date(1990-01-01,1990-12-31,2006-01-02)}"`:          "'2006-01-02'",
 		`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`:    "'",
 		`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
-		`"{date(1990-01-01,1990-12-31,'x')}"`:                 "text",
-		`"{date(1990-01-01,1990-12-31,'')}"`:                  "text",
 		`"{date(1990-01-01,1990-12-31)}"`:                     "3 arguments",
 		`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`:     "'January 2, 2006'",
 		`"{time(3:04 PM, Mon)}"`:                              "'3:04 PM, Mon'",
 		`"{time(15:04)}"`:                                     "'15:04'",
-		`"{time('2006-01-02 15:04')}"`:                        "date(",
-		`"{time('x')}"`:                                       "text",
-		`"{date(1990-01-01,1990-12-31,'15:04')}"`:             "time('15:04')",
 	} {
-		_, err := compile(parse(t, tmpl))
+		_, err := resolved(t, tmpl)
 		if err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
 		}
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
 }
 
 // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
-// draws every second of it, and without one it could only emit one value.
+// draws every second of it, and without one it prints that day.
 func TestBuiltinDateSpansOneDay(t *testing.T) {
 	f := engine(1)
 	seen := map[string]bool{}
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
 	if len(seen) < 400 {
 		t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
 	}
-	_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`))
+	if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
-	if err == nil || !strings.Contains(err.Error(), "write it as text") {
+		t.Fatalf("one day in a date-only layout = %q, want that day", got)
-		t.Fatalf("one day in a date-only layout = %v, want it named a constant", err)
+	}
+}
+
+func TestClassBuiltins(t *testing.T) {
+	cases := map[string]*regexp.Regexp{
+		`"{digits(1)}"`:          regexp.MustCompile(`^[0-9]$`),
+		`"{digits(3)}"`:          regexp.MustCompile(`^[0-9]{3}$`),
+		`"{int(1,9)}"`:           regexp.MustCompile(`^[1-9]$`),
+		`"{upper(1)}"`:           regexp.MustCompile(`^[A-Z]$`),
+		`"{lower(1)}"`:           regexp.MustCompile(`^[a-z]$`),
+		`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
+	}
+	f := engine(7)
+	for tmpl, re := range cases {
+		for i := 0; i < 100; i++ {
+			if got := mustRender(t, f, tmpl); !re.MatchString(got) {
+				t.Fatalf("%s produced %q, want %s", tmpl, got, re)
+			}
+		}
+	}
+}
+
+func TestFunctionTokenLuhn(t *testing.T) {
+	// {luhn()} appends a Luhn check digit over the digits emitted so far in the
+	// current expansion (non-digits skipped but kept). It reads the output
+	// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
+	f := engine(1)
+	cases := map[string]string{
+		`"811218987{luhn()}"`:                      "8112189876",  // personnummer body
+		`"7992739871{luhn()}"`:                     "79927398713", // classic Luhn vector
+		`"811218-987{luhn()}"`:                     "811218-9876", // '-' skipped, kept
+		`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876",  // over a rendered token
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Fatalf("%s = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestBuiltinCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{upper(x)}"`,         // count must be an integer
+		`"{int(a,b)}"`,         // non-integer args
+		`"{int(5,1)}"`,         // min > max
+		`"{nanoid(-1)}"`,       // negative count
+		`"{float(1,2,-1)}"`,    // negative decimals
+		`"{float(NaN,NaN,2)}"`, // bounds must be finite
+		`"{float(Inf,Inf,2)}"`, // same-sign infinities
+		`"{float(1,NaN,2)}"`,   // one NaN bound
+		`"{float(-Inf,1,2)}"`,  // one infinite bound
+		`"{iban(US)}"`,         // unsupported country
+		`"{seq(a,b)}"`,         // seq takes at most one name
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestArgErrorsNameTheSpelling(t *testing.T) {
+	if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
+		t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
+	}
+}
+
+func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
+	for long, short := range map[string]string{
+		`"{float(1,2,02)}"`:                 `"{float(1,2,2)}"`,
+		`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
+		`"{digits(+5)}"`:                    `"{digits(5)}"`,
+		`"{digits(05)}"`:                    `"{digits(5)}"`,
+		`"{int(007,9)}"`:                    `"{int(7,9)}"`,
+		`"{int(+1,5)}"`:                     `"{int(1,5)}"`,
+	} {
+		sameRenders(t, long, short)
 	}
 }

calc.go

@@ -2,392 +2,149 @@ package fejkdata
 
 import (
 	"fmt"
-	"math"
+	"slices"
 	"strconv"
 	"strings"
-	"unicode"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// calcNode is a parsed expression node. It evaluates over the operand values expand
+// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
-// read, so the evaluator touches neither the rng nor the node tree.
+// field is a name, which checkCalcNames checks once names resolve.
-type calcNode interface {
+func checkCalcFields(args []string, fields map[string]node) error {
-	eval(operands []string) float64
+	return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
-}
+		if n, ok := fields[name]; ok {
-
+			return []node{n}
-type calcNum float64 // a number literal
-type calcVar string  // a sibling-field name, before indexVars places it
-type calcIdx int     // an operand, by its position in the values expand read
-type calcNeg struct{ x calcNode }
-type calcBin struct { // a + - * / b
-	op   byte
-	l, r calcNode
-}
-
-func (n calcNum) eval([]string) float64 { return float64(n) }
-
-func (n calcIdx) eval(operands []string) float64 {
-	v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
-	if err != nil {
-		return math.NaN() // a non-numeric operand stays visible, never an error
-	}
-	return v
-}
-
-// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
-// expression it compiles, so only a calcIdx reaches a render. It panics rather
-// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
-// silently wrong number.
-func (n calcVar) eval([]string) float64 {
-	panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
-}
-
-func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
-
-func (n calcBin) eval(operands []string) float64 {
-	l, r := n.l.eval(operands), n.r.eval(operands)
-	switch n.op {
-	case '+':
-		return l + r
-	case '-':
-		return l - r
-	case '*':
-		return l * r
-	default: // '/'
-		return l / r
-	}
-}
-
-// checkCalc validates a calc token at compile time: a parseable expression whose
-// operands all name existing fields, and an optional non-negative integer dp. It
-// is a builtin check (fields first), so calc dispatches through the registry like
-// every other {name(args)} function.
-func checkCalc(fields map[string]node, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return fmt.Errorf("calc(%q): %w", args[0], err)
-	}
-	for _, name := range calcVars(expr) {
-		operand, ok := fields[name]
-		if !ok {
-			return fmt.Errorf("calc(%q): no field %q", args[0], name)
 		}
-		if text, never := neverNumeric(operand); never {
+		return nil
-			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
+	})
+}
+
+// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
+func checkCalcNames(label string, t *template) error {
+	for _, o := range t.compiled.ops {
+		if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
+			continue
+		}
+		if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
+			return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
 		}
 	}
-	if divisor, zero := constantZeroDivisor(expr, fields); zero {
+	return nil
-		return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
+}
+
+// operandNodes lists every node each operand of o may render, once its template and names resolve.
+func operandNodes(o op) func(name string) []node {
+	return func(name string) []node {
+		a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
+		if len(a.leaves) == 0 {
+			panic(invariant.Broken("calc operand %q was read before it was compiled", name))
+		}
+		return a.leaves
 	}
-	if len(args) == 2 {
+}
-		dp, err := plainInt(args[1])
+
-		if err != nil {
+// checkOperands refuses an operand that is never a number and a division by a constant zero.
-			return fmt.Errorf("calc decimals %w", err)
+// operand lists every node an operand may render, nil while that is unknown.
-		}
+func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
-		if dp < 0 || dp > maxDecimals {
+	for _, name := range c.Operands {
-			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
+		if rendered, never := allNeverNumeric(operand(name)); never {
+			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
 		}
 	}
+	if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
+		return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
+	}
 	return nil
 }
 
 // constantZeroDivisor finds a division whose right side is a constant zero: number
 // literals and fixed operands folded, anything that varies left unknown.
-func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
+func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
 	switch n := n.(type) {
-	case calcNeg:
+	case grammar.CalcNeg:
-		return constantZeroDivisor(n.x, fields)
+		return constantZeroDivisor(n.X, operand)
-	case calcBin:
+	case grammar.CalcBin:
-		if n.op == '/' {
+		if n.Operator == '/' {
-			if v, known := constantValue(n.r, fields); known && v == 0 {
+			if v, known := constantValue(n.R, operand); known && v == 0 {
-				return calcText(n.r), true
+				return grammar.CalcText(n.R), true
 			}
 		}
-		if d, zero := constantZeroDivisor(n.l, fields); zero {
+		if d, zero := constantZeroDivisor(n.L, operand); zero {
 			return d, true
 		}
-		return constantZeroDivisor(n.r, fields)
+		return constantZeroDivisor(n.R, operand)
 	}
 	return "", false
 }
 
 // constantValue evaluates an expression whose every operand is fixed.
-func constantValue(n calcNode, fields map[string]node) (float64, bool) {
+func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
 	switch n := n.(type) {
-	case calcNum:
+	case grammar.CalcNum:
 		return float64(n), true
-	case calcVar:
+	case grammar.CalcVar:
-		t, ok := fields[string(n)].(*template)
+		nodes := operand(n.Name)
-		if !ok || !t.fixed || t.repeat > 1 {
+		if len(nodes) != 1 {
 			return 0, false
 		}
-		v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
+		t, ok := nodes[0].(*template)
+		if !ok || t.repeat != 1 {
+			return 0, false
+		}
+		lit, fixed := t.fixedText()
+		if !fixed {
+			return 0, false
+		}
+		v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
 		return v, err == nil
-	case calcNeg:
+	case grammar.CalcNeg:
-		v, ok := constantValue(n.x, fields)
+		v, ok := constantValue(n.X, operand)
 		return -v, ok
-	case calcBin:
+	case grammar.CalcBin:
-		l, lok := constantValue(n.l, fields)
+		l, lok := constantValue(n.L, operand)
-		r, rok := constantValue(n.r, fields)
+		r, rok := constantValue(n.R, operand)
 		if !lok || !rok {
 			return 0, false
 		}
-		return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
+		return builtinfunc.Arith(n.Operator, l, r), true
 	}
 	return 0, false
 }
 
-// calcText spells an expression node the way an author would read it.
+// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
-func calcText(n calcNode) string {
+// text that does not parse, or a choice of only such items. text is one such render.
-	switch n := n.(type) {
-	case calcNum:
-		return strconv.FormatFloat(float64(n), 'f', -1, 64)
-	case calcVar:
-		return string(n)
-	case calcNeg:
-		return "-" + calcText(n.x)
-	case calcBin:
-		return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
-	}
-	return "?"
-}
-
-// neverNumeric reports a node no render of which is a number: a null, fixed text that
-// does not parse, or a choice of only such items. text is one such render.
 func neverNumeric(n node) (text string, never bool) {
 	switch n := n.(type) {
-	case *null:
+	case *nullItem:
 		return "", true
 	case *template:
-		if !n.fixed || n.repeat > 1 {
+		lit, fixed := n.fixedText()
+		if n.repeat == 0 {
+			return "", true
+		}
+		if !fixed || n.repeat != 1 {
 			return "", false
 		}
-		if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
+		if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
-			return n.lit, true
+			return lit, true
 		}
 	case *choice:
-		for _, it := range n.items {
+		return allNeverNumeric(n.items)
-			t, itemNever := neverNumeric(it)
-			if !itemNever {
-				return "", false
-			}
-			text = t
-		}
-		return text, true
 	}
 	return "", false
 }
 
-// calcPrep parses the expression and decimals once, at compile time, and places each
+// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
-// operand name at the position expand will read it into. checkCalc proved both args
+func allNeverNumeric(nodes []node) (text string, never bool) {
-// valid, so no step here can fail; dp -1 prints the minimal form.
+	for _, n := range nodes {
-func calcPrep(args []string) callFn {
+		t, nodeNever := neverNumeric(n)
-	expr, err := parseCalc(args[0])
+		if !nodeNever {
-	if err != nil { // a nil AST would be a nil dereference per render, with no message
+			return "", false
-		panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
-	}
-	at := make(map[string]int)
-	for i, name := range calcVars(expr) {
-		at[name] = i
-	}
-	placed := indexVars(expr, at)
-	dp := calcDecimals(args)
-	return func(_ *session, _ string, operands []string) string {
-		return formatFloat(placed.eval(operands), dp)
-	}
-}
-
-// calcDecimals is a calc's decimals count, or -1 for the shortest form.
-func calcDecimals(args []string) int {
-	if len(args) == 2 {
-		return atoi(args[1])
-	}
-	return -1
-}
-
-// indexVars replaces each operand name with its position in the values expand reads.
-// Both sides take that order from calcVars, so they cannot drift.
-func indexVars(n calcNode, at map[string]int) calcNode {
-	switch n := n.(type) {
-	case calcVar:
-		i, placed := at[string(n)]
-		if !placed { // calcVars named every operand, so a miss means the two disagree
-			panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
 		}
-		return calcIdx(i)
+		text = t
-	case calcNeg:
-		return calcNeg{indexVars(n.x, at)}
-	case calcBin:
-		return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
 	}
-	return n
+	return text, len(nodes) > 0
-}
-
-// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
-// an expression that does not parse, so one that does not simply names nothing.
-func calcOperands(args []string) []string {
-	if len(args) == 0 {
-		return nil
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return nil
-	}
-	return calcVars(expr)
-}
-
-// calcVars lists the distinct field names an expression reads, in the order it first
-// names each. That order is the contract between expand, which reads the operands
-// into a slice, and indexVars, which places each name at its position in it.
-func calcVars(n calcNode) []string {
-	var out []string
-	seen := map[string]bool{}
-	var walk func(calcNode)
-	walk = func(n calcNode) {
-		switch n := n.(type) {
-		case calcVar:
-			if name := string(n); !seen[name] {
-				seen[name] = true
-				out = append(out, name)
-			}
-		case calcNeg:
-			walk(n.x)
-		case calcBin:
-			walk(n.l)
-			walk(n.r)
-		}
-	}
-	walk(n)
-	return out
-}
-
-// calcParser is a recursive-descent parser over the expression runes, threading
-// expr -> term -> factor for the standard * / before + - precedence.
-type calcParser struct {
-	rs  []rune
-	pos int
-}
-
-// parseCalc parses a whole expression, requiring it to consume all input.
-func parseCalc(expr string) (calcNode, error) {
-	p := &calcParser{rs: []rune(expr)}
-	if p.space(); p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("empty expression")
-	}
-	n, err := p.expr()
-	if err != nil {
-		return nil, err
-	}
-	if p.space(); p.pos != len(p.rs) {
-		return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
-	}
-	return n, nil
-}
-
-func (p *calcParser) space() {
-	for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
-		p.pos++
-	}
-}
-
-func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
-func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
-
-// binary parses a left-associative run of next() operands joined by the given
-// operators, the one shape expr and term share.
-func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
-	n, err := next()
-	if err != nil {
-		return nil, err
-	}
-	for {
-		p.space()
-		if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
-			return n, nil
-		}
-		op := byte(p.rs[p.pos])
-		p.pos++
-		r, err := next()
-		if err != nil {
-			return nil, err
-		}
-		n = calcBin{op, n, r}
-	}
-}
-
-// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
-func (p *calcParser) factor() (calcNode, error) {
-	p.space()
-	if p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("unexpected end of expression")
-	}
-	switch c := p.rs[p.pos]; {
-	case c == '-':
-		p.pos++
-		x, err := p.factor()
-		if err != nil {
-			return nil, err
-		}
-		return calcNeg{x}, nil
-	case c == '(':
-		p.pos++
-		n, err := p.expr()
-		if err != nil {
-			return nil, err
-		}
-		if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
-			return nil, fmt.Errorf("missing ')'")
-		}
-		p.pos++
-		return n, nil
-	case c == '.' || c >= '0' && c <= '9':
-		return p.number()
-	case c == '_' || unicode.IsLetter(c):
-		return p.ident()
-	default:
-		return nil, fmt.Errorf("unexpected %q", string(c))
-	}
-}
-
-func (p *calcParser) number() (calcNode, error) {
-	start, dot := p.pos, false
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c >= '0' && c <= '9' {
-			p.pos++
-		} else if c == '.' && !dot {
-			dot, p.pos = true, p.pos+1
-		} else {
-			break
-		}
-	}
-	v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
-	if err != nil {
-		return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
-	}
-	return calcNum(v), nil
-}
-
-// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
-// is always the minus operator, so a hyphenated field name can't be an operand.
-func (p *calcParser) ident() (calcNode, error) {
-	start := p.pos
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
-			p.pos++
-		} else {
-			break
-		}
-	}
-	return calcVar(string(p.rs[start:p.pos])), nil
-}
-
-func contains(bs []byte, b byte) bool {
-	for _, x := range bs {
-		if x == b {
-			return true
-		}
-	}
-	return false
 }

calc_test.go

@@ -3,8 +3,11 @@ package fejkdata
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
 )
 
 // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
 	if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
 		t.Fatalf("calc(a - b) = %q, want 7", got)
 	}
+	if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
+		t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
+	}
 }
 
 // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
 	}
 }
 
-// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
+// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
-// is not a calc, and for one whose expression does not parse: nothing, either way.
+// call that is not a calc, and for one whose expression does not parse: none, either way.
-// checkCalc is what reports a bad expression, so the callers that run after it
+func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
-// never meet one — but they must not have to depend on that order to be safe.
+	for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
-func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
+		if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
-	for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
+			t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
-		if got := tokenOperands(body); got != nil {
-			t.Errorf("tokenOperands(%q) = %v, want none", body, got)
 		}
 	}
-	if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
+	if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
-		t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
+		t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
-	}
-}
-
-// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
-// the expression before prep sees it, and indexVars places every name calcVars
-// read, so a break in either is reported rather than computed around.
-func TestCalcGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
-		"an operand name never placed":              func() { calcVar("n").eval(nil) },
-		"indexVars on a name it did not read":       func() { indexVars(calcVar("n"), map[string]int{}) },
-	} {
-		mustPanic(t, name, call)
 	}
 }
 
 // --- one draw, one value: a calc operand reads the expansion's draw ---
 
-// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
+// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
-// A field the format renders and a calc reads is drawn once per expansion, so the
+// shows: the format and the calc read one name.
-// operand shown is the operand computed — the correlation the dotted-path rule
+func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
-// already gives a level (see bound_test.go), applied to a plain sibling.
-func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(3))
 	for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	}
 }
 
-// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
+// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
-// expansion, not to the calc, so the bare token rendering the same name reads it too.
+func TestACalcOperandDrawsAfresh(t *testing.T) {
-func TestCalcOperandSharesOneDraw(t *testing.T) {
+	dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
-	dir := writeData(t, map[string]string{
-		"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
-	})
 	f := newGenerator(t, dir, WithSeed(5))
 	for i := 0; i < 200; i++ {
-		got := fake(t, f, "same")
+		if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
-		if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
-			t.Fatalf("same = %q, want one value twice", got)
-		}
-	}
-}
-
-// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
-// held, so an ordinary {w} {w} still draws twice.
-func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
-	dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
-	f := newGenerator(t, dir, WithSeed(5))
-	for i := 0; i < 200; i++ {
-		if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
 			return
 		}
 	}
-	t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
+	t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
 }
 
-// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
+// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
-// its own expansion, so it draws again while staying self-consistent.
+// iteration, staying one value within it.
-func TestCalcHoldIsPerExpansion(t *testing.T) {
+func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
+		"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(11))
 	varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
 	}
 }
 
-// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
+// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
-// {v} and its calc agree with each other, not with the outer pair.
+// pair agrees with itself, not with the outer pair.
-func TestCalcHoldIsPerTemplate(t *testing.T) {
+func TestTwoNamesPickApart(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
+		"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
-			"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
+			"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
 	})
 	f := newGenerator(t, dir, WithSeed(13))
 	differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["a","b"]}`:       `"x"`,
 		`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
 	} {
-		_, err := compile(parse(t, src))
+		_, err := resolved(t, src)
 		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
 			t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
 		}
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
 		`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
 	} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
 		}
 	}
@@ -250,14 +223,14 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 
 func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	for src, want := range map[string]string{
-		`"{calc(1/0)}"`:                                  "divides by 0",
+		`"{calc(1/0)}"`:     "divides by 0",
-		`"{calc(2/(1-1))}"`:                              "divides by (1 - 1)",
+		`"{calc(2/(1-1))}"`: "divides by (1 - 1)",
 		`{"format":"{calc(x/y)}","x":"1","y":"0"}`:       "divides by y",
 		`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
 		`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`:   "divides by 0",
 		`{"format":"{calc(a/-0)}","a":"1"}`:              "divides by -0",
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
 		}
 	}
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	}
 	t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
 }
+
+func TestCalcCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{calc(1/0)}"`, // a constant zero divisor
+		`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
+		`"{calc()}"`,        // calc needs an expression
+		`"{calc(1 +)}"`,     // dangling operator
+		`"{calc((1 + 2)}"`,  // unbalanced parenthesis
+		`"{calc(1 2)}"`,     // two operands, no operator
+		`"{calc(price)}"`,   // operand names no field
+		`"{calc(1, 2, 3)}"`, // too many args
+		`"{calc(1, x)}"`,    // decimals arg not an integer
+		`"{calc(1, -1)}"`,   // decimals negative
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestCalcReadsAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3","5","7"]`,
+		"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	for i := 0; i < 50; i++ {
+		var k, double int
+		got := fake(t, f, "card")
+		if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
+			t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
+		}
+	}
+}
+
+func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3"]`,
+		"card": `"{/n as k}{calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	if got := fake(t, f, "card"); got != "4" && got != "6" {
+		t.Fatalf("card = %q, want 4 or 6", got)
+	}
+}
+
+func TestCalcRefusesABadOperand(t *testing.T) {
+	for card, want := range map[string]string{
+		`"{/word as w}{w}{calc(w * 2)}"`:    `operand "w" is never a number: it renders "def"`,
+		`"{/zero as z}{z}{calc(1 / z)}"`:    `divides by z, which is always zero`,
+		`"{calc(nope * 2)}"`:                `no field "nope"`,
+		`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
+	} {
+		dir := writeData(t, map[string]string{
+			"word": `["abc","def"]`,
+			"zero": `"0"`,
+			"card": card,
+		})
+		if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
+		}
+	}
+}
+
+func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
+		"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
+	})))
+	if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
+		t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
+	}
+}
+
+func TestATypedColumnProvesACalcOverAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":     `"{int(1,9)}"`,
+		"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	r, err := f.FakeRecord("order")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
+		t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
+	}
+	if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
+		t.Fatalf("JSON = %s, want double written as a number", r.JSON())
+	}
+	bad := writeData(t, map[string]string{
+		"word":  `["1","x"]`,
+		"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
+	})
+	if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
+		t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
+	}
+}
+
+func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
+	if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
+		t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
+	}
+}

cli/cli.go

@@ -0,0 +1,706 @@
+// Package cli is the fejkdata command: it renders the template on its stdin with fake
+// values from the modules it is given and any directories layered over them. A program
+// passing its own modules to Main is a fejkdata CLI carrying them, for example
+// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
+package cli
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata"
+)
+
+const usage = `Usage: <template> | COMMAND [flags]
+       COMMAND --list | --help | --version
+
+COMMAND reads one template from stdin and renders it:
+
+  echo 'id: {int(1,999)}' | COMMAND
+  echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
+  COMMAND < template.txt
+
+COMMAND <<'EOF'
+born {date(1990-01-01,2010-12-31,'2006-01-02')}
+EOF
+
+A template is a format string, or a JSON object, array, string or null. Its {…} tokens
+reach the data by reference from the root: {/path}, for any path --list prints, shipped
+or layered with --data-path. A reference names a category or a dotted path into one, a
+table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
+links to.
+
+Under --format text, the default, what you send is what renders, and COMMAND adds
+nothing: a format string keeps every byte, so echo's newline ends each render, and
+printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
+object, array or string drops the whitespace around it, and null alone renders nothing;
+42 or true alone is a format string, printed as written. An empty or blank stdin is a
+format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
+a backslash or a quote as written.
+
+With --format json, ndjson, csv or sql the template must be a record — a JSON
+template whose fields are its columns, or a lone reference such as {/path}:
+one reference and nothing else, with one newline after it allowed — and the rows are
+written as one JSON array, one JSON object per line, one CSV row (after a header), or
+one INSERT.
+
+  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
+      --format F         output form: text (default), json, ndjson, csv or sql
+  -h, --help             print this help, then exit
+      --list             list the paths the data offers, each written after {/ in a template, then exit;
+                         --repeat, --separator, --format and --table do nothing here
+      --no-shipped-data  load only the --data-path directories
+  -n, --repeat N         render the template N times, 0..1048576 (default 1); 0 prints nothing,
+                         [] under --format json, or the header under --format csv
+  -s, --seed N           same seed, version and data: identical output (default: a random seed)
+      --separator S      string between repeated renders (default empty); refused with a record
+                         --format, which has its own separators
+      --table T          the INSERT target for --format sql, ignored under any other
+                         (default: a lone reference's last segment, else records)
+      --version          print the version, then exit
+
+A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
+-- ends the flags.
+
+Exit codes: 0 success, 1 runtime error, 2 misuse.
+`
+
+type invocation struct {
+	args         []string
+	dirs         []string
+	format       string
+	help         bool
+	list         bool
+	noShipped    bool
+	repeat       int
+	seed         uint64
+	seeded       bool
+	separator    string
+	separatorSet bool
+	table        string
+	tableSet     bool
+	version      bool
+}
+
+type flagDef struct {
+	long  string
+	short string
+	value bool
+	set   func(*invocation, string) error
+}
+
+var flagDefs = []flagDef{
+	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
+	{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
+	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
+	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
+	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
+	{"repeat", "n", true, func(in *invocation, v string) error {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n > fejkdata.MaxRepeat {
+			return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
+		}
+		in.repeat = n
+		return nil
+	}},
+	{"seed", "s", true, func(in *invocation, v string) error {
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
+		}
+		in.seed, in.seeded = n, true
+		return nil
+	}},
+	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
+	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
+	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
+}
+
+func flagByLong(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].long == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+func flagByShort(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].short == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+// flagArg is one flag as written: its definition and the value attached to it,
+// if any.
+type flagArg struct {
+	def    *flagDef
+	value  string
+	inline bool
+}
+
+// splitFlags reads one argv element as flags: --name or --name=value, or -abc
+// where each letter is a short flag and the first that takes a value takes the
+// rest of the element.
+func splitFlags(arg string) ([]flagArg, error) {
+	if strings.HasPrefix(arg, "--") {
+		name, value, inline := strings.Cut(arg[2:], "=")
+		def := flagByLong(name)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag --%s", name)
+		}
+		return []flagArg{{def, value, inline}}, nil
+	}
+	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
+		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
+	}
+	var flags []flagArg
+	letters := arg[1:]
+	for i, r := range letters {
+		letter := string(r)
+		def := flagByShort(letter)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag -%s", letter)
+		}
+		if !def.value {
+			flags = append(flags, flagArg{def: def})
+			continue
+		}
+		rest := letters[i+len(letter):]
+		if strings.HasPrefix(rest, "=") {
+			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
+				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
+		}
+		return append(flags, flagArg{def, rest, rest != ""}), nil
+	}
+	return flags, nil
+}
+
+func parseArgs(argv []string) (invocation, error) {
+	in := invocation{repeat: 1, format: "text"}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		if arg == "--" {
+			in.args = append(in.args, argv[i+1:]...)
+			return in, nil
+		}
+		if len(arg) < 2 || arg[0] != '-' {
+			in.args = append(in.args, arg)
+			continue
+		}
+		flags, err := splitFlags(arg)
+		if err != nil {
+			return in, err
+		}
+		for _, fl := range flags {
+			if !fl.def.value {
+				if fl.inline {
+					return in, fmt.Errorf("--%s takes no value", fl.def.long)
+				}
+				_ = fl.def.set(&in, "")
+				continue
+			}
+			value := fl.value
+			if !fl.inline {
+				if i++; i >= len(argv) {
+					return in, fmt.Errorf("--%s needs a value", fl.def.long)
+				}
+				value = argv[i]
+			}
+			if err := fl.def.set(&in, value); err != nil {
+				return in, err
+			}
+		}
+	}
+	return in, nil
+}
+
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
+type recordFormat struct {
+	header func(*fejkdata.Record) string
+	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
+	writes string // what it writes between values, which --separator would seem to set
+}
+
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson writes one object per line.
+var recordFormats = map[string]recordFormat{
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
+	"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
+}
+
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
+func (in invocation) writesRecords() bool {
+	return in.format != "text"
+}
+
+// formatNames lists the --format values, text included, for the misuse error.
+func formatNames() string {
+	names := []string{"text"}
+	for name := range recordFormats {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
+}
+
+// checkFlags rejects a flag value or combination that cannot run.
+func (in invocation) checkFlags() error {
+	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
+		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
+	}
+	if in.tableSet && in.table == "" && in.format == "sql" {
+		return errors.New("--table names the INSERT target, so it cannot be empty")
+	}
+	if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
+		return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
+	}
+	return nil
+}
+
+func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
+
+// operandError refuses an argument that is no flag, naming the spelling that reads what
+// it most likely means: dropping it for -, a file for a file's name, else a template
+// piped in.
+func operandError(arg, name string) error {
+	unexpected := "unexpected argument '" + arg + "'"
+	if arg == "-" {
+		return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
+	}
+	if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
+		return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
+	}
+	ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
+	_, argLone := loneReference(arg)
+	inline, err := fejkdata.IsTemplate(arg)
+	_, refLone := loneReference(ref)
+	example := pipeHint(name)
+	switch {
+	case argLone || inline:
+		example = piped(arg, name)
+	case err == nil && refLone:
+		example = piped(ref, name)
+	}
+	return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
+}
+
+// check rejects a flag combination or an operand that cannot run, before any data is
+// read.
+func (in invocation) check(name string) error {
+	if err := in.checkFlags(); err != nil {
+		return err
+	}
+	if len(in.args) > 0 {
+		return operandError(in.args[0], name)
+	}
+	return nil
+}
+
+func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
+	if in.noShipped {
+		modules = nil
+	}
+	opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
+	for _, dir := range in.dirs {
+		opts = append(opts, fejkdata.WithDataPath(dir))
+	}
+	seed := in.seed
+	if !in.seeded {
+		seed = systemSeed
+	}
+	return append(opts, fejkdata.WithSeed(seed))
+}
+
+// write streams the input's renders to w, repeat of them joined by the separator; a write
+// failure surfaces from Flush, bufio keeping the first one.
+func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
+	if in.writesRecords() {
+		return in.writeRecords(f, src, w)
+	}
+	t, err := f.NewTemplate(src.template)
+	if err != nil {
+		return templateError(err)
+	}
+	out := bufio.NewWriter(w)
+	for i := 0; i < in.repeat; i++ {
+		if i > 0 {
+			out.WriteString(in.separator)
+		}
+		out.WriteString(t.Fake())
+	}
+	return out.Flush()
+}
+
+// recordStream builds the record drawer for the input, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
+	record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
+	if src.record == "" {
+		t, err := f.NewRecordTemplate(src.template)
+		if err != nil {
+			return nil, "", templateError(spacedReference(src.template, err))
+		}
+		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
+	}
+	table := in.table
+	if table == "" {
+		table = defaultTable(src.record)
+	}
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
+	record, table, err := in.recordStream(f, src)
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	if in.repeat == 0 {
+		return writeNoRecord(record, format, w)
+	}
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
+		r, err := record()
+		if err != nil {
+			return err
+		}
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
+		}
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
+}
+
+// writeNoRecord writes zero records: the header where the format writes one, an empty document
+// where it frames one, else nothing. It draws one record, which names the header's columns, so a
+// lone reference to nothing fails as it would with records.
+func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
+	r, err := record()
+	switch {
+	case err != nil:
+		return err
+	case format.header != nil:
+		_, err = io.WriteString(w, format.header(r)+"\n")
+	case format.open != "":
+		_, err = io.WriteString(w, format.open+format.close+"\n")
+	}
+	return err
+}
+
+// defaultTable names the INSERT target when --table is absent: the path's last
+// segment, or "records" when stdin is not a lone reference.
+func defaultTable(path string) string {
+	if path == "" {
+		return "records"
+	}
+	segments := strings.Split(withoutSelectors(path), ".")
+	return segments[len(segments)-1]
+}
+
+// withoutSelectors is path with its [selectors] cut out.
+func withoutSelectors(path string) string {
+	var names strings.Builder
+	for depth, i := 0, 0; i < len(path); i++ {
+		switch {
+		case path[i] == '[':
+			depth++
+		case path[i] == ']':
+			depth--
+		case depth == 0:
+			names.WriteByte(path[i])
+		}
+	}
+	return names.String()
+}
+
+// inputError marks a failure that is the input's own fault — nothing piped in, a template
+// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
+// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
+// error (exit 1).
+type inputError struct{ error }
+
+func (e inputError) Unwrap() error { return e.error }
+
+// jsonSpace is the whitespace JSON allows around a value.
+const jsonSpace = " \t\r\n"
+
+// input is what stdin holds: the template, and the path a lone reference names as a
+// record.
+type input struct {
+	template string
+	record   string
+}
+
+// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
+// reference once JSON's surrounding whitespace, or one final newline, is cut.
+func parseInput(raw string) input {
+	trimmed := strings.Trim(raw, jsonSpace)
+	text := raw
+	if json.Valid([]byte(trimmed)) {
+		text = trimmed
+	}
+	text, found := strings.CutSuffix(text, "\n")
+	if found {
+		text = strings.TrimSuffix(text, "\r")
+	}
+	record, _ := loneReference(text)
+	return input{template: raw, record: record}
+}
+
+// loneReference returns the path that text reads when it is one reference from the root
+// and nothing else, bare or as a JSON string.
+func loneReference(text string) (string, bool) {
+	var unquoted string
+	if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
+		text = unquoted
+	}
+	path, opens := strings.CutPrefix(text, "{/")
+	path, closes := strings.CutSuffix(path, "}")
+	if !opens || !closes || !isPath(path) {
+		return "", false
+	}
+	return path, true
+}
+
+// isPath reports whether a reference's body after its / is a path alone: no second /, no
+// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
+// It copies the grammar, since the CLI reads only the library's public API.
+func isPath(body string) bool {
+	if body == "" || strings.ContainsAny(body[:1], "/.") {
+		return false
+	}
+	names := withoutSelectors(body)
+	if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
+		return false
+	}
+	inline, err := fejkdata.IsTemplate(body)
+	return !inline && err == nil
+}
+
+// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
+// step up, and only a step up may end the path.
+func hasEmptySegment(path string) bool {
+	segments := strings.Split(path, ".")
+	stepUp := false
+	for i, seg := range segments[1:] {
+		last := i == len(segments)-2
+		switch {
+		case seg != "":
+			stepUp = false
+		case stepUp != last:
+			return true
+		default:
+			stepUp = !stepUp
+		}
+	}
+	return false
+}
+
+// spacedReference adds to a record refusal that whitespace around a lone reference made
+// the template text.
+func spacedReference(template string, err error) error {
+	if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
+		if path, lone := loneReference(trimmed); lone {
+			return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
+		}
+	}
+	return err
+}
+
+// piped is the command piping template into the command called name; some shells' echo
+// reads a backslash as an escape, so a template holding one goes through printf.
+func piped(template, name string) string {
+	if strings.ContainsRune(template, '\\') {
+		return "printf '%s' " + shellQuoted(template) + " | " + name
+	}
+	return "echo " + shellQuoted(template) + " | " + name
+}
+
+// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
+func shellQuoted(s string) string {
+	if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
+		return s
+	}
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// Main runs the command on the process's arguments and standard streams over modules, each
+// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
+// 2 misuse. --no-shipped-data leaves modules out.
+func Main(modules ...fs.FS) int {
+	var stdin io.Reader = os.Stdin
+	if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+		stdin = nil
+	}
+	return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
+}
+
+// commandName is the name the command runs under: the file arg0 names, less a Windows
+// .exe, or fejkdata where arg0 names none.
+func commandName(arg0 string) string {
+	name := filepath.Base(arg0)
+	if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
+		name = strings.TrimSuffix(name, ext)
+	}
+	if name == "." || name == "" {
+		return "fejkdata"
+	}
+	return name
+}
+
+// run is Main over the given streams, run as the command name, seeded by systemSeed where
+// no --seed is given. A nil stdin is a terminal, which holds no template.
+func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
+	in, err := parseArgs(args)
+	if err != nil {
+		return misuse(stderr, name, err)
+	}
+	if in.help {
+		io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
+		return 0
+	}
+	if in.version {
+		info, _ := debug.ReadBuildInfo()
+		fmt.Fprintln(stdout, versionLine(name, info))
+		return 0
+	}
+	if err := in.check(name); err != nil {
+		return misuse(stderr, name, err)
+	}
+	var src input
+	if !in.list {
+		if src, err = readStdin(stdin, name); err != nil {
+			return fail(stderr, name, err)
+		}
+	}
+	f, err := fejkdata.New(in.options(modules, systemSeed)...)
+	if err != nil {
+		return fail(stderr, name, err)
+	}
+	if in.list {
+		for _, p := range f.List() {
+			fmt.Fprintln(stdout, p)
+		}
+		return 0
+	}
+	if err := in.write(f, src, stdout); err != nil {
+		return fail(stderr, name, err)
+	}
+	return 0
+}
+
+func readStdin(stdin io.Reader, name string) (input, error) {
+	if stdin == nil {
+		return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
+	}
+	return parseInput(string(raw)), nil
+}
+
+// templateError is misuse unless the data the template reads failed to load.
+func templateError(err error) error {
+	if errors.Is(err, fejkdata.ErrLoad) {
+		return err
+	}
+	return inputError{err}
+}
+
+func fail(stderr io.Writer, name string, err error) int {
+	var te inputError
+	if errors.As(err, &te) {
+		return misuse(stderr, name, te.error)
+	}
+	report(stderr, name, err)
+	if errors.Is(err, fejkdata.ErrNoColumns) {
+		fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
+	}
+	return 1
+}
+
+func misuse(stderr io.Writer, name string, err error) int {
+	report(stderr, name, err)
+	fmt.Fprintf(stderr, "try '%s --help'\n", name)
+	return 2
+}
+
+// report writes err under the command name, in place of the library's own on each line.
+func report(stderr io.Writer, name string, err error) {
+	msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
+	fmt.Fprintf(stderr, "%s: %s\n", name, msg)
+}
+
+// fejkdataModule is the path of the module holding the library; the root package's path is
+// the module's.
+var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
+
+// versionLine names the fejkdata version the binary is built with. Where another module's
+// main package carries the CLI, the main module's version comes first, under the command name.
+func versionLine(name string, info *debug.BuildInfo) string {
+	if info == nil {
+		return "fejkdata devel"
+	}
+	if info.Main.Path == fejkdataModule {
+		return "fejkdata " + moduleVersion(info.Main)
+	}
+	for _, m := range info.Deps {
+		if m.Path == fejkdataModule {
+			return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
+		}
+	}
+	return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
+}
+
+// moduleVersion is the version m is built at: the replacement's where a replace directive
+// names one, and "devel" where it carries none, as a local directory does.
+func moduleVersion(m debug.Module) string {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Version == "" || m.Version == "(devel)" {
+		return "devel"
+	}
+	return m.Version
+}

cli/cli_test.go

@@ -0,0 +1,961 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// shipped holds the shipped modules as read from the checkout, each folder under ../data
+// holding a manifest, so these tests link no data package.
+var shipped = func() []fs.FS {
+	var modules []fs.FS
+	err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
+		if err == nil && d.Name() == ".fejkdata.json" {
+			modules = append(modules, os.DirFS(filepath.Dir(path)))
+		}
+		return err
+	})
+	if err != nil || len(modules) == 0 {
+		panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
+	}
+	return modules
+}()
+
+// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
+// layers its categories unprefixed with -d.
+var svSE, enUS, misc string
+
+func TestMain(m *testing.M) {
+	tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
+	if err != nil {
+		panic(err)
+	}
+	svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
+	code := m.Run()
+	os.RemoveAll(tmp)
+	os.Exit(code)
+}
+
+// bareCopy copies the category and rows files of the shipped module in data/<module> into a
+// folder under tmp, and returns that folder.
+func bareCopy(tmp, module string) string {
+	src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
+	if err := os.Mkdir(dst, 0o755); err != nil {
+		panic(err)
+	}
+	for _, pattern := range []string{"*.json", "*.tsv"} {
+		files, err := filepath.Glob(filepath.Join(src, pattern))
+		if err != nil {
+			panic(err)
+		}
+		for _, f := range files {
+			if strings.HasPrefix(filepath.Base(f), ".") {
+				continue
+			}
+			b, err := os.ReadFile(f)
+			if err == nil {
+				err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
+			}
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	return dst
+}
+
+func runOut(stdin string, args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+// runTerminal runs as from a shell with nothing piped in.
+func runTerminal(args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
+	module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
+		t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
+	}
+	out.Reset()
+	if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
+		t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
+	}
+}
+
+func TestRunOutputsValue(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if out == "" || strings.Contains(out, "\n") {
+		t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
+	}
+}
+
+func TestRunDotPath(t *testing.T) {
+	code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person run = %d", code)
+	}
+	code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(last) == "" {
+		t.Fatal("empty person.last output")
+	}
+	if last == full {
+		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
+}
+
+func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
+	var drawn, seeded, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
+		t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
+		t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if drawn.String() != seeded.String() {
+		t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
+	}
+}
+
+func TestRunSeedSpellings(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"--seed=42", "--data-path", svSE},
+		{"-s", "42", "-d", svSE},
+		{"--data-path", svSE, "--seed", "42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+}
+
+func TestRunShortFlagValues(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"-s42", "-d", svSE},
+		{"-s", "42", "-d" + svSE},
+		{"-d", svSE, "-s42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+	_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
+	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
+		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-s=42", "-d", svSE}, "-s42"},
+		{[]string{"-s=42", "-d", svSE}, "--seed=42"},
+		{[]string{"-d=" + svSE}, "--data-path="},
+		{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
+		{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
+	} {
+		code, out, errb := runOut("{/word}", c.args...)
+		if code != 2 || out != "" {
+			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+	code, out, _ := runOut("", "-hd", svSE)
+	if code != 0 || !strings.Contains(out, "Usage") {
+		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
+	}
+}
+
+func TestRunDoubleDashEndsFlags(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
+	if code != 2 || !strings.Contains(errb, "stdin") {
+		t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSingleDashLongIsRejected(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
+		{[]string{"-seed=42", "-d", svSE}, "use --seed"},
+		{[]string{"-data-path", svSE}, "use --data-path"},
+		{[]string{"-list", "-d", svSE}, "use --list"},
+		{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
+		{[]string{"-x", "-d", svSE}, "unknown flag -x"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunFlagValues(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
+		{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
+		{[]string{"--seed", "x", "-d", svSE}, "--seed"},
+		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
+		{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
+		{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunRepeat(t *testing.T) {
+	code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
+	}
+	_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
+	if short != out {
+		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
+	}
+}
+
+func TestRunSeparator(t *testing.T) {
+	code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if n := strings.Count(out, "\n"); n != 0 {
+		t.Errorf("want no newline, got %d: %q", n, out)
+	}
+	if !strings.Contains(out, ",") {
+		t.Errorf("values should be comma-joined: %q", out)
+	}
+}
+
+func TestRunRepeatAdvancesRNG(t *testing.T) {
+	code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	uniq := map[string]bool{}
+	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+		uniq[l] = true
+	}
+	if len(uniq) < 2 {
+		t.Errorf("repeat should vary output, all identical: %q", out)
+	}
+}
+
+func TestRunMisuse(t *testing.T) {
+	for _, c := range []struct {
+		stdin string
+		args  []string
+	}{
+		{"{/person}", []string{"-d", svSE, "word"}},
+		{"", []string{"-d", svSE, "--list", "person"}},
+		{"{/sv_SE.person}", []string{"--no-shipped-data"}},
+	} {
+		code, out, errb := runOut(c.stdin, c.args...)
+		if code != 2 {
+			t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
+		}
+		if !strings.Contains(errb, "try 'fejkdata --help'") {
+			t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
+		}
+		if out != "" {
+			t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
+		}
+	}
+}
+
+func TestRunTakesNoOperand(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "users.tmpl")
+	if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
+		{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
+		{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
+		{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
+		{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
+		{[]string{"-"}, "drop it"},
+		{[]string{file}, "fejkdata < " + file},
+	} {
+		code, out, errb := runOut("", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+type failingReader struct{}
+
+func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
+
+func TestRunUnreadableStdinIsMisuse(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
+		t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
+	}
+}
+
+func TestRunTerminalStdinIsMisuse(t *testing.T) {
+	code, out, errb := runTerminal("--seed", "1")
+	if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
+		t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
+	}
+	for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
+		if code, out, errb := runTerminal(args...); code != 0 || out == "" {
+			t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
+		}
+	}
+}
+
+func TestRunRendersExactlyWhatIsSent(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"hihi":                           "hihi",
+		"hihi\n":                         "hihi\n",
+		"hihi\r\n":                       "hihi\r\n",
+		"hihi\n\n":                       "hihi\n\n",
+		"{{x}}\n":                        "{x}\n",
+		"\"a\"\n":                        "a",
+		" \t\"a\"\r\n\n":                 "a",
+		"{/misc.territory[SE].alpha2}":   "SE",
+		"{/misc.territory[SE].alpha2}\n": "SE\n",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
+		}
+	}
+	if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
+		t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
+	}
+	if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
+		t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
+	}
+}
+
+func TestRunRendersBlankStdin(t *testing.T) {
+	for _, stdin := range []string{"", "\n", " \r\n\t"} {
+		if code, out, errb := runOut(stdin); code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunTextPrintsAsWritten(t *testing.T) {
+	for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
+		code, out, errb := runOut(stdin)
+		if code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunMultipleDataPaths(t *testing.T) {
+	code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(out) == "" {
+		t.Fatal("empty output for multi-dir run")
+	}
+}
+
+func TestRunUnknownCategoryFails(t *testing.T) {
+	code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if !strings.Contains(errb, "nope") {
+		t.Errorf("stderr %q should name the unknown category", errb)
+	}
+}
+
+func TestRunList(t *testing.T) {
+	code, out, errb := runOut("", "--data-path", svSE, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	for _, want := range []string{"person", "person.last", "address"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("list output missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRunHelpOnStdout(t *testing.T) {
+	for _, h := range []string{"-h", "--help"} {
+		code, out, errb := runOut("", h)
+		if code != 0 {
+			t.Errorf("%s = %d, want 0", h, code)
+		}
+		if !strings.Contains(out, "Usage") || errb != "" {
+			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
+		}
+	}
+}
+
+func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
+	fejkdata := func(version string, replace *debug.Module) *debug.Module {
+		return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
+	}
+	mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
+	for _, c := range []struct {
+		info *debug.BuildInfo
+		want string
+	}{
+		{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
+		{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
+		{nil, "fejkdata devel"},
+	} {
+		if got := versionLine("mycli", c.info); got != c.want {
+			t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
+		}
+	}
+}
+
+func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
+	for arg0, want := range map[string]string{
+		"/usr/local/bin/mycli": "mycli",
+		"mycli.EXE":            "mycli",
+		"mycli.exe":            "mycli",
+		"my.cli":               "my.cli",
+		"":                     "fejkdata",
+	} {
+		if got := commandName(arg0); got != want {
+			t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
+		}
+	}
+}
+
+func TestRunNamesItselfByTheCommandName(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
+		t.Errorf("mycli --nope = %d, %q", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
+		t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
+	}
+	if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
+		t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	code, out, errb := runOut("", "--version")
+	if code != 0 {
+		t.Fatalf("--version = %d, stderr=%q", code, errb)
+	}
+	version, ok := strings.CutPrefix(out, "fejkdata ")
+	if !ok || strings.TrimSpace(version) == "" {
+		t.Errorf("--version = %q, want the command name and a version on stdout", out)
+	}
+}
+
+func TestRunMissingDirFails(t *testing.T) {
+	code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if errb == "" {
+		t.Error("want an error message on stderr")
+	}
+}
+
+func TestRunShippedDataByDefault(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, list, _ := runOut("", "--list")
+	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
+		t.Errorf("--list without --data-path = %d, %q", code, list)
+	}
+	code, _, errb = runOut("{/person}")
+	if code != 2 || !strings.Contains(errb, "person") {
+		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSelectsATableRow(t *testing.T) {
+	dir := t.TempDir()
+	for name, content := range map[string]string{
+		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
+		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
+		t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
+	}
+	if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
+		t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
+	}
+	if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
+		t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
+	}
+	if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
+		t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
+	}
+	if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
+		t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
+	}
+}
+
+func TestUsageExamplesRenderWithNoData(t *testing.T) {
+	examples := regexp.MustCompile(`(?m)^  echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n  "+pipeHint("COMMAND")+"\n", -1)
+	if len(examples) < 3 {
+		t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
+	}
+	for _, e := range examples {
+		if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
+			t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
+		}
+	}
+	for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
+		if token != "{/path}" && token != "{/table[KEY]}" {
+			t.Errorf("usage names %s, a path a custom CLI may not carry", token)
+		}
+	}
+	if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
+		t.Errorf("usage does not list the exit codes")
+	}
+}
+
+func TestRunOperandMisuseBeforeLoad(t *testing.T) {
+	code, _, errb := runOut("", "--no-shipped-data", "x")
+	if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
+		t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
+	}
+}
+
+func TestRunInlineTemplate(t *testing.T) {
+	code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
+	if code != 0 || (out != "name: bosse" && out != "name: lina") {
+		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
+	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
+		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
+	if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
+		t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
+	}
+}
+
+func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
+	for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
+		if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
+			t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
+		}
+	}
+	if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
+		t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
+	}
+}
+
+func TestRunTemplateMisuse(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"{bad":                  "unterminated",
+		"name: {/no.such.path}": "no entry",
+		"a } b":                 "}}",
+		"{//sv_SE.person}":      "write {/sv_SE.person}",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
+			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
+		}
+		if strings.Contains(errb, "fejkdata: fejkdata:") {
+			t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
+		}
+	}
+}
+
+func TestRunNoShippedData(t *testing.T) {
+	code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
+		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
+	}
+	code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Errorf("run = %d, out=%q", code, out)
+	}
+	if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
+		t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
+	}
+	if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
+		t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
+	}
+}
+
+func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{nil, ""},
+		{[]string{"--format", "json"}, "[]\n"},
+		{[]string{"--format", "ndjson"}, ""},
+		{[]string{"--format", "csv"}, "first,last\n"},
+		{[]string{"--format", "sql"}, ""},
+	} {
+		if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
+			t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
+		}
+	}
+	if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
+		t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
+	}
+}
+
+func TestRunRepeatIsBounded(t *testing.T) {
+	for _, args := range [][]string{
+		{"--repeat", "9223372036854775807"},
+		{"--repeat", "1048577"},
+		{"-n", "99999999999"},
+	} {
+		code, out, errb := runOut("{/sv_SE.word}", args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
+			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
+		}
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
+	if code != 0 || len(out) != 1048576 {
+		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
+	}
+}
+
+func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		stdin         string
+		base, ignored []string
+	}{
+		{"", []string{"--list"}, []string{"-n", "3"}},
+		{"", []string{"--list"}, []string{"--separator", ","}},
+		{"", []string{"--list"}, []string{"--format", "json"}},
+		{"", []string{"--list"}, []string{"--table", "t"}},
+		{"{/users}", nil, []string{"--table", "t"}},
+		{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
+	} {
+		args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
+		_, want, _ := runOut(c.stdin, args...)
+		code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
+		if code != 0 || want == "" || got != want {
+			t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
+		}
+	}
+}
+
+func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
+	if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
+		t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
+	}
+}
+
+func TestRunRendersAnyJSONValue(t *testing.T) {
+	for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
+		if code, out, errb := runOut(stdin); code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
+		}
+	}
+}
+
+func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "-ä")
+	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
+		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
+	}
+}
+
+func TestRunEmptyDataPathIsNamed(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
+	if code != 1 || !strings.Contains(errb, "empty") {
+		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
+	}
+}
+
+func recordDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
+	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestRunRecordJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
+		}
+	}
+}
+
+func TestRunRecordCSVRoundTrips(t *testing.T) {
+	code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 4 {
+		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
+	}
+	if lines[0] != "first,last" {
+		t.Fatalf("csv header = %q, want first,last", lines[0])
+	}
+	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
+	if err != nil {
+		t.Fatalf("csv output is not valid CSV: %v", err)
+	}
+	if len(r) != 4 || len(r[0]) != 2 {
+		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
+	}
+}
+
+func TestRunRecordSQL(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
+		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
+	}
+}
+
+func TestRunRecordDefaultTable(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
+	}
+}
+
+func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
+	code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
+	}
+	code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
+	if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
+		t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
+	}
+}
+
+func TestRunRecordInlineTemplate(t *testing.T) {
+	code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
+	if code != 0 {
+		t.Fatalf("inline record = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
+	}
+}
+
+func TestRunRecordMisuse(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
+		{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
+		{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
+		{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
+	} {
+		code, out, errb := runOut("{/users}", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
+	if code != 1 {
+		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
+	}
+}
+
+func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
+	dir := t.TempDir()
+	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
+	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
+	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
+	}
+	code, _, errb = runOut(content, "--format", "json")
+	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
+	}
+	if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
+		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
+	}
+}
+
+func TestRunNumbersFollowTheShell(t *testing.T) {
+	_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
+	code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
+	if code != 0 || got != want {
+		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	}
+}
+
+// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
+// for columns: the record to write, and the quoting a shell needs to take it. The
+// template is well-formed and only the data cannot serve it, so the code is 1.
+func TestRunRecordOfAFieldlessPath(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
+	}
+	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
+		if !strings.Contains(errb, want) {
+			t.Errorf("stderr = %q, want it to name %s", errb, want)
+		}
+	}
+	if out != "" {
+		t.Errorf("stdout = %q, want nothing", out)
+	}
+}
+
+func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
+	dir := t.TempDir()
+	for name, data := range map[string]string{
+		".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
+		"bad.json":       `"{/nope}"`,
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
+		if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
+			t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
+		}
+	}
+}

cli/lone_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// grammarLoneReference is the path a format reads when the grammar parses it as one path
+// read of one reference from the root, and nothing else.
+func grammarLoneReference(text string) (string, bool) {
+	toks, err := grammar.ParseFormat(text)
+	if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
+		return "", false
+	}
+	sigil, path, err := grammar.RefShape(toks[0].Arms[0])
+	if err != nil || sigil != "/" {
+		return "", false
+	}
+	return path, true
+}
+
+func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
+	for _, text := range []string{
+		" {/x}",
+		"x",
+		"{.x}",
+		"{/.x}",
+		"{//x}",
+		"{/a...b}",
+		"{/a..b}",
+		"{/a..}",
+		"{/a.[x]}",
+		"{/a.}",
+		"{/a|/b}",
+		"{/a}{/b}",
+		"{/misc.currency[US Dollar (Next day)]}",
+		"{/sv_SE.person}",
+		"{/t[a as b].c}",
+		"{/t[f(x)]}",
+		"{/x as p}",
+		"{/x(1)}",
+		"{/x}}",
+		"{/}",
+	} {
+		path, lone := loneReference(text)
+		want, wantLone := grammarLoneReference(text)
+		if lone != wantLone || path != want {
+			t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
+		}
+	}
+}

cmd/fejkdata/main.go

@@ -1,512 +1,12 @@
-// Command fejkdata prints fake values from the shipped data and any directories
+// Command fejkdata renders the template on its stdin with fake values from every shipped
-// layered over it.
+// module; package cli documents it.
-//
-//	fejkdata sv_SE.person                        # a full person
-//	fejkdata sv_SE.person.last                   # just the surname
-//	fejkdata --data-path ./mydata sv_SE.person   # layer custom data; the last dir wins
-//	fejkdata --seed 42 sv_SE.address
 package main
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"runtime/debug"
-	"sort"
-	"strconv"
-	"strings"
 
-	"gitea.larvit.se/larvit/fejkdata"
+	"github.com/larvit/fejkdata/cli"
+	"github.com/larvit/fejkdata/data"
 )
 
-const usage = `Usage: fejkdata [flags] <path|template>
+func main() { os.Exit(cli.Main(data.Modules()...)) }
-
-  <path>                 a category, or a dotted path into one (person, person.last);
-                         a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'
-  <template>             a format string or JSON value to render inline, e.g.
-                         'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
-
-A layout inside a template is single-quoted, so quote the whole argument with " to
-keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
-
-An argument containing a { token, or a JSON object, array or string, is a
-template; any other argument is a path (a path never contains a brace or a quote,
-and a bracket only as a [selector] after a table's name). Templates reach the
-data by reference from the root — {/sv_SE.person.last} — whether the data is
-shipped or layered with --data-path. An argument carrying a closing brace or a
-quote but no valid JSON names neither.
-
-With --format json, ndjson, csv or sql the argument must name a record — a
-template whose fields are its columns — and the rows are written as one JSON
-array, one JSON object per line, one CSV row (after a header), or one INSERT.
-
-  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, ndjson, csv or sql
-  -h, --help             print this help, then exit
-      --list             list the paths the data offers, then exit
-      --no-shipped-data  load only the --data-path directories
-  -n, --repeat N         render the value N times, 1..1048576 (default 1)
-  -s, --seed N           seed for reproducible output
-      --separator S      string between repeated values (default newline)
-      --table T          the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
-      --version          print the version, then exit
-
-Flags may come before or after <path|template>; -- ends the flags. A short flag's
-value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
-`
-
-type invocation struct {
-	dirs         []string
-	format       string
-	formatSet    bool
-	help         bool
-	list         bool
-	noShipped    bool
-	paths        []string
-	repeat       int
-	repeatSet    bool
-	seed         uint64
-	seeded       bool
-	separator    string
-	separatorSet bool
-	table        string
-	tableSet     bool
-	version      bool
-}
-
-type flagDef struct {
-	long  string
-	short string
-	value bool
-	set   func(*invocation, string) error
-}
-
-var flagDefs = []flagDef{
-	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
-	{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
-	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
-	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
-	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
-	{"repeat", "n", true, func(in *invocation, v string) error {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > fejkdata.MaxRepeat {
-			return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
-		}
-		in.repeat, in.repeatSet = n, true
-		return nil
-	}},
-	{"seed", "s", true, func(in *invocation, v string) error {
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
-		}
-		in.seed, in.seeded = n, true
-		return nil
-	}},
-	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
-	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
-	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
-}
-
-func flagByLong(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].long == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-func flagByShort(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].short == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-// flagArg is one flag as written: its definition and the value attached to it,
-// if any.
-type flagArg struct {
-	def    *flagDef
-	value  string
-	inline bool
-}
-
-// splitFlags reads one argv element as flags: --name or --name=value, or -abc
-// where each letter is a short flag and the first that takes a value takes the
-// rest of the element.
-func splitFlags(arg string) ([]flagArg, error) {
-	if strings.HasPrefix(arg, "--") {
-		name, value, inline := strings.Cut(arg[2:], "=")
-		def := flagByLong(name)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag --%s", name)
-		}
-		return []flagArg{{def, value, inline}}, nil
-	}
-	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
-		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
-	}
-	var flags []flagArg
-	letters := arg[1:]
-	for i, r := range letters {
-		letter := string(r)
-		def := flagByShort(letter)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag -%s", letter)
-		}
-		if !def.value {
-			flags = append(flags, flagArg{def: def})
-			continue
-		}
-		rest := letters[i+len(letter):]
-		if strings.HasPrefix(rest, "=") {
-			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
-				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
-		}
-		return append(flags, flagArg{def, rest, rest != ""}), nil
-	}
-	return flags, nil
-}
-
-func parseArgs(argv []string) (invocation, error) {
-	in := invocation{repeat: 1, separator: "\n", format: "text"}
-	for i := 0; i < len(argv); i++ {
-		arg := argv[i]
-		if arg == "--" {
-			in.paths = append(in.paths, argv[i+1:]...)
-			return in, nil
-		}
-		if len(arg) < 2 || arg[0] != '-' {
-			in.paths = append(in.paths, arg)
-			continue
-		}
-		flags, err := splitFlags(arg)
-		if err != nil {
-			return in, err
-		}
-		for _, fl := range flags {
-			if !fl.def.value {
-				if fl.inline {
-					return in, fmt.Errorf("--%s takes no value", fl.def.long)
-				}
-				_ = fl.def.set(&in, "")
-				continue
-			}
-			value := fl.value
-			if !fl.inline {
-				if i++; i >= len(argv) {
-					return in, fmt.Errorf("--%s needs a value", fl.def.long)
-				}
-				value = argv[i]
-			}
-			if err := fl.def.set(&in, value); err != nil {
-				return in, err
-			}
-		}
-	}
-	return in, nil
-}
-
-// recordFormat is one way to write a record out: the line each record renders,
-// the header that precedes the first one, and the open/close frame plus the
-// between-record separator a document form needs.
-type recordFormat struct {
-	header func(*fejkdata.Record) string
-	line   func(r *fejkdata.Record, table string) string
-	open   string
-	close  string
-	sep    string
-}
-
-// recordFormats is every --format that writes records. json frames the records
-// as one array document; ndjson is the same column, one object per line.
-var recordFormats = map[string]recordFormat{
-	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
-	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
-	"ndjson": {line: jsonLine, sep: "\n"},
-	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
-}
-
-func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
-
-// writesRecords reports whether the format writes records rather than plain text.
-func (in invocation) writesRecords() bool {
-	return in.format != "text"
-}
-
-// formatNames lists the --format values, text included, for the misuse error.
-func formatNames() string {
-	names := []string{"text"}
-	for name := range recordFormats {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
-}
-
-// checkFlags rejects a flag value or combination that cannot run.
-func (in invocation) checkFlags() error {
-	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
-		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
-	}
-	if in.tableSet && in.table == "" {
-		return errors.New("--table names the INSERT target, so it cannot be empty")
-	}
-	if in.tableSet && in.format != "sql" {
-		return errors.New("--table names the INSERT target, so it needs --format sql")
-	}
-	if in.writesRecords() && in.separatorSet {
-		return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
-	}
-	return nil
-}
-
-// check rejects a flag combination or an argument that cannot run, and reports
-// what the argument names, so its shape is settled before any data is read.
-func (in invocation) check() (argKind, error) {
-	if err := in.checkFlags(); err != nil {
-		return argPath, err
-	}
-	if in.list && len(in.paths) > 0 {
-		return argPath, errors.New("--list takes no path")
-	}
-	if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
-		return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
-	}
-	if in.list {
-		return argPath, nil
-	}
-	if len(in.paths) != 1 {
-		return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
-	}
-	return classify(in.paths[0])
-}
-
-func (in invocation) options() []fejkdata.Option {
-	var opts []fejkdata.Option
-	if in.noShipped {
-		opts = append(opts, fejkdata.WithoutShippedData())
-	}
-	for _, dir := range in.dirs {
-		opts = append(opts, fejkdata.WithDataPath(dir))
-	}
-	if in.seeded {
-		opts = append(opts, fejkdata.WithSeed(in.seed))
-	}
-	return opts
-}
-
-// write streams the argument's renders to w, repeat of them joined by the
-// separator and ended by a newline. A value that renders once renders every time,
-// so a render failure comes before anything is written; a write failure surfaces
-// from Flush, bufio keeping the first one.
-func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	if in.writesRecords() {
-		return in.writeRecords(f, kind, w)
-	}
-	draw, err := in.textDraw(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	out := bufio.NewWriter(w)
-	for i := 0; i < in.repeat; i++ {
-		v, err := draw()
-		if err != nil {
-			return err
-		}
-		if i > 0 {
-			out.WriteString(in.separator)
-		}
-		out.WriteString(v)
-	}
-	out.WriteString("\n")
-	return out.Flush()
-}
-
-// textDraw builds what one text render yields: the value, from a path or an
-// inline template.
-func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if kind != argTemplate {
-		return func() (string, error) { return f.Fake(arg) }, nil
-	}
-	t, err := f.NewTemplate(arg)
-	if err != nil {
-		return nil, templateError{err}
-	}
-	return func() (string, error) { return t.Fake(), nil }, nil
-}
-
-// recordStream builds the record drawer for the argument, plus the INSERT table
-// a sql format names.
-func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
-	record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
-	if kind == argTemplate {
-		t, err := f.NewRecordTemplate(arg)
-		if err != nil {
-			return nil, "", templateError{err}
-		}
-		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
-	}
-	table := in.table
-	if table == "" {
-		table = defaultTable(arg, kind)
-	}
-	return record, table, nil
-}
-
-// writeRecords streams a record per line in the chosen format, framing a document
-// form with its open/close brackets and a header preceding the first record.
-func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	record, table, err := in.recordStream(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	format := recordFormats[in.format]
-	out := bufio.NewWriter(w)
-	if format.open != "" {
-		out.WriteString(format.open)
-		out.WriteByte('\n')
-	}
-	for i := 0; i < in.repeat; i++ {
-		r, err := record()
-		if err != nil {
-			return err
-		}
-		if i == 0 && format.header != nil {
-			out.WriteString(format.header(r))
-			out.WriteByte('\n')
-		}
-		if i > 0 {
-			out.WriteString(format.sep)
-		}
-		out.WriteString(format.line(r, table))
-	}
-	if format.close != "" {
-		out.WriteByte('\n')
-		out.WriteString(format.close)
-	}
-	out.WriteByte('\n')
-	return out.Flush()
-}
-
-// defaultTable names the INSERT target when --table is absent: the path's last
-// segment, or "records" for an inline template that sits in no folder.
-func defaultTable(arg string, kind argKind) string {
-	if kind == argTemplate {
-		return "records"
-	}
-	var names strings.Builder // the path with its [selectors] cut out
-	for depth, i := 0, 0; i < len(arg); i++ {
-		switch {
-		case arg[i] == '[':
-			depth++
-		case arg[i] == ']':
-			depth--
-		case depth == 0:
-			names.WriteByte(arg[i])
-		}
-	}
-	segments := strings.Split(names.String(), ".")
-	return segments[len(segments)-1]
-}
-
-// templateError marks a render failure that is the argument's own fault — an
-// inline template that does not compile. run reports it as misuse (exit 2, with a
-// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
-type templateError struct{ error }
-
-func (e templateError) Unwrap() error { return e.error }
-
-type argKind int
-
-const (
-	argPath argKind = iota
-	argTemplate
-)
-
-// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
-func classify(arg string) (argKind, error) {
-	inline, err := fejkdata.IsTemplate(arg)
-	if inline {
-		return argTemplate, nil
-	}
-	return argPath, err
-}
-
-func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
-
-// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
-func run(args []string, stdout, stderr io.Writer) int {
-	in, err := parseArgs(args)
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	if in.help {
-		fmt.Fprint(stdout, usage)
-		return 0
-	}
-	if in.version {
-		fmt.Fprintln(stdout, "fejkdata "+buildVersion())
-		return 0
-	}
-	kind, err := in.check()
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	f, err := fejkdata.New(in.options()...)
-	if errors.Is(err, fejkdata.ErrNoData) {
-		return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
-	}
-	if err != nil {
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	if in.list {
-		for _, p := range f.List() {
-			fmt.Fprintln(stdout, p)
-		}
-		return 0
-	}
-	if err := in.write(f, kind, stdout); err != nil {
-		var te templateError
-		if errors.As(err, &te) {
-			return misuse(stderr, te.error)
-		}
-		fmt.Fprintln(stderr, err)
-		if errors.Is(err, fejkdata.ErrNoColumns) {
-			fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
-		}
-		return 1
-	}
-	return 0
-}
-
-// templateSplit names the cure for a template a shell split on its spaces, which is
-// what several arguments carrying a token mean.
-func templateSplit(paths []string) string {
-	for _, p := range paths {
-		if strings.Contains(p, "{") {
-			return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
-		}
-	}
-	return ""
-}
-
-func misuse(stderr io.Writer, err error) int {
-	// A library error already names the program, so the prefix is not doubled.
-	fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
-	return 2
-}
-
-// buildVersion is the module version go install stamps into the binary, or "devel".
-func buildVersion() string {
-	if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
-		return info.Main.Version
-	}
-	return "devel"
-}

cmd/fejkdata/main_test.go

@@ -2,640 +2,46 @@ package main
 
 import (
 	"bytes"
-	"encoding/csv"
-	"encoding/json"
 	"os"
-	"path/filepath"
+	"os/exec"
-	"regexp"
 	"strings"
 	"testing"
 )
 
-const (
+func TestMain(m *testing.M) {
-	svSE = "../../data/sv_SE"
+	if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
-	enUS = "../../data/en_US"
+		main()
-	misc = "../../data/misc"
+	}
-)
+	os.Exit(m.Run())
-
-func runOut(args ...string) (int, string, string) {
-	var out, errb bytes.Buffer
-	code := run(args, &out, &errb)
-	return code, out.String(), errb.String()
 }
 
-func TestRunOutputsValue(t *testing.T) {
+func TestMainRendersTheShippedData(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "person")
+	cmd := exec.Command(os.Args[0], "--seed", "1")
-	if code != 0 {
+	cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-		t.Fatalf("run = %d, stderr=%q", code, errb)
+	cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
-	}
+	var stderr bytes.Buffer
-	if strings.TrimSpace(out) == "" {
+	cmd.Stderr = &stderr
-		t.Fatalf("empty output, stderr=%q", errb)
+	out, err := cmd.Output()
-	}
+	if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
-	if !strings.HasSuffix(out, "\n") {
+		t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
-		t.Errorf("output should end with newline, got %q", out)
 	}
 }
 
-func TestRunDotPath(t *testing.T) {
+func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
-	code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
+	var outs [2]string
-	if code != 0 {
+	for i := range outs {
-		t.Fatalf("person run = %d", code)
+		cmd := exec.Command(os.Args[0])
-	}
+		cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-	code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
+		cmd.Stdin = strings.NewReader("{uuid()}")
-	if code != 0 {
+		var stderr bytes.Buffer
-		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+		cmd.Stderr = &stderr
-	}
+		out, err := cmd.Output()
-	if strings.TrimSpace(last) == "" {
+		if err != nil || len(out) == 0 {
-		t.Fatal("empty person.last output")
+			t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
-	}
-	if last == full {
-		t.Errorf("person.last %q should differ from person %q", last, full)
-	}
-}
-
-func TestRunSeedSpellings(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"--seed=42", "--data-path", svSE, "address"},
-		{"-s", "42", "-d", svSE, "address"},
-		{"--data-path", svSE, "address", "--seed", "42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
 		}
-	}
+		outs[i] = string(out)
-}
+	}
-
+	if outs[0] == outs[1] {
-func TestRunShortFlagValues(t *testing.T) {
+		t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"-s42", "-d", svSE, "address"},
-		{"-s", "42", "-d" + svSE, "address"},
-		{"-d", svSE, "address", "-s42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-	_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
-	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
-		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
-	}
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
-		{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
-		{[]string{"-d=" + svSE, "address"}, "--data-path="},
-		{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
-		{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" {
-			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-	code, out, _ := runOut("-hd", svSE)
-	if code != 0 || !strings.Contains(out, "Usage") {
-		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
-	}
-}
-
-func TestRunDoubleDashEndsFlags(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--", "person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, _, errb = runOut("--data-path", svSE, "--", "--list")
-	if code != 1 || !strings.Contains(errb, "--list") {
-		t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestRunSingleDashLongIsRejected(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-data-path", svSE, "person"}, "use --data-path"},
-		{[]string{"-list", "-d", svSE}, "use --list"},
-		{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
-		{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunFlagValues(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
-		{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
-		{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
-		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
-		{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
-		{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunRepeat(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 3 {
-		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
-	}
-	_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
-	if short != out {
-		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
-	}
-}
-
-func TestRunSeparator(t *testing.T) {
-	code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if n := strings.Count(out, "\n"); n != 1 {
-		t.Errorf("want one trailing newline, got %d: %q", n, out)
-	}
-	if !strings.Contains(out, ",") {
-		t.Errorf("values should be comma-joined: %q", out)
-	}
-}
-
-func TestRunRepeatAdvancesRNG(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	uniq := map[string]bool{}
-	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
-		uniq[l] = true
-	}
-	if len(uniq) < 2 {
-		t.Errorf("repeat should vary output, all identical: %q", out)
-	}
-}
-
-func TestRunMisuse(t *testing.T) {
-	for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
-		code, out, errb := runOut(args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", args, code)
-		}
-		if !strings.Contains(errb, "try 'fejkdata --help'") {
-			t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
-		}
-		if out != "" {
-			t.Errorf("run(%v) stdout = %q, want nothing", args, out)
-		}
-	}
-	// A template a shell split names both quotings, since which one cures it turns
-	// on whether the argument carries a double-quoted JSON.
-	_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
-	for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
-		}
-	}
-}
-
-func TestRunMultipleDataPaths(t *testing.T) {
-	code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatal("empty output for multi-dir run")
-	}
-}
-
-func TestRunUnknownCategoryFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", svSE, "nope")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if !strings.Contains(errb, "nope") {
-		t.Errorf("stderr %q should name the unknown category", errb)
-	}
-}
-
-func TestRunList(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	for _, want := range []string{"person", "person.last", "address"} {
-		if !strings.Contains(out, want) {
-			t.Errorf("list output missing %q:\n%s", want, out)
-		}
-	}
-}
-
-func TestRunHelpOnStdout(t *testing.T) {
-	for _, h := range []string{"-h", "--help"} {
-		code, out, errb := runOut(h)
-		if code != 0 {
-			t.Errorf("%s = %d, want 0", h, code)
-		}
-		if !strings.Contains(out, "Usage") || errb != "" {
-			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
-		}
-	}
-}
-
-func TestRunVersion(t *testing.T) {
-	code, out, errb := runOut("--version")
-	if code != 0 {
-		t.Fatalf("--version = %d, stderr=%q", code, errb)
-	}
-	version, ok := strings.CutPrefix(out, "fejkdata ")
-	if !ok || strings.TrimSpace(version) == "" {
-		t.Errorf("--version = %q, want the command name and a version on stdout", out)
-	}
-}
-
-func TestRunMissingDirFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", "../../data/nope", "person")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if errb == "" {
-		t.Error("want an error message on stderr")
-	}
-}
-
-func TestRunShippedDataByDefault(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "sv_SE.person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, list, _ := runOut("--list")
-	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
-		t.Errorf("--list without --data-path = %d, %q", code, list)
-	}
-	code, _, errb = runOut("person")
-	if code != 1 || !strings.Contains(errb, "person") {
-		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestClassify(t *testing.T) {
-	for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
-		if got, err := classify(arg); err != nil || got != want {
-			t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
-		}
-	}
-	if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
-		t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
-	}
-	if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
-		t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
-	}
-}
-
-func TestRunSelectsATableRow(t *testing.T) {
-	dir := t.TempDir()
-	for name, content := range map[string]string{
-		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
-		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
-	} {
-		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
-		t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
-		t.Fatalf("region[Skåne län].code = %d, %q", code, out)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
-		t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
-		t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
-		t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
-		t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out)
-	}
-}
-
-func TestUsageReferencesResolve(t *testing.T) {
-	for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
-		path := token[2 : len(token)-1]
-		code, out, errb := runOut("--seed", "1", path)
-		if code != 0 || strings.TrimSpace(out) == "" {
-			t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
-		}
-	}
-}
-
-func TestRunShapeMisuseBeforeLoad(t *testing.T) {
-	code, _, errb := runOut("--no-shipped-data", "[abc]")
-	if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
-		t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
-	}
-}
-
-func TestRunInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
-	if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
-		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
-	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
-		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
-	}
-}
-
-func TestRunTemplateMisuse(t *testing.T) {
-	for arg, want := range map[string]string{
-		"{bad":                  "unterminated",
-		"[red,green]":           `starts with "["`,
-		`{"format":"x"}`:        "is a string",
-		"name: {/no.such.path}": "no entry",
-		"{/sv_SE.person}":       "write sv_SE.person",
-		"x[1]y":                 `"]"`,
-		` ["a","b"] `:           "may not be padded",
-	} {
-		code, out, errb := runOut("--seed", "1", arg)
-		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
-			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
-		}
-		if strings.Contains(errb, "fejkdata: fejkdata:") {
-			t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
-		}
-	}
-}
-
-func TestRunNoShippedData(t *testing.T) {
-	code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
-		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
-	}
-	code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Errorf("run = %d, out=%q", code, out)
-	}
-	code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
-	if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
-		t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
-	}
-}
-
-func TestRunRepeatIsBounded(t *testing.T) {
-	for _, args := range [][]string{
-		{"--repeat", "9223372036854775807", "sv_SE.word"},
-		{"--repeat", "1048577", "sv_SE.word"},
-		{"-n", "99999999999", "sv_SE.word"},
-	} {
-		code, out, errb := runOut(args...)
-		if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
-			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
-		}
-	}
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
-	if code != 0 || len(out) != 1048576+1 {
-		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
-	}
-}
-
-func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
-	for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
-		code, _, errb := runOut(args...)
-		if code != 2 || !strings.Contains(errb, "--list takes no") {
-			t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
-		}
-	}
-}
-
-func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
-	code, _, errb := runOut("-ä", "sv_SE.word")
-	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
-		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
-	}
-}
-
-func TestRunEmptyDataPathIsNamed(t *testing.T) {
-	code, _, errb := runOut("--data-path=", "sv_SE.word")
-	if code != 1 || !strings.Contains(errb, "empty") {
-		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
-	}
-}
-
-func recordDir(t *testing.T) string {
-	t.Helper()
-	dir := t.TempDir()
-	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
-	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
-func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil {
-		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
-	}
-	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
-		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
-	}
-}
-
-func TestRunRecordNDJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
-	}
-	for _, line := range lines {
-		var m map[string]string
-		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
-			t.Fatalf("ndjson line %q is not one object: %v", line, err)
-		}
-	}
-}
-
-func TestRunRecordCSVRoundTrips(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 4 {
-		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
-	}
-	if lines[0] != "first,last" {
-		t.Fatalf("csv header = %q, want first,last", lines[0])
-	}
-	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
-	if err != nil {
-		t.Fatalf("csv output is not valid CSV: %v", err)
-	}
-	if len(r) != 4 || len(r[0]) != 2 {
-		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
-	}
-}
-
-func TestRunRecordSQL(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
-		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
-	}
-}
-
-func TestRunRecordDefaultTable(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
-	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
-		t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
-	}
-}
-
-func TestRunRecordInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
-	if code != 0 {
-		t.Fatalf("inline record = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
-		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
-	}
-}
-
-func TestRunRecordMisuse(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
-		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
-		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
-		}
-	}
-}
-
-func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
-	if code != 1 {
-		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
-	}
-}
-
-func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
-	dir := t.TempDir()
-	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
-	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
-	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
-	}
-	code, _, errb = runOut("--format", "json", content)
-	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
-		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
-	}
-}
-
-func TestRunNumbersFollowTheShell(t *testing.T) {
-	_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
-	code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
-	if code != 0 || got != want {
-		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
-	}
-}
-
-// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
-// for columns: the record to write, and the quoting a shell needs to take it. The
-// argument is well-formed and only the data cannot serve it, so the code is 1.
-func TestRunRecordOfAFieldlessPath(t *testing.T) {
-	code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
-	if code != 1 {
-		t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
-	}
-	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("stderr = %q, want it to name %s", errb, want)
-		}
-	}
-	if out != "" {
-		t.Errorf("stdout = %q, want nothing", out)
 	}
 }

compose.yaml

@@ -1,21 +1,5 @@
-# Docker-based dev workflow — no local tooling beyond Docker is required.
-#
-#   docker compose run --rm test
-#   docker compose run --rm vet
-#   docker compose run --rm build
-#
-# The full gate — vet, formatting and tests — is `docker build .`, which is what
-# CI runs.
-#
-# Build caches persist in the `gocache` volume, so re-runs are fast and the
-# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
-# ownership when run with `--user "$(id -u):$(id -g)"`.
-#
-# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
-# supported Go with: GO_VERSION=1.22 docker compose run --rm test
-
 x-go: &go
-  image: golang:${GO_VERSION:-1.26.4}
+  image: golang:${GO_VERSION:-1.27.2}
   working_dir: /app
   volumes:
     - .:/app
@@ -33,12 +17,16 @@ services:
 
   cover:
     <<: *go
-    command: go test -cover ./...
+    command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
 
   bench:
     <<: *go
     command: go test -run XXX -bench . -benchmem ./...
 
+  generate:
+    <<: *go
+    command: go generate ./...
+
   vet:
     <<: *go
     command: go vet ./...
@@ -47,6 +35,10 @@ services:
     <<: *go
     command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
 
+  vulncheck:
+    <<: *go
+    command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
+
   build:
     <<: *go
     command: go build ./...
@@ -59,13 +51,12 @@ services:
     <<: *go
     command: go mod tidy
 
-  # Interactive shell for ad-hoc work: docker compose run --rm dev
   dev:
     <<: *go
     command: sh
 
   data-import:
-    image: python:3.14.7-slim
+    image: python:3.14.8-slim
     working_dir: /app
     volumes:
       - .:/app

data-import/geo-se.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
 
-    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
+    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import csv
 import io
 import json
-import math
 import os
 import re
 import sys
@@ -16,6 +14,7 @@ import urllib.request
 import zipfile
 from pathlib import Path
 
+import geo
 import source
 import tsv
 import xlsx
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
 POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
 NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
 NVDB_PAGE = 50000
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
-CACHE = Path(__file__).resolve().parent / "cache"
 TIMEZONE = "Europe/Stockholm"
+# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
 ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
+# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
 UNMATCHED_POPULATION = 200
+# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
+# or a description in one.
+NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
+PARTICLES = {"af", "de", "von"}
 
 
 def scb_codes(cache):
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
         yield name, float(lat), float(lon)
 
 
-class Nearest:
-    """Nearest point by an equirectangular distance, over a degree grid."""
-
-    def __init__(self, points, cell=0.05):
-        self.cell = cell
-        self.grid = collections.defaultdict(list)
-        for lat, lon, value in points:
-            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
-
-    def find(self, lat, lon):
-        ci, cj = int(lat // self.cell), int(lon // self.cell)
-        best, best_d = None, math.inf
-        ring = 0
-        while ring < 400:
-            for i in range(ci - ring, ci + ring + 1):
-                for j in range(cj - ring, cj + ring + 1):
-                    if max(abs(i - ci), abs(j - cj)) != ring:
-                        continue
-                    for plat, plon, value in self.grid.get((i, j), ()):
-                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
-                        if d < best_d:
-                            best, best_d = value, d
-            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
-                return best
-            ring += 1
-        return best
-
-
 def street_delivery(name, codes):
-    """The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
+    """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
+    least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
+    for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
+    docs/research/research-geo-se.md, Postal codes → Structure."""
     if name in ONE_POSITION:
         return [c for c in codes if c[1] != "0"]
     largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
 
 
 def well_cased(name):
+    """Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
+    capitals or lower case."""
     return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
 
 
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
         municipality, method = municipality_of(name, rows, tatorter, municipalities)
         how[method] += 1
         kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
-        with_point = [r for r in rows if r["lat"] is not None]
+        point = geo.centroid(rows)
-        if municipality is None or not kept or not with_point or not well_cased(name):
+        if municipality is None or not kept or point is None or not well_cased(name):
             continue
-        lat = sum(r["lat"] for r in with_point) / len(with_point)
+        lat, lon = point
-        lon = sum(r["lon"] for r in with_point) / len(with_point)
         out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
-    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr)
+    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
     return out
 
 
-def streets(segments, codes, localities, per_locality):
+def street_name(name):
+    # A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
+    opening = name[0].isupper() or name.split()[0] in PARTICLES
+    return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
+
+
+def streets(segments, codes, localities, share, minimum):
     """The names with most segments per locality, each segment at its nearest code centroid."""
-    nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
+    nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
     count = collections.Counter()
     for name, lat, lon in segments:
-        if name[0].isalpha():
+        if street_name(name):
             count[(nearest.find(lat, lon), name)] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "segments")
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
     key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
     if not key:
         sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
-    cache, out = Path(a.cache), Path(a.out)
+    cache, out = geo.directories(a)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
-    regions, municipalities = scb_codes(cache)
+    region_names, municipality_names = scb_codes(cache)
     population = scb_population(cache)
     codes = geonames(cache)
-    places = localities(codes, scb_tatorter(cache), municipalities, population)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
-    named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
+    named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
-    places = {name: l for name, l in places.items() if name in named}
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
-    if empty:
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
-        sys.exit(f"municipalities without a locality: {empty}")
+    geo.log_dropped_localities(dropped["locality"])
+    if dropped["municipality"] or dropped["region"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo-us.py

@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
-"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
+"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
 
-    data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
+    data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import concurrent.futures
 import csv
 import io
+import math
+import random
 import re
 import struct
 import sys
+import urllib.parse
 import zipfile
-from pathlib import Path
 
+import geo
 import source
 import tsv
 
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
 PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
 ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
 TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
-CACHE = Path(__file__).resolve().parent / "cache"
 ESTIMATE = "POPESTIMATE2025"
 CDP = "57"
+# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
+CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
+SELECT ?iso ?capitalLabel ?gnis WHERE {
+  ?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
+  OPTIONAL { ?capital wdt:P590 ?gnis. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}"""})
+SHIPPED_POPULATION = 100000
 HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
 SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
-# Places whose Census name is a merged government's; the postal city is what an address carries.
+# Places whose Census name is not the postal city an address carries.
-NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
+NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
 TIMEZONES = {
     "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
     "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -107,8 +115,8 @@ def place_name(geoid, name):
     return stripped
 
 
-def localities(cache, min_population, counties):
+def localities(cache, counties):
-    """Each shipped place with its county, population and centroid."""
+    """Each incorporated place with its county, population and centroid."""
     place_population, county_part = {}, collections.defaultdict(list)
     for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
         if r["SUMLEV"] == "162":
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
     out = {}
     for r in gazetteer(cache, "place"):
         geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
-        if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid):
+        if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
             continue
         county = max(county_part[geoid])[1]
         if county not in counties:
             print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
             continue
-        out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
+        out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
     return out
 
 
-def postal_codes(cache, localities):
+def capitals(cache, places, counties):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
+    """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
+    by_gnis = {p["gnis"]: code for code, p in places.items()}
+    named = collections.defaultdict(list)
+    for code, p in places.items():
+        named[(counties[p["municipality"]]["region"], p["name"])].append(code)
+    by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
+    gnis_of = collections.defaultdict(set)
+    for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
+        gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
+    out = set()
+    for (state, name), ids in sorted(gnis_of.items()):
+        code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
+        if code:
+            out.add(code)
+        else:
+            print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
+    return out
+
+
+def drawn(places, share):
+    """The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
+    sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
+    the sources moves only the places near the cut."""
+    peopled = [code for code, p in places.items() if p["population"] > 0]
+    key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
+    return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
+
+
+def zcta_localities(cache, localities):
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
     return {zcta: place for zcta, place in largest.items() if place in localities}
 
 
-def streets(cache, counties, locality_of_zcta, per_locality):
+def streets(cache, counties, locality_of_zcta, share, minimum):
     """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
     with concurrent.futures.ThreadPoolExecutor(3) as pool:
         list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
             if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
                 for z in zips.get(r["TLID"], ()):
                     count[(locality_of_zcta[z], r["FULLNAME"])] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "addresses"), addresses
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
-    return named, addresses
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
+    cache, out = geo.directories(a)
-    p.add_argument("--min-population", type=int, default=25000)
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
     state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
     regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
@@ -182,20 +208,29 @@ def main():
         counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
 
-    places = localities(cache, a.min_population, counties)
+    places = localities(cache, counties)
-    locality_of_zcta = postal_codes(cache, places)
+    owner_of_zcta = zcta_localities(cache, places)
-    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
+    capital_codes = capitals(cache, places, counties)
-    for geoid in [l for l in places if l not in named]:
+    owners = {p: places[p] for p in set(owner_of_zcta.values())}
-        print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr)
+    for code in sorted(capital_codes - set(owners)):
-        del places[geoid]
+        print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
-    kept_counties = {l["municipality"] for l in places.values()}
+    places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
+    locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
+    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped_localities(dropped["locality"])
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
-    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo.py

@@ -0,0 +1,98 @@
+"""The steps a country's geo import may share with another's."""
+import argparse
+import collections
+import math
+import sys
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
+CACHE = Path(__file__).resolve().parent / "cache"
+
+
+def parser(doc, country, street_share, **own):
+    """A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
+    "_" standing for "-"."""
+    options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
+    p = argparse.ArgumentParser(description=doc.splitlines()[0])
+    for name in sorted(options):
+        p.add_argument("--" + name.replace("_", "-"), **options[name])
+    return p
+
+
+def directories(args):
+    """The cache and out directories of the parsed options, both created."""
+    cache, out = Path(args.cache), Path(args.out)
+    cache.mkdir(parents=True, exist_ok=True)
+    out.mkdir(parents=True, exist_ok=True)
+    return cache, out
+
+
+def centroid(rows):
+    """The mean lat and lon of the rows with a point, or None where no row has one."""
+    points = [r for r in rows if r["lat"] is not None]
+    if not points:
+        return None
+    return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
+
+
+class Nearest:
+    """Nearest point by an equirectangular distance, over a degree grid."""
+
+    def __init__(self, points, cell=0.05):
+        self.cell = cell
+        self.grid = collections.defaultdict(list)
+        for lat, lon, value in points:
+            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
+
+    def find(self, lat, lon):
+        ci, cj = int(lat // self.cell), int(lon // self.cell)
+        best, best_d = None, math.inf
+        ring = 0
+        while ring < 400:
+            for i in range(ci - ring, ci + ring + 1):
+                for j in range(cj - ring, cj + ring + 1):
+                    if max(abs(i - ci), abs(j - cj)) != ring:
+                        continue
+                    for plat, plon, value in self.grid.get((i, j), ()):
+                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
+                        if d < best_d:
+                            best, best_d = value, d
+            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
+                return best
+            ring += 1
+        return best
+
+
+def top_streets(count, share, minimum, column):
+    """count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
+    them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
+    of = collections.defaultdict(list)
+    for (locality, name), n in count.items():
+        of[locality].append((n, name))
+    return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
+
+
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
+
+
+def log_dropped_localities(rows):
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)

data-import/httpmethod.py

@@ -23,7 +23,7 @@ CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["idempotent", "method", "safe"]
 CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
 BOOLEAN = {"yes": "true", "no": "false"}
-SELECTOR = '[]{}"|'  # mirrors inSelector in table.go
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
 EXPECTED = 9
 
 

data-import/protocol.py

@@ -21,7 +21,7 @@ OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
 CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["keyword", "name", "number"]
 SKIP = ("Reserved", "deprecated")
-SELECTOR = '[]{}"|'  # mirrors inSelector in table.go
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
 
 
 def rows(text):

data-import/source.py

@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
     if path.exists():
         return path.read_bytes()
     for attempt in range(1, 6):
-        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
+        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
         try:
             with urllib.request.urlopen(req, timeout=600) as r:
                 body = r.read()

data-import/tsv.py

@@ -5,14 +5,14 @@ from pathlib import Path
 
 
 def write(path, columns, rows):
-    """Write the rows as a TSV; the table needs two rows, and every cell must be non-empty and free of tabs, newlines and braces."""
+    """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
     lines = ["\t".join(columns)]
     for row in rows:
-        cells = [str(row[c]) for c in columns]
+        cells = [str(row[c]).strip() for c in columns]
         if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
             raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
         lines.append("\t".join(cells))
-    if len(lines) < 3:
+    if len(lines) < 2:
-        raise ValueError(f"{path}: has {len(lines) - 1} rows below its header; a table is at least two rows")
+        raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
     Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
     print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)

data.go

@@ -1,195 +1,175 @@
 package fejkdata
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
+// loadSources merges sources in order, the last winning a clash. A source whose manifest
-// label prefixes file names in errors; onDisk marks path as a directory that must
+// carries an index leaves its categories unloaded; every other source loads here, with
-// exist.
+// the unloaded categories it needs.
-type dataSource struct {
+// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
-	fsys   fs.FS
+func loadSources(sources []datafiles.Source) (folder, error) {
-	label  string
+	root := folder{children: map[string]node{}}
-	onDisk bool
-	path   string
-	root   string
-}
-
-func (s dataSource) name(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-// loadData loads every source into one namespace tree and returns its root
-// children. A directory becomes a folder; each *.json file in it compiles to a node
-// keyed by its base name (address.json -> "address"); each subdirectory becomes a
-// nested folder, so folders turn into dot-path segments. Sources merge left to right:
-// matching folders merge by their children, and any other clash is won by the last
-// source loaded. Once merged, linkRefs binds every reference against the
-// final tree.
-func loadData(sources []dataSource) (map[string]node, error) {
-	root := map[string]node{}
 	for _, src := range sources {
-		if src.onDisk {
+		m, err := src.Manifest()
-			if src.path == "" {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.path)
-			if err != nil {
-				return nil, fmt.Errorf("data path %s: %w", src.path, err)
-			}
-			if !info.IsDir() {
-				return nil, fmt.Errorf("%s is not a directory", src.path)
-			}
-		}
-		dir := src.root
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
 		if err != nil {
-			return nil, err
+			return folder{}, err
+		}
+		if err := addSource(&root, src, m); err != nil {
+			return folder{}, err
 		}
-		mergeChildren(root, g.children)
 	}
-	if len(root) == 0 {
+	return root, loadReached(&root, categorySites(&root), nil)
-		return nil, fmt.Errorf("no .json data found")
-	}
-	setTablePaths(root)
-	if err := linkTables(root); err != nil {
-		return nil, err
-	}
-	if err := linkRefs(root); err != nil {
-		return nil, err
-	}
-	if err := checkNoCycles(root); err != nil {
-		return nil, err
-	}
-	if err := checkScope(treeScope(root)); err != nil {
-		return nil, err
-	}
-	return root, nil
 }
 
-// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
+func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
-// by name, so the tree is built deterministically. Empty subdirectories (no JSON
+	src = src.Mounted(m.Namespace)
-// anywhere under them) are skipped rather than added as empty namespaces.
+	if m.Index != nil {
-func loadDir(src dataSource, dir string) (*folder, error) {
+		placeIndex(root, src, m.Index)
-	entries, err := fs.ReadDir(src.fsys, dir)
+		return nil
-	if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.name(dir), err)
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
-	for _, e := range entries {
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
-		}
-	}
-	for _, e := range entries {
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
-		}
-		full := path.Join(dir, e.Name())
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
-		}
-	}
-	return g, nil
-}
-
-// categoryFiles is what a category may name beside itself: the rows files of its
-// directory, each marked once a category names it.
-type categoryFiles struct {
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-// rows reads the rows file a category names beside it.
-func (c *categoryFiles) rows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
-	if err != nil {
 		return err
 	}
-	if len(child.children) == 0 {
+	mergeFolder(root, g)
-		return nil
-	}
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.name(full), err)
-	}
-	g.children[name] = child
 	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
+func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
-// is skipped.
+	return func(c datafiles.Category) error {
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
+		n, err := compileCategory(c)
-	if !strings.HasSuffix(file, ".json") {
+		if err != nil {
+			return err
+		}
+		place(c.Folders).put(c.Name, n)
 		return nil
 	}
-	name := strings.TrimSuffix(file, ".json")
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.name(full), err)
-	}
-	b, err := fs.ReadFile(src.fsys, full)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	g.children[name] = n
-	return nil
 }
 
-// mergeChildren overlays src onto dst. Two folders under the same key merge
+// madeFolder is the folder that dir names under root, made where it is missing or a category.
-// recursively (so locales/categories from several paths combine); every other
+func madeFolder(root *folder, dir []string) *folder {
-// key is replaced, making the last-loaded directory win on a conflict.
+	g := root
-func mergeChildren(dst, src map[string]node) {
+	for _, seg := range dir {
-	for k, v := range src {
+		sub, isFolder := g.children[seg].(*folder)
-		if dg, ok := dst[k].(*folder); ok {
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.put(seg, sub)
+		}
+		g = sub
+	}
+	return g
+}
+
+func mergeFolder(dst, src *folder) {
+	for k, v := range src.children {
+		if dg, ok := dst.children[k].(*folder); ok {
 			if sg, ok := v.(*folder); ok {
-				mergeChildren(dg.children, sg.children)
+				mergeFolder(dg, sg)
 				continue
 			}
 		}
-		dst[k] = v
+		dst.put(k, v)
 	}
 }
+
+// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
+// single inline node.
+type nodeSet func(fn func(label string, n node) error) error
+
+// categorySite is a loaded category, where it sits and its own dot path.
+type categorySite struct {
+	categoryAt
+	n    node
+	path string
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, siteIn(dir, g, name))
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func siteIn(dir []string, in *folder, name string) categorySite {
+	return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
+}
+
+func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
+
+func siteNodes(sites []categorySite) nodeSet {
+	return func(fn func(label string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func inlineNodes(n node, label string) nodeSet {
+	return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
+}
+
+// pipeline is the one load pipeline, run over one nodeSet; its fields are where
+// entry points differ.
+type pipeline struct {
+	nodes   nodeSet
+	resolve func() error
+	// Set by a caller that proves the columns against their Go types itself.
+	typedByGo bool
+}
+
+// categoryPipeline is the pipeline over the categories of one load, their references
+// resolving against root.
+func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
+	return pipeline{
+		nodes: siteNodes(sites),
+		resolve: func() error {
+			if err := linkTables(sites); err != nil {
+				return err
+			}
+			return resolveCategoryTemplates(sites, root)
+		},
+	}
+}
+
+func (p pipeline) run() error {
+	if err := p.resolve(); err != nil {
+		return err
+	}
+	if err := checkNoCycles(p.nodes); err != nil {
+		return err
+	}
+	settleRecords(p.nodes)
+	if !p.typedByGo {
+		if err := checkColumns(p.nodes); err != nil {
+			return err
+		}
+	}
+	return checkNodeFences(p.nodes)
+}
+
+// checkNodeFences runs the per-node fences over s, each over every node before the next,
+// so which of several broken nodes is reported does not depend on the walk. Its walks
+// recurse unguarded, so pipeline.run refuses cycles first.
+func checkNodeFences(s nodeSet) error {
+	mem := renderCounts{}
+	if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
+		return err
+	}
+	return s((&valueProof{}).checkDatatype)
+}

data/en_US/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"en_US.color": {"paths":[""]},
+		"en_US.company": {"paths":["","base","suffix"]},
+		"en_US.date": {"paths":[""]},
+		"en_US.email": {"paths":["","domain","local","local.n"]},
+		"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"en_US.ip": {"paths":[""]},
+		"en_US.itin": {"paths":["","area","group","serial"]},
+		"en_US.last-name": {"paths":["","count","name"]},
+		"en_US.person": {"paths":["","first","last","prefix","sex"]},
+		"en_US.phone": {"paths":["","area","exchange","line"]},
+		"en_US.phone-exchange": {"paths":[""]},
+		"en_US.price": {"paths":["","amt","cents"]},
+		"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
+		"en_US.ssn": {"paths":["","area","group","serial"]},
+		"en_US.time": {"paths":[""]},
+		"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
+		"en_US.url": {"paths":["","host","path"]},
+		"en_US.username": {"paths":["","n"]},
+		"en_US.version": {"paths":["","n","pre","suffix"]},
+		"en_US.word": {"paths":[""]}
+	},
+	"namespace": "en_US"
+}

data/en_US/address.json

@@ -1,8 +1,8 @@
 {
-  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
+  "format": "{/geo.US.address as a}{a}",
-  "street": "{/geo.US.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.US.address.street-number}",
+  "street-number": "{a.street-number}",
-  "locality": "{/geo.US.address.locality}",
+  "locality": "{a.locality}",
-  "region": "{/geo.US.address.region}",
+  "region": "{a.region}",
-  "postal-code": "{/geo.US.address.postal-code}"
+  "postal-code": "{a.postal-code}"
 }

data/en_US/fs.go

@@ -0,0 +1,19 @@
+// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
+// geo.US, which package geous holds.
+package enus
+
+import (
+	"embed"
+	"io/fs"
+
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.US.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }

data/en_US/fs_test.go

@@ -0,0 +1,22 @@
+package enus_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	enus "github.com/larvit/fejkdata/data/en_US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/en_US/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
-  "prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }],
+  "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data/en_US/phone-exchange.json

@@ -0,0 +1,17 @@
+[
+  { "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
+  { "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
+  { "format": "{int(2,9)}10", "weight": 8 },
+  { "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
+  { "format": "55{int(0,4)}", "weight": 5 },
+  { "format": "55{int(6,9)}", "weight": 4 },
+  { "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
+  { "format": "95{int(1,7)}", "weight": 7 },
+  { "format": "96{digits(1)}", "weight": 10 },
+  { "format": "97{int(0,5)}", "weight": 6 },
+  { "format": "97{int(7,9)}", "weight": 3 }
+]

data/en_US/phone.json

@@ -1,15 +1,15 @@
 [
   {
-    "format": "({area}) {exch}-{line}",
+    "format": "({area}) {exchange}-{line}",
     "weight": 2,
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   },
   {
-    "format": "{area}-{exch}-{line}",
+    "format": "{area}-{exchange}-{line}",
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   }
 ]

data/geo/SE/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
+		"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
+		"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
+		"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
+	},
+	"namespace": "geo.SE"
+}

data/geo/SE/address.json

@@ -1,6 +1,6 @@
 {
   "format": "{street} {street-number}\n{postal-code} {locality}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(1,9)}",
     "{int(10,99)}",
@@ -9,6 +9,6 @@
     { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
     { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
   ],
-  "postal-code": "{.postal-code.code}",
+  "postal-code": "{l.postal-code.code}",
-  "locality": "{.locality.name}"
+  "locality": "{l.name}"
 }

data/geo/SE/fs.go

@@ -0,0 +1,15 @@
+// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
+package geose
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/SE/fs_test.go

@@ -0,0 +1,22 @@
+package geose_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/geo/SE/locality.tsv

@@ -866,7 +866,6 @@ Nynäshamn	0192	16447	58.9034	17.9479
 Nyvång	1277	200	56.1333	12.9000
 Nyåker	2403	200	63.7500	19.3167
 Nälden	2309	986	63.3500	14.2500
-Nämdö	0120	200	59.1833	18.6833
 Nässjö	0682	18639	57.6531	14.6968
 Näsum	1272	1274	56.1667	14.4833
 Näsviken	2184	994	61.7667	16.8667

data/geo/SE/postal-code.tsv

@@ -949,7 +949,6 @@ code	locality
 130 25	Ingmarsö
 130 33	Gällnöby
 130 34	Skälvik
-130 36	Nämdö
 130 37	Stavsnäs
 130 38	Runmarö
 130 39	Sandhamn

data/geo/US/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
+		"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
+		"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
+		"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
+	},
+	"namespace": "geo.US"
+}

data/geo/US/address.json

@@ -1,13 +1,13 @@
 {
   "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 2 },
     { "format": "{int(1000,9999)}", "weight": 0.5 },
     { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
   ],
-  "locality": "{.locality.name}",
+  "locality": "{l.name}",
-  "region": "{.region.abbr}",
+  "region": "{l..municipality..region.abbr}",
-  "postal-code": "{.postal-code.code}"
+  "postal-code": "{l.postal-code.code}"
 }

data/geo/US/fs.go

@@ -0,0 +1,15 @@
+// Package geous is fejkdata's US places module, whose categories are geo.US.*.
+package geous
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/US/fs_test.go

@@ -0,0 +1,22 @@
+package geous_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/.fejkdata.json

@@ -0,0 +1,25 @@
+{
+	"index": {
+		"misc.car": {"paths":["","make","model"]},
+		"misc.coordinate": {"paths":["","lat","lon"]},
+		"misc.creditcard": {"paths":["","d"]},
+		"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
+		"misc.datetime": {"paths":[""]},
+		"misc.emoji": {"paths":[""]},
+		"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
+		"misc.httpstatus": {"paths":["","code","reason"]},
+		"misc.language": {"paths":["","code","code3","name"]},
+		"misc.loglevel": {"paths":["","code","keyword","severity"]},
+		"misc.mac": {"paths":[""]},
+		"misc.mimetype": {"paths":["","ext","type"]},
+		"misc.objectid": {"paths":[""]},
+		"misc.port": {"paths":["","number","service"]},
+		"misc.protocol": {"paths":["","keyword","name","number"]},
+		"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
+		"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
+		"misc.tld": {"paths":["","tld","type","unicode"]},
+		"misc.useragent": {"paths":["","browser","device","os","ua"]},
+		"misc.uuid": {"paths":["","variant"]}
+	},
+	"namespace": "misc"
+}

data/misc/currency.tsv

@@ -71,7 +71,7 @@ JPY	0	Yen	392	¥
 KES	2	Kenyan Shilling	404	KES
 KGS	2	Som	417	⃀
 KHR	2	Riel	116	៛
-KMF	0	Comorian Franc 	174	CF
+KMF	0	Comorian Franc	174	CF
 KPW	2	North Korean Won	408	₩
 KRW	0	Won	410	₩
 KWD	3	Kuwaiti Dinar	414	KWD

data/misc/fs.go

@@ -0,0 +1,15 @@
+// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
+package misc
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/misc/fs_test.go

@@ -0,0 +1,22 @@
+package misc_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	"github.com/larvit/fejkdata/data/misc"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/territory.tsv

@@ -48,7 +48,7 @@ CO	COL	57	Bogota	CO	COP	🇨🇴	es-CO	Colombia	170	.co
 CR	CRI	506	San Jose	CR	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
 CU	CUB	53	Havana	CU	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
 CV	CPV	238	Praia	CV	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
-CW	CUW	599	 Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
+CW	CUW	599	Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
 CX	CXR	61	Flying Fish Cove	AU	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
 CY	CYP	357	Nicosia	CY	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
 CZ	CZE	420	Prague	CZ	CZK	🇨🇿	cs,sk	Czechia	203	.cz

data/modules.go

@@ -0,0 +1,19 @@
+// Package data lists every module fejkdata ships.
+package data
+
+import (
+	"io/fs"
+
+	enus "github.com/larvit/fejkdata/data/en_US"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+	"github.com/larvit/fejkdata/data/misc"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
+
+// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS {
+	return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
+}

data/sv_SE/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
+		"sv_SE.color": {"paths":[""]},
+		"sv_SE.company": {"paths":["","base","suffix"]},
+		"sv_SE.date": {"paths":[""]},
+		"sv_SE.email": {"paths":["","domain","local","local.n"]},
+		"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"sv_SE.ip": {"paths":[""]},
+		"sv_SE.last-name": {"paths":["","count","name"]},
+		"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
+		"sv_SE.personnummer": {"paths":[""]},
+		"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
+		"sv_SE.price": {"paths":["","amt","ore"]},
+		"sv_SE.samordningsnummer": {"paths":[""]},
+		"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
+		"sv_SE.time": {"paths":[""]},
+		"sv_SE.title": {"paths":["","name","share"]},
+		"sv_SE.url": {"paths":["","host","path"]},
+		"sv_SE.username": {"paths":["","n"]},
+		"sv_SE.version": {"paths":["","n","pre","suffix"]},
+		"sv_SE.word": {"paths":[""]}
+	},
+	"namespace": "sv_SE"
+}

data/sv_SE/address.json

@@ -1,7 +1,7 @@
 {
-  "format": "{street} {street-number}\n{postal-code} {locality}",
+  "format": "{/geo.SE.address as a}{a}",
-  "street": "{/geo.SE.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.SE.address.street-number}",
+  "street-number": "{a.street-number}",
-  "postal-code": "{/geo.SE.address.postal-code}",
+  "postal-code": "{a.postal-code}",
-  "locality": "{/geo.SE.address.locality}"
+  "locality": "{a.locality}"
 }

data/sv_SE/fs.go

@@ -0,0 +1,19 @@
+// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
+// geo.SE, which package geose holds.
+package svse
+
+import (
+	"embed"
+	"io/fs"
+
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.SE.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }

data/sv_SE/fs_test.go

@@ -0,0 +1,22 @@
+package svse_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/sv_SE/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
   "prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data_test.go

@@ -1,7 +1,10 @@
 package fejkdata
 
 import (
+	"io/fs"
+	"os"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
 			regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
 	}
 
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, c := range cases {
 		for i := 0; i < 200; i++ {
 			if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
 	}
 }
 
-// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper
+// TestShippedMiscCategories covers the locale-neutral misc module: a proper
 // v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
 // whose trailing {luhn()} check passes.
 func TestShippedMiscCategories(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
 	mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
 	objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
@@ -93,7 +96,7 @@ func TestShippedMiscCategories(t *testing.T) {
 // codes follow their standard shape, dotted sub-paths resolve, emoji is a real
 // glyph, and a coordinate parses to a point within geographic bounds.
 func TestShippedMiscReferenceData(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	re := map[string]*regexp.Regexp{
 		"currency":              regexp.MustCompile(`^[A-Z]{3}$`),
 		"currency.name":         regexp.MustCompile(`\p{L}`),
@@ -156,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
 // TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
 // regexes miss: no generated name repeats a character three times over.
 func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
 		for i := 0; i < 20000; i++ {
 			name := fake(t, f, path)
@@ -175,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
 // male), the trailing digit is a Luhn checksum over the other nine, and a
 // samordningsnummer is the same with 60 added to the day.
 func TestSwedishPersonnummer(t *testing.T) {
-	sv := newGenerator(t, "data", WithSeed(1))
+	sv := shipped(t, WithSeed(1))
 	re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
 	sawLongMonthEnd, birth := false, map[string]bool{}
 	for i := 0; i < 2000; i++ {
@@ -208,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
 	if !birth["238"] || !birth["239"] {
 		t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
 	}
-	for i := 0; i < 200; i++ {
-		got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
-		sex, id, _ := strings.Cut(got, " ")
-		if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
-			t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
-		}
-	}
 }
 
 // TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
 // 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
 // in 50-65, 70-88, 90-92 or 94-99.
 func TestShippedUSTaxIds(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
 	itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
 	for i := 0; i < 2000; i++ {
@@ -245,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
 // TestShippedPersonNames pins the name tables: a sex pins its first names, a name
 // both sexes carry appears under both, and the record's sex agrees with its name.
 func TestShippedPersonNames(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(9))
+	f := shipped(t, WithSeed(9))
 	for path, want := range map[string]string{
 		"sv_SE.sex[f]":                        "kvinna",
 		"sv_SE.sex[man].code":                 "m",
@@ -273,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
 		seen := map[string]bool{}
 		for i := 0; i < 2000; i++ {
 			seen[fake(t, f, locale+".sex[f].first-name")] = true
-			first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex")
+			got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
-			if first == "" || sex == "" {
+			first, sex, _ := strings.Cut(got, "|")
-				t.Fatalf("%s.person lacks a first name or a sex", locale)
+			fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
-			}
 		}
 		if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
 			t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
 		}
-		got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`)
+	}
-		code, first, _ := strings.Cut(got, "|")
+}
-		if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
+
-			t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code)
+// TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
+// from one pick, and that two names bound to one person category are two people.
+func TestShippedPersonIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(9))
+	for _, locale := range []string{"sv_SE", "en_US"} {
+		apart := false
+		for i := 0; i < 300; i++ {
+			got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
+			parts := strings.Split(got, "|")
+			for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
+				fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
+			}
+			if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
+				fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
+			}
+			apart = apart || parts[0] != parts[3]
+		}
+		if !apart {
+			t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
 		}
 	}
 }
 
 func TestShippedSwedishPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
 	for i := 0; i < 50; i++ {
 		if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
@@ -300,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
 }
 
 func TestShippedSwedishAddress(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(3))
+	f := shipped(t, WithSeed(3))
 	digit := regexp.MustCompile(`\d`)
 	for i := 0; i < 30; i++ {
 		a := fake(t, f, "sv_SE.address")
@@ -317,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
 	}
 }
 
+// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
+// locality and region are one place in the geo tables, read through a name in a template or
+// across a struct's tags, and that a reference beside the name is another address.
+func TestShippedAddressIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(3))
+	for _, c := range []struct{ locale, country, template, largest string }{
+		{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
+		{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
+	} {
+		places := geoPlaces(t, c.country)
+		apart, localities := false, map[string]int{}
+		for i := 0; i < 2000; i++ {
+			got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
+			parts := strings.Split(got, "|")
+			if !places[strings.Join(parts[:4], "|")] {
+				t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
+			}
+			apart = apart || parts[0] != parts[4]
+			localities[parts[2]]++
+		}
+		if !apart {
+			t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
+		}
+		if c.country == "SE" {
+			var row struct {
+				Street     string `fake:"{/sv_SE.address as a}{a.street}"`
+				PostalCode string `fake:"{a.postal-code}"`
+				Locality   string `fake:"{a.locality}"`
+			}
+			for i := 0; i < 200; i++ {
+				if err := f.FakeStruct(&row); err != nil {
+					t.Fatal(err)
+				}
+				if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
+					t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
+				}
+			}
+		}
+		for l, n := range localities {
+			if n > localities[c.largest] {
+				t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
+			}
+		}
+	}
+}
+
+// 0644000 is Los Angeles.
+func TestShippedPlacesSpreadAddresses(t *testing.T) {
+	for country, city := range map[string]struct {
+		key   string
+		least int
+	}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
+		streets := 0
+		for _, r := range geoRows(t, country, "street") {
+			if r[1] == city.key {
+				streets++
+			}
+		}
+		if streets < city.least {
+			t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
+		}
+	}
+	header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
+	population := slices.Index(strings.Split(header, "\t"), "population")
+	if population < 0 {
+		t.Fatal("geo.US locality.tsv has no population column")
+	}
+	for _, r := range geoRows(t, "US", "locality") {
+		n, err := strconv.Atoi(r[population])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n < 2500 {
+			return
+		}
+	}
+	t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
+}
+
+// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
+// draw by population leaves out.
+func TestShippedCapitalsAndBigPlaces(t *testing.T) {
+	state := map[string]string{}
+	for _, r := range geoRows(t, "US", "municipality") {
+		state[r[0]] = r[2]
+	}
+	shipped := map[string]bool{}
+	for _, r := range geoRows(t, "US", "locality") {
+		shipped[r[1]+", "+state[r[2]]] = true
+	}
+	for _, place := range []string{
+		"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
+		"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
+		"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
+		"Ventura, CA",
+	} {
+		if !shipped[place] {
+			t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
+		}
+	}
+}
+
+func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
+	refused := map[string]bool{
+		"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
+		"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
+		"Öresundsvägen, Bondevägen": true,
+	}
+	for _, r := range geoRows(t, "SE", "street") {
+		if refused[r[0]] {
+			t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
+		}
+	}
+}
+
+func geoTSV(t *testing.T, country, table string) string {
+	b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func geoRows(t *testing.T, country, table string) [][]string {
+	var out [][]string
+	for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
+		out = append(out, strings.Split(line, "\t"))
+	}
+	return out
+}
+
+// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
+// one locality holds, the region empty for SE, whose address carries none.
+func geoPlaces(t *testing.T, country string) map[string]bool {
+	rows := func(table string) [][]string { return geoRows(t, country, table) }
+	name, region := map[string]string{}, map[string]string{}
+	if country == "US" {
+		county := map[string]string{}
+		for _, r := range rows("municipality") {
+			county[r[0]] = r[2]
+		}
+		for _, r := range rows("locality") {
+			name[r[0]], region[r[0]] = r[1], county[r[2]]
+		}
+	} else {
+		for _, r := range rows("locality") {
+			name[r[0]] = r[0]
+		}
+	}
+	codes := map[string][]string{}
+	for _, r := range rows("postal-code") {
+		codes[r[1]] = append(codes[r[1]], r[0])
+	}
+	places := map[string]bool{}
+	for _, r := range rows("street") {
+		for _, code := range codes[r[1]] {
+			places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
+		}
+	}
+	return places
+}
+
 func TestShippedPersonHasParts(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(7))
+	f := shipped(t, WithSeed(7))
 	for _, path := range []string{"sv_SE.person", "en_US.person"} {
 		for i := 0; i < 30; i++ {
 			if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
@@ -328,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
 	}
 }
 
+// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
 func TestShippedUSPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
-	re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`)
+	re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
-	for i := 0; i < 50; i++ {
+	unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
-		if n := fake(t, f, "en_US.phone"); !re.MatchString(n) {
+	for i := 0; i < 2000; i++ {
+		n := fake(t, f, "en_US.phone")
+		m := re.FindStringSubmatch(n)
+		if m == nil {
 			t.Fatalf("phone %q does not match %s", n, re)
 		}
+		if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
+			t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
+		}
 	}
 }
 
-// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and
+func TestShippedTableCellsAreTrimmed(t *testing.T) {
-// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person.
+	err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil || !strings.HasSuffix(path, ".tsv") {
+			return err
+		}
+		b, err := fs.ReadFile(os.DirFS("data"), path)
+		if err != nil {
+			return err
+		}
+		for i, line := range strings.Split(string(b), "\n") {
+			for _, cell := range strings.Split(line, "\t") {
+				if cell != strings.TrimSpace(cell) {
+					t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
+// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
 func TestShippedNamespacedTree(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
 		if got := fake(t, f, path); got == "" {
 			t.Fatalf("Fake(%q) returned empty", path)
@@ -385,7 +590,7 @@ func luhnValid(s string) bool {
 var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
 
 func TestShippedStreetIsARegisteredName(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
@@ -397,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
 func TestShippedLastNameComposition(t *testing.T) {
 	// last is a choice of patronymic {first}sson templates, compound
 	// {first}{last} templates and literal surnames.
-	f := newGenerator(t, "data", WithSeed(6))
+	f := shipped(t, WithSeed(6))
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
 			t.Fatalf("last name %q is not a Swedish surname", s)
@@ -407,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
 
 func TestShippedStreetNumberFormats(t *testing.T) {
 	// Reachable via a hyphenated path; covers all five weighted number variants.
-	f := newGenerator(t, "data", WithSeed(8))
+	f := shipped(t, WithSeed(8))
 	re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
 	for i := 0; i < 300; i++ {
 		if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
@@ -419,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
 // TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
 // that the everyday titles are reachable.
 func TestShippedUSTitleAgreesWithSex(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(4))
+	f := shipped(t, WithSeed(4))
 	female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
 	seen := map[string]bool{}
 	for i := 0; i < 3000; i++ {
-		got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`)
+		got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
 		title, sex, _ := strings.Cut(got, "|")
 		title = strings.TrimSpace(title)
 		seen[title] = true

datatype.go

@@ -3,97 +3,83 @@ package fejkdata
 import (
 	"errors"
 	"fmt"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
-// DataType is what a record column holds, which decides how a record writes its value.
+// DataType is what a record column holds, which decides how a record writes its value. It
-type DataType int
+// prints as data spells it: string, integer, number or boolean.
+type DataType = datatype.DataType
 
 // The datatypes a column declares with "datatype"; a column without one is a string.
 const (
-	DataTypeString DataType = iota
+	DataTypeString  DataType = datatype.String
-	DataTypeInteger
+	DataTypeInteger DataType = datatype.Integer
-	DataTypeNumber
+	DataTypeNumber  DataType = datatype.Number
-	DataTypeBoolean
+	DataTypeBoolean DataType = datatype.Boolean
 )
 
-var (
+// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
-	dataTypeNames = [...]string{"string", "integer", "number", "boolean"}
+func datatypeOf(m map[string]any, pos position) (*DataType, error) {
-	dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
-)
-
-// String is the datatype as data spells it.
-func (d DataType) String() string {
-	if d < 0 || int(d) >= len(dataTypeNames) {
-		return fmt.Sprintf("DataType(%d)", int(d))
-	}
-	return dataTypeNames[d]
-}
-
-// position is where a JSON value sits, which decides whether it may carry a datatype or
-// be null.
-type position int
-
-const (
-	inFormat position = iota // rendered by a format, so neither
-	atTop                    // a category or an inline template, whose fields may be columns
-	inColumn                 // a column, or a choice item standing in for one
-)
-
-// datatypeOf reads a template's "datatype" (default DataTypeString).
-func datatypeOf(m map[string]any, pos position) (DataType, error) {
 	v, ok := m["datatype"]
 	if !ok {
-		return DataTypeString, nil
+		return nil, nil
 	}
 	name, ok := v.(string)
 	if !ok {
-		return 0, fmt.Errorf("datatype must be a string, got %T", v)
+		return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
 	}
-	if name == DataTypeString.String() {
+	for d := DataTypeString; d < datatype.Count; d++ {
-		return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
-	}
-	for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
 		if name != d.String() {
 			continue
 		}
-		if pos != inColumn {
+		if !pos.column() {
-			return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here")
+			return nil, nil
 		}
-		return d, nil
+		return &d, nil
 	}
-	return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name)
+	return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
 }
 
 // checkColumns rejects a record column whose items hold different datatypes, checking a column
 // after the columns its items read, so a column read is named before its readers.
-func checkColumns(s nodeScope) error {
+func checkColumns(s nodeSet) error {
 	checked := map[node]bool{}
-	var check func(path, name string, column node) error
+	var check func(label, name string, column node) error
-	check = func(path, name string, column node) error {
+	check = func(label, name string, column node) error {
 		if checked[column] {
 			return nil
 		}
 		checked[column] = true
 		items, _ := columnItems(column)
 		for _, it := range items {
-			if r := it.readsColumn; r != nil {
+			if r := it.readsColumn; r != nil && r.category != "" {
-				if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil {
+				if err := check(r.category, r.field, r.column); err != nil {
 					return err
 				}
 			}
 		}
-		if _, err := columnDatatype(column); err != nil {
+		if len(items) == 0 {
-			return fmt.Errorf("%s: field %q: %w", path, name, err)
+			return nil
+		}
+		first := columnDatatype(column)
+		for _, it := range items[1:] {
+			if d := itemDatatype(it); d != first {
+				return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
+			}
 		}
 		return nil
 	}
-	return s(func(path string, n node) error {
+	return s(func(label string, n node) error {
 		t, ok := n.(*template)
-		if !ok || !t.record {
+		if !ok || !t.isRecord {
 			return nil
 		}
-		for _, name := range recordColumns(t) {
+		for _, name := range sortedNames(t.fields) {
-			if err := check(path, name, t.fields[name]); err != nil {
+			if err := check(label, name, t.fields[name]); err != nil {
 				return err
 			}
 		}
@@ -101,41 +87,25 @@ func checkColumns(s nodeScope) error {
 	})
 }
 
-// columnDatatype is the datatype a column's items hold. They must agree, since a
+// columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
-// column holds one; a column only ever null is a string.
+// a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
-func columnDatatype(n node) (DataType, error) {
+func columnDatatype(n node) DataType {
 	items, _ := columnItems(n)
 	if len(items) == 0 {
-		return DataTypeString, nil
+		return DataTypeString
 	}
-	first := itemDatatype(items[0])
+	return itemDatatype(items[0])
-	for _, t := range items[1:] {
-		if d := itemDatatype(t); d != first {
-			return first, disagreement(items[0], first, t, d)
-		}
-	}
-	return first, nil
 }
 
-// itemDatatype is the datatype a column item declares, else that of the column it is.
+// itemDatatype is the datatype a column item declares, else that of the column it reads.
 func itemDatatype(t *template) DataType {
-	if t.datatype != DataTypeString {
+	switch {
-		return t.datatype
+	case t.datatype != nil:
+		return *t.datatype
+	case t.readsColumn != nil:
+		return columnDatatype(t.readsColumn.column)
 	}
-	return readDatatype(t)
+	return DataTypeString
-}
-
-// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
-// when t reads none.
-func readDatatype(t *template) DataType {
-	if t.readsColumn == nil {
-		return DataTypeString
-	}
-	items, _ := columnItems(t.readsColumn.column)
-	if len(items) == 0 {
-		return DataTypeString
-	}
-	return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
 }
 
 // columnItems is a column's template items, its choices unwrapped, and whether one is null.
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
 			}
 		case *template:
 			items = append(items, n)
-		case *null:
+		case *nullItem:
 			nullable = true
+		case *tableColumn:
+		default:
+			panic(invariant.Broken("columnItems has no case for node %T", n))
 		}
 	}
 	collect(n)
 	return items, nullable
 }
 
-// disagreement names the fix for two items of one column holding different datatypes.
+// itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
-func disagreement(a *template, da DataType, b *template, db DataType) error {
+type itemKind int
-	bare, typed, want := b, a, da
+
-	if da == DataTypeString {
+const (
-		bare, typed, want = a, b, db
+	kindText itemKind = iota
-	}
+	kindReads
+	kindDeclares
+)
+
+func kindOf(t *template) itemKind {
 	switch {
-	case da != DataTypeString && db != DataTypeString:
+	case t.datatype != nil:
-		return bothTyped(a, da, b, db)
+		return kindDeclares
-	case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "":
+	case itemDatatype(t) != DataTypeString:
-		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed))
+		return kindReads
-	case bare.fromString: // an object may carry a weight, which this spelling would drop
-		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
 	}
-	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)
+	return kindText
 }
 
-// bothTyped names the fix for two items holding different datatypes: the one taking its datatype
+// disagreement names the fix for items a and b of one column holding different datatypes, fixing
-// from the column it reads is typed as the other where its values prove it, else read as text.
+// the one that declares least, a when both declare alike.
-func bothTyped(a *template, da DataType, b *template, db DataType) error {
+func disagreement(a *template, da DataType, b *template, db DataType) error {
-	read, other := a, db
+	c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
-	if a.datatype != DataTypeString {
+	if kindOf(b) < kindOf(a) {
-		read, other = b, da
+		c.fix, c.other, c.want = b, a, da
 	}
-	switch {
+	message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
-	case read.datatype != DataTypeString:
+	if !ok {
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
+		panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
-	case (&valueProof{}).columnItem(read).not[other] == "":
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
 	}
-	return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read))
+	return message(c)
+}
+
+// clash is two items of one column holding different datatypes: other holds want; held says
+// what each holds.
+type clash struct {
+	fix, other *template
+	want       DataType
+	held       string
+}
+
+// disagreements is a clash's message by the kinds of its items, fix's first.
+var disagreements = map[[2]itemKind]func(clash) error{
+	{kindDeclares, kindDeclares}: clash.bothDeclare,
+	{kindReads, kindReads}:       clash.retypeRead,
+	{kindReads, kindDeclares}:    clash.retypeRead,
+	{kindText, kindReads}:        clash.textBesideRead,
+	{kindText, kindDeclares}:     clash.declareText,
+}
+
+func (c clash) bothDeclare() error { return errors.New(c.held) }
+
+// fits reports whether fix may hold want.
+func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
+
+func (c clash) retypeRead() error {
+	if c.fits() {
+		return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
+	}
+	return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
+}
+
+func (c clash) textBesideRead() error {
+	if !c.fits() {
+		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
+	}
+	return c.declareText()
+}
+
+func (c clash) declareText() error {
+	if c.fix.fromString { // an object may carry a weight, which this spelling would drop
+		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
+	}
+	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
 }
 
 // typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
 	return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
 }
 
-// asText names the spelling that reads the column a column-read item reads as text, keeping the
+// columnRead is a record's column read by a format that only reads one reference or name,
-// other keys an object item carries.
+// which is the column: it takes the column's datatype and null. category and field name it;
-func asText(t *template) string {
+// category is "" for a column of the reading template's own record, which checkColumns reaches
-	if t.fromString {
+// on its own.
-		return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format)
+type columnRead struct {
-	}
+	a               arm
-	return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format)
+	category, field string
+	column          node
+}
+
+// columnReadOf is the record's column t's format reads, where the format only reads one reference
+// or name, and nil where it does not or t declares a string, which reads the column as text.
+func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
+	ops := t.compiled.ops
+	if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
+		return nil
+	}
+	a := ops[0].arms[0]
+	var start node
+	var category string
+	var tail []string
+	switch b := a.named; {
+	case a.kind == namedRead && b.bindsField():
+		// {f as n}{n}: column f of the record binding n, in t's own category.
+		start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
+	case a.kind == namedRead:
+		// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
+		start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
+		tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
+	case grammar.IsRef(a.head):
+		// {/c.x}: column x of category c.
+		start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
+	default:
+		return nil
+	}
+	if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
+		return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
+	}
+	return nil
 }

datatype_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 )
 
-func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
+func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
 	for _, src := range []string{
 		`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
 		`{"format":"","n":{"format":"42","datatype":"integer"}}`,
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 		`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
 		`{"format":"","pick":[[null,"a"],"b"]}`,
 	} {
-		if _, err := compile(parse(t, src)); err != nil {
+		if _, err := resolved(t, src); err != nil {
 			t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
 		}
 	}
 	for src, want := range map[string]string{
-		`{"format":"","n":{"format":"1","datatype":"int"}}`:                             `datatype takes "integer", "number" or "boolean", got "int"`,
+		`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
-		`{"format":"","n":{"format":"1","datatype":1}}`:                                 "datatype must be a string",
+		`{"format":"","n":{"format":"1","datatype":1}}`:     "datatype must be a string, not a number",
-		`{"format":"{int(1,9)}","datatype":"integer"}`:                                  "datatype only types a record column",
-		`[{"format":"1","datatype":"integer"},"x"]`:                                     "datatype only types a record column",
-		`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
-		`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`:           "datatype only types a record column",
-		`null`: `so write ""`,
-		`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
 		}
 	}
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 	tree := map[string]string{
 		"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
-		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
 	}
 	for _, c := range []struct{ name, column, want string }{
 		{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
 		{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
-		{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
 		{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
 		{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
-		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
-		{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
 		{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
-		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
 		{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
-		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`},
+		{"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
+		{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
+		{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
+		{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
 		{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
 		{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
 		{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
 		{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
 		{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
 		{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
-		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"},
+		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
 		{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
 		{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
 		{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
-		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"},
+		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
 		{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
 		{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
-		{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
-		{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
 	} {
 		row := `{"format":"","col":` + c.column + `}`
 		files := map[string]string{"row": row}
 		for name, body := range tree {
 			files[name] = body
 		}
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
+		_, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
 		if err == nil || !strings.Contains(err.Error(), c.want) {
 			t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
 		}
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 			t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
 		}
 	}
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
+	for column, want := range map[string]string{
+		`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
+		`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`:   "its items hold integer and number; a column holds one datatype",
+	} {
+		row := `{"format":"","col":` + column + `}`
+		if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
+		}
+		if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
+		}
+	}
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
 		"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
 		"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
 	})))
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
 	} {
 		row := `{"format":"","col":` + column + `}`
 		src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
-		f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
+		f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
 		if err != nil {
 			t.Errorf("%s: New = %v, want it loaded", column, err)
 			continue
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
 		t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
 	}
 }
+
+func TestDisagreementRefusesTwoTextItems(t *testing.T) {
+	mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
+}
+
+func TestASignedZeroIsZero(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
+	r, err := f.FakeRecord("row")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
+		t.Errorf("columns = %+v, want each written as its format", c)
+	}
+}
+
+func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{
+		"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
+	}), WithSeed(1))
+	empty := 0
+	for i := 0; i < 50; i++ {
+		r, err := f.FakeRecord("row")
+		if err != nil {
+			t.Fatal(err)
+		}
+		same, text := r.Columns()[0], r.Columns()[1]
+		if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
+			t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
+		}
+		if text.Value == "" {
+			empty++
+		}
+	}
+	if empty == 0 {
+		t.Error(`50 records never read a null as "", want the string column to render one so`)
+	}
+}

doc.go

@@ -0,0 +1,46 @@
+// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
+//
+// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
+// or a folder. A generator loads the modules you name, such as data.Modules(), every
+// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
+// become a dot-path namespace, and you generate values by path:
+//
+//	f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
+//	f.Fake("sv_SE.address")          // "Trekungagatan 43\n442 42 Kungälv"
+//	f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
+//
+// Several sources merge in order, the last winning a name clash, so custom data
+// layers over the built-ins. The JSON template format is documented in the [README].
+//
+// [README]: https://github.com/larvit/fejkdata#readme
+package fejkdata
+
+// Vocabulary
+//
+//   - draw — one item taken from a choice, or one row from a table: `drawItem`,
+//     `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
+//   - expansion — one render of one format: `expand`.
+//   - render — one walk of the tree from an entry point, over one `frameStack`:
+//     `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
+//     start one each, and `FakeStruct` one per record.
+//   - env — what a render reads names and rows through: `renderEnv`.
+//   - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
+//   - named pick — the draw a token such as {/person as p} binds to a name, drawn
+//     on its first read and kept while its scope renders: `nameBinding`,
+//     `namedPick`, `pickFrame`.
+//   - level, pick key, addressed key — `pickKey`.
+//   - memo — what a named pick keeps its draws in, the variant drawn at each level
+//     and the value each path read: `drawMemo`.
+//   - pin — fixing which row of a table one path, or one named pick, uses:
+//     `pinSet`, `namedPick.pins`, `table.followStep`.
+//   - link — a table's tie to its parent table, each row to its parent row:
+//     `table.linkParent`.
+//   - family — a table and every table reaching it through a chain of parent
+//     links.
+//   - resolve — tying a reference or a name to what it names in the assembled tree:
+//     `resolveTemplates`, `resolveRefs`.
+//   - head — the string a read starts from: a field or a name as written, or a
+//     reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
+//     the read's start: `template.startOf`, `nameTarget.start`.
+//   - fence — a load-time check, run over each category, template or struct as it
+//     loads or compiles, so rendering a compiled tree cannot fail.

doc_test.go

@@ -0,0 +1,286 @@
+package fejkdata
+
+import (
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"os"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Every backtick in the Vocabulary is a symbol.
+var backticked = regexp.MustCompile("`([^`\n]+)`")
+
+// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
+func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
+	_, files := sourceFiles(t)
+	declared := declaredSymbols(files)
+	if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
+		t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
+	}
+}
+
+// namesDeclared is every symbol text backticks, failing for each one the package
+// declares nowhere.
+func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
+	t.Helper()
+	var named []string
+	for _, m := range backticked.FindAllStringSubmatch(text, -1) {
+		if !declared[m[1]] {
+			t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
+		}
+		named = append(named, m[1])
+	}
+	return named
+}
+
+// namesAPass fails unless one symbol text backticks is a function: a field names
+// what a pass reads or writes, never the pass.
+func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
+	t.Helper()
+	named := namesDeclared(t, where, text, declared)
+	if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
+		t.Errorf("%s names no function, so it names no pass", where)
+	}
+}
+
+func packageDir(t *testing.T) *build.Package {
+	t.Helper()
+	pkg, err := build.ImportDir(".", 0)
+	if err != nil {
+		t.Fatalf("read the package directory: %v", err)
+	}
+	return pkg
+}
+
+func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	return parseFiles(t, packageDir(t).GoFiles)
+}
+
+func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	fset := token.NewFileSet()
+	var files []*ast.File
+	for _, name := range names {
+		f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
+		if err != nil {
+			t.Fatalf("parse %s: %v", name, err)
+		}
+		files = append(files, f)
+	}
+	return fset, files
+}
+
+func vocabulary(t *testing.T, files []*ast.File) string {
+	t.Helper()
+	var sections strings.Builder
+	for _, f := range files {
+		for _, c := range f.Comments {
+			section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
+			if found && c.Pos() > f.Package {
+				sections.WriteString(section)
+			}
+		}
+	}
+	if sections.Len() == 0 {
+		t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
+	}
+	return sections.String()
+}
+
+// declaredSymbols is every name the package declares at the top level, a method and
+// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
+func declaredSymbols(files []*ast.File) map[string]bool {
+	names := declaredFuncs(files)
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			if d, isGen := decl.(*ast.GenDecl); isGen {
+				addSpecs(names, d)
+			}
+		}
+	}
+	return names
+}
+
+// declaredFuncs is every function the package declares, a method keyed under its
+// type: template.fixedText.
+func declaredFuncs(files []*ast.File) map[string]bool {
+	names := map[string]bool{}
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isFunc := decl.(*ast.FuncDecl)
+			if !isFunc {
+				continue
+			}
+			names[d.Name.Name] = true
+			if recv := receiverType(d.Recv); recv != "" {
+				names[recv+"."+d.Name.Name] = true
+			}
+		}
+	}
+	return names
+}
+
+func addSpecs(names map[string]bool, d *ast.GenDecl) {
+	for _, spec := range d.Specs {
+		switch s := spec.(type) {
+		case *ast.TypeSpec:
+			names[s.Name.Name] = true
+			addFields(names, s)
+		case *ast.ValueSpec:
+			for _, id := range s.Names {
+				names[id.Name] = true
+			}
+		}
+	}
+}
+
+func addFields(names map[string]bool, s *ast.TypeSpec) {
+	st, isStruct := s.Type.(*ast.StructType)
+	if !isStruct {
+		return
+	}
+	for _, field := range st.Fields.List {
+		for _, id := range field.Names {
+			names[s.Name.Name+"."+id.Name] = true
+		}
+	}
+}
+
+func receiverType(recv *ast.FieldList) string {
+	if recv == nil || len(recv.List) == 0 {
+		return ""
+	}
+	e := recv.List[0].Type
+	if star, isPointer := e.(*ast.StarExpr); isPointer {
+		e = star.X
+	}
+	id, isIdent := e.(*ast.Ident)
+	if !isIdent {
+		return ""
+	}
+	return id.Name
+}
+
+const fillHeader = "Filled by "
+
+// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
+// fills, and to the template doc.
+func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
+	fset, files := sourceFiles(t)
+	declared, funcs := declaredSymbols(files), declaredFuncs(files)
+	doc, st, headers := templateStruct(t, fset, files)
+	namesAPass(t, "the template doc", doc, declared, funcs)
+	for _, field := range st.Fields.List {
+		if len(field.Names) == 0 {
+			t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
+		}
+		for _, id := range field.Names {
+			header := headerAbove(headers, id.Pos())
+			if header == "" {
+				t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
+				continue
+			}
+			namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
+		}
+	}
+}
+
+// templateStruct is the template struct, its doc, and the fill headers standing
+// between its fields, in source order.
+func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
+	t.Helper()
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isGen := decl.(*ast.GenDecl)
+			if !isGen {
+				continue
+			}
+			for _, spec := range d.Specs {
+				s, isType := spec.(*ast.TypeSpec)
+				if !isType || s.Name.Name != "template" {
+					continue
+				}
+				st, isStruct := s.Type.(*ast.StructType)
+				if !isStruct {
+					t.Fatal("template is not a struct")
+				}
+				return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
+			}
+		}
+	}
+	t.Fatal("the package declares no template struct")
+	return "", nil, nil
+}
+
+// fillHeaders skips a header trailing a field: it names that field, never the
+// fields below it.
+func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
+	trailing := map[int]bool{}
+	for _, field := range st.Fields.List {
+		trailing[fset.Position(field.End()).Line] = true
+	}
+	var found []*ast.CommentGroup
+	for _, c := range f.Comments {
+		if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
+			continue
+		}
+		if strings.HasPrefix(c.Text(), fillHeader) {
+			found = append(found, c)
+		}
+	}
+	return found
+}
+
+// headerAbove is the last fill header standing above pos.
+func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
+	text := ""
+	for _, c := range headers {
+		if c.Pos() < pos {
+			text = c.Text()
+		}
+	}
+	return text
+}
+
+var (
+	printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
+	printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
+)
+
+// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
+func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
+	for _, name := range []string{"doc.go", "README.md"} {
+		src, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines := printedCall.FindAllString(string(src), -1)
+		if lines == nil {
+			t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
+		}
+		f, err := New(withShipped(), WithSeed(42))
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, line := range lines {
+			c := printedFake.FindStringSubmatch(line)
+			if c == nil {
+				t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
+			}
+			want, err := strconv.Unquote(c[2])
+			if err != nil {
+				t.Fatalf("%s prints %s: %v", name, c[2], err)
+			}
+			if got := fake(t, f, c[1]); got != want {
+				t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
+			}
+		}
+	}
+}

docs/comprehension-history.md

@@ -0,0 +1,722 @@
+# Comprehension history
+
+## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
+- `datatype.go:217` `columnRead`: beside the code that reads it
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
+- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
+- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
+
+## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
+- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
+- `node.go:70` `template.readsColumn`: names the four readers that change together
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
+- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
+
+## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
+- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
+- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
+- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
+- `path.go:119` `compileStep`: six parameters, the one place that reads harder
+
+## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
+- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
+- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
+- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
+- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
+
+## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
+- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
+- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
+- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
+- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
+
+## 2026-10-07, scoring run at ad59967
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
+
+## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
+| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
+| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
+| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
+
+Junior A, hardest first:
+
+- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
+- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
+
+Mid A, hardest first:
+
+- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
+- `node.go:70` `readsColumn`: four functions in three files must change together
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
+- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
+
+Inherited architect, would restructure first:
+
+- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
+- `node.go:52` `template`: filled in four phases by four functions
+
+## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
+- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
+- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
+
+Inherited architect, decided by:
+
+- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
+- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
+- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
+
+## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
+- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
+
+Inherited architect, decided by:
+
+- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
+- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
+- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
+- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
+
+## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
+- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
+- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
+- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
+- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
+- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
+
+## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
+- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
+- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
+- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
+- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
+- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
+- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
+
+## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
+- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
+- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
+- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
+
+Inherited architect, decided by:
+
+- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
+- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
+- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
+
+## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | better | same | better | better |
+| Junior A | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
+- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
+- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
+- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
+- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
+
+Junior A, decided by:
+
+- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
+- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
+- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
+
+## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
+- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
+- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
+- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
+- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
+- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
+- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
+- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
+- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
+- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
+
+## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | worse | worse | better | same | worse |
+| Maintainability senior | worse | same | worse | same | worse |
+| Junior A | same | worse | same | same | same |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
+- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
+- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
+- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
+- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
+- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
+- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
+- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
+
+Maintainability senior, decided by:
+
+- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
+- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
+- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
+- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
+
+Junior A, decided by:
+
+- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
+- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
+
+## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | worse | same | worse |
+| Inherited architect | worse | same | worse | same | worse |
+
+Mid A, decided by:
+
+- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
+- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
+- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
+- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
+
+Inherited architect, decided by:
+
+- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
+- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
+- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
+- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
+- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
+
+## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | worse | worse | same | worse |
+| Inherited architect | worse | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
+- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
+- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
+- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
+- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
+
+## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
+
+Ruling: worse
+
+Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | better | same | same | same |
+| Inherited architect | worse | same | same | same | same |
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6 | 6 | 6 |
+| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
+| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
+- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
+- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
+- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
+
+Inherited architect, decided by:
+
+- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
+- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
+- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
+
+## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | better | better | same | better | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
+- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
+
+Inherited architect, decided by:
+
+- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
+- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
+- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
+- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
+
+## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | same | worse | worse | same | worse |
+| Maintainability senior | same | worse | same | same | worse |
+| Junior A | same | worse | worse | better | worse |
+
+Mid A, decided by:
+
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
+- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
+- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
+- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
+- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
+- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
+- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
+- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
+- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
+
+Junior A, decided by:
+
+- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
+- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
+
+## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
+- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
+- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
+- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
+- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
+- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
+- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
+- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
+
+## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
+- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
+- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
+- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
+- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
+- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
+
+## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | better | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
+- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
+- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
+- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
+- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
+- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
+- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
+
+Inherited architect, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
+- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
+- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
+
+## 2026-10-12T16:30:06Z, PR #216 at 52e0304
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
+| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
+| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
+
+Junior A, hardest first:
+
+- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
+- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
+- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
+- `node.go:65` `template`: filled in four phases by different files, with a change-together set
+- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
+
+Mid A, hardest first:
+
+- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
+- `render.go:146` `readField`: four read paths, and the name undersells it
+- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
+- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
+- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
+- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
+- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
+- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
+- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
+
+Inherited architect, would restructure first:
+
+- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
+- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
+- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
+- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
+- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
+
+## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | better | worse | same | same | better |
+| Inherited architect | better | worse | same | same | same |
+| Maintainability senior | better | worse | same | same | same |
+| Junior A | better | worse | same | better | better |
+
+Mid A, decided by:
+
+- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
+- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
+- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
+
+Inherited architect, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
+- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
+- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
+- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
+
+Maintainability senior, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
+- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
+- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
+
+Junior A, decided by:
+
+- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
+- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
+- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
+- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
+
+## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-14T11:34:34Z, PR #225 at f734c9e, against f90b4d6
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | same |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | worse | same | worse | worse |
+| Junior A | same | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality: `Dockerfile:26` `vulnerability-filing`: the filer's test reads the real todo.md inside docker build ., so a plan edit can turn the Go gate red
+- Locality: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the filer splices todo.md as text, assuming its row shape, the Next ID marker and the Details heading
+- Locality: `.github/workflows/test.yml:106` `release`: a release now also waits on vuln.go.dev's answer, visible only in a needs list
+
+Inherited architect, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the plan is copied into the merge-gate build, so a planning edit that changes the table's shape fails docker build .
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: edits todo.md as text; a missing Next ID crashes with exit 1, which reads like a vulnerability already planned
+- Locality: `.github/workflows/test.yml:69` `vulnerabilities`: whether a release happens depends on vuln.go.dev that day, and every PR shows a red job while one is known
+
+Maintainability senior, decided by:
+
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: finds the release by an unnamed regex group, never checks the Next ID match, and splices Details by a replace that does nothing when the heading is missing
+- Locality, Self-sufficiency: `.github/workflows/test.yml:83` `File each one in todo.md`: one exit code holds the release and triggers the proposal, which must tell filed, planned and crashed apart by a git diff
+- Locality: `Dockerfile:26` `vulnerability-filing`: copies todo.md into the gate build, and the path filter lists todo.md
+- Self-sufficiency: `README.md:1213` "Development": says the build runs the release tooling's tests and leaves out the filer's
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:2` module docstring: builds on rank_todo.py, which the repository does not hold
+
+Junior A, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the Go merge gate copies todo.md and files an item into it, so editing the plan can turn the build red
+- Locality: `.github/workflows/test.yml:106` `release`: whether a push to main releases also depends on vuln.go.dev's answer that day
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the regex and the Details splice encode todo.md's layout, which the script never shows
+- Self-sufficiency: `README.md:1213` "Development": the gate paragraph leaves out the filer's tests while AGENTS.md now defers to it

docs/reading-cost.md

@@ -0,0 +1,637 @@
+# Reading cost
+
+## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
+
+68314 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `compose.yaml` 3
+- `data-import/territory.py` 2
+- `data_test.go` 2
+- `shipped_data_test.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `data-import/tsv.py` 1
+- `data/en_US/phone.json` 1
+
+## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
+
+118930 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 8
+- `README.md` 6
+- `table_test.go` 4
+- `render.go` 3
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `fejkdata.go` 1
+
+## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
+
+93128 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `data_test.go` 4
+- `todo.md` 3
+- `format_stability_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data/en_US/email.json` 1
+- `data/en_US/first-name.tsv` 1
+- `data/en_US/sex.tsv` 1
+- `data/en_US/title.tsv` 1
+
+## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
+
+79628 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `todo.md` 5
+- `format.go` 4
+- `valueproof.go` 4
+- `calc_test.go` 3
+- `datatype_test.go` 2
+- `name.go` 2
+- `name_test.go` 2
+- `record_test.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
+
+366332 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `table_test.go` 17
+- `table.go` 9
+- `struct.go` 8
+- `README.md` 7
+- `docs/decisions.md` 7
+- `record.go` 5
+- `calc.go` 4
+- `data_test.go` 4
+- `reference.go` 4
+- `render.go` 4
+
+## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
+
+705270 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 16
+- `todo.md` 16
+- `README.md` 10
+- `render.go` 5
+- `table.go` 5
+- `format.go` 4
+- `AGENTS.md` 3
+- `calc.go` 3
+- `data.go` 3
+- `fejkdata.go` 3
+
+## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
+
+59846 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `CHANGELOG.md` 2
+- `README.md` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `fejkdata_test.go` 2
+- `go.mod` 2
+- `.dockerignore` 1
+- `builtins_test.go` 1
+- `cmd/fejkdata/main.go` 1
+
+## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
+
+98933 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `format_test.go` 5
+- `todo.md` 5
+- `calc.go` 4
+- `calc_test.go` 4
+- `format.go` 4
+- `path.go` 3
+- `reference.go` 3
+- `README.md` 2
+- `builtins_test.go` 2
+- `inline.go` 2
+
+## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
+
+129910 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `builtins_test.go` 7
+- `todo.md` 5
+- `format.go` 4
+- `calc_test.go` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `reference.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `calc.go` 1
+
+## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
+
+91864 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `table_test.go` 4
+- `data.go` 3
+- `todo.md` 3
+- `graph.go` 2
+- `node.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+
+## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
+
+98023 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `path_test.go` 4
+- `README.md` 3
+- `docs/decisions.md` 3
+- `imports_test.go` 2
+- `path.go` 2
+- `reference_test.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
+
+115125 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `name_test.go` 5
+- `todo.md` 5
+- `shape_test.go` 3
+- `README.md` 2
+- `graph.go` 2
+- `inline.go` 2
+- `node.go` 2
+- `record_test.go` 2
+- `struct.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
+
+53085 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `AGENTS.md` 3
+- `README.md` 3
+- `todo.md` 3
+- `.gitignore` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
+
+67696 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 3
+- `pick_test.go` 2
+- `render.go` 2
+- `table_test.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `data/geo/SE/address.json` 1
+- `data/sv_SE/address.json` 1
+- `docs/decisions.md` 1
+- `pick.go` 1
+
+## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
+
+75959 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 5
+- `README.md` 2
+- `inline.go` 2
+- `namefence.go` 2
+- `path.go` 2
+- `pick_test.go` 2
+- `record.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
+
+129293 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `README.md` 6
+- `format.go` 6
+- `docs/decisions.md` 5
+- `record.go` 5
+- `inline.go` 4
+- `render.go` 4
+- `shipped.go` 4
+- `CHANGELOG.md` 3
+- `data.go` 3
+
+## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
+
+79101 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `format.go` 5
+- `todo.md` 5
+- `path.go` 3
+- `docs/decisions.md` 2
+- `env.go` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
+
+2986167 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 24
+- `AGENTS.md` 18
+- `CHANGELOG.md` 18
+- `data-import/geo-se.py` 17
+- `record.go` 17
+- `cmd/fejkdata/main.go` 16
+- `struct.go` 16
+- `testdata/shipped_shape.txt` 16
+- `.github/workflows/test.yml` 15
+- `arm.go` 15
+
+## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
+
+96842 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `CHANGELOG.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 2
+- `doc.go` 2
+- `docs/decisions.md` 2
+- `path.go` 2
+- `resolve.go` 2
+- `AGENTS.md` 1
+- `arm.go` 1
+
+## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
+
+152553 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 8
+- `data-import/geo-us.py` 5
+- `AGENTS.md` 4
+- `fejkdata.go` 4
+- `graph.go` 4
+- `docs/decisions.md` 3
+- `name.go` 3
+- `pick.go` 3
+- `resolve.go` 3
+
+## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
+
+2852445 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 23
+- `cmd/fejkdata/main.go` 23
+- `pick.go` 18
+- `AGENTS.md` 17
+- `CHANGELOG.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+- `arm.go` 15
+- `fejkdata.go` 15
+- `path.go` 15
+
+## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
+
+93957 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `name.go` 6
+- `README.md` 5
+- `graph.go` 4
+- `env.go` 3
+- `path.go` 3
+- `resolve.go` 3
+- `arm.go` 2
+- `compose.yaml` 2
+- `docs/decisions.md` 2
+
+## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
+
+68500 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `render_test.go` 6
+- `todo.md` 3
+- `README.md` 2
+- `docs/decisions.md` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `doc.go` 1
+- `doc_test.go` 1
+- `node.go` 1
+- `render.go` 1
+
+## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
+
+69756 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 3
+- `docs/decisions.md` 3
+- `CHANGELOG.md` 2
+- `AGENTS.md` 1
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/tsv.py` 1
+
+## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
+
+60588 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 2
+- `docs/decisions.md` 2
+- `record.go` 2
+- `render.go` 2
+- `CHANGELOG.md` 1
+- `docs/comprehension-history.md` 1
+- `inline.go` 1
+- `reference.go` 1
+- `shipped.go` 1
+
+## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
+
+2855972 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 25
+- `CHANGELOG.md` 22
+- `cmd/fejkdata/main.go` 20
+- `node.go` 19
+- `testdata/shipped_shape.txt` 19
+- `AGENTS.md` 18
+- `arm.go` 17
+- `DATA-LICENSES.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+
+## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
+
+340530 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 15
+- `docs/decisions.md` 14
+- `README.md` 4
+- `.github/workflows/test.yml` 2
+- `compose.yaml` 2
+- `fejkdata.go` 2
+- `go.mod` 2
+- `release-tooling/publish_release.py` 2
+- `shipped.go` 2
+- `shippedindex.go` 2
+
+## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
+
+91452 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `todo.md` 5
+- `cmd/fejkdata/main_test.go` 3
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `imports_test.go` 2
+- `.dockerignore` 1
+- `.github/workflows/test.yml` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
+
+112728 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `docs/decisions.md` 5
+- `shipped_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data.go` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+- `internal/datafiles/datafiles.go` 1
+
+## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
+
+200600 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 13
+- `todo.md` 8
+- `CHANGELOG.md` 6
+- `cmd/fejkdata/main.go` 6
+- `docs/decisions.md` 6
+- `shipped_test.go` 6
+- `DATA-LICENSES.md` 5
+- `bench_test.go` 5
+- `data_test.go` 5
+- `data.go` 4
+
+## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
+
+87725 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 4
+- `data_test.go` 4
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/source.py` 1
+
+## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
+
+90566 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `data-import/geo-us.py` 6
+- `README.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 3
+- `compose.yaml` 2
+- `data/geo/US/locality.tsv` 2
+- `data_test.go` 2
+- `.gitignore` 1
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
+
+76079 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 10
+- `docs/decisions.md` 6
+- `README.md` 4
+- `AGENTS.md` 2
+- `data-import/geo-se.py` 2
+- `CHANGELOG.md` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+
+## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
+
+102267 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `CHANGELOG.md` 6
+- `todo.md` 6
+- `cli/cli_test.go` 4
+- `docs/decisions.md` 3
+- `cmd/fejkdata/main_test.go` 2
+- `AGENTS.md` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+- `data/modules.go` 1
+
+## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
+
+372546 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 30
+- `todo.md` 10
+- `cli/cli.go` 5
+- `cli/cli_test.go` 3
+- `CHANGELOG.md` 2
+- `data/sv_SE/.fejkdata.json` 2
+- `docs/decisions.md` 2
+- `internal/datafiles/datafiles.go` 2
+- `record.go` 2
+- `compose.yaml` 1
+
+## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
+
+78327 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 8
+- `README.md` 5
+- `CHANGELOG.md` 1
+- `DATA-LICENSES.md` 1
+- `data/misc/car.tsv` 1
+- `docs/decisions.md` 1
+- `docs/research/research-sources-se.md` 1
+- `docs/research/research-sources-world.md` 1
+
+## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
+
+60817 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `shipped_data_test.go` 2
+- `.dockerignore` 1
+- `.gitignore` 1
+- `data_test.go` 1
+- `docs/decisions.md` 1
+- `release-tooling/publish_release.py` 1
+- `shipped_test.go` 1
+
+## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
+
+78241 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 10
+- `todo.md` 7
+- `CHANGELOG.md` 3
+- `docs/decisions.md` 3
+- `compose.yaml` 1
+- `data/en_US/itin.json` 1
+- `data/misc/timezone.json` 1
+- `data/misc/timezone.tsv` 1
+- `go.mod` 1

docs/research/research-age-bands.md

@@ -0,0 +1,10 @@
+# Population by age band — research 2026-10-04
+
+Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
+
+| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
+|---|---|---|---|---|---|---|---|
+| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
+| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
+
+At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.

docs/research/research-sources-se.md

@@ -1,7 +1,7 @@
 # Swedish (sv_SE) fake-data sources — research 2026-09-17
 
 Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
-"Verified" = fetched and read this session. Local copies of inspected files: `/home/lilleman/.claude/jobs/e377c8d4/tmp/`.
+"Verified" = fetched and read for this research.
 
 ## Licence summary (embed in MIT repo?)
 
@@ -15,7 +15,7 @@ Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other
 | Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
 | Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
 | Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
-| PTS numbering plan PDF | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
+| PTS numbering plan PDF, pts.se number pages | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
 | Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
 | SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
 | Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
@@ -90,6 +90,7 @@ Recommendation: embed SCB 2022 xlsx-derived lists (top-N per sheet with counts)
 ## 3. Phone
 
 - Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
+- Reserved for fiction, PTS "Telefonnummer till böcker och filmer" (verified 2026-09-21; curl and Playwright are Radware-blocked, a rendering fetcher gets through): https://www.pts.se/internet-och-telefoni/telefonnummer-och-adressering/telefonnummer-till-bocker-och-filmer/ — mobile 070-1740605–1740699, geographic 08-46500400–46500499 (Stockholm), 031-3900600–3900699 (Göteborg), 040-6280400–6280499 (Malmö), 0980-319200–319299 (Kiruna). 495 numbers PTS holds back so they reach nobody, which is goal 15's never-reaching path for `sv_SE.phone`. Same licence status as the numbering plan: none stated, facts only, cite PTS. The mobile block fits the shipped `{a} {b} {c}` grouping (`174 06 05`–`99`), as do 031 and 040; Stockholm's 8 subscriber digits and Kiruna's 6 need a format of their own, and 0980 is not among the 20 shipped area codes.
 - 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
 - Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
 - Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).

draw.go

@@ -1,461 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// drawSet is one render's reference draws: the unnamed draw group's, and each named one's.
-type drawSet struct {
-	unnamed draws
-	named   map[string]*draws
-}
-
-// drawScope is where a render reads its reference paths: a draw set, in the draw group of the
-// template rendering, and the row a table's format is rendering, which its columns read.
-type drawScope struct {
-	set   *drawSet
-	group string
-	t     *table
-	row   int
-}
-
-// newDrawSet makes the unnamed group's maps where the set is declared, keeping them on that frame's
-// stack for a render that reads through them; a zero drawSet makes them on its first read instead.
-func newDrawSet(s *session) drawSet {
-	return drawSet{unnamed: draws{variant: map[string]node{}, value: map[string]draw{}, s: s}}
-}
-
-// renderOnce renders n as one render, over draws of its own.
-func renderOnce(s *session, n node) string {
-	set := drawSet{unnamed: draws{s: s}}
-	return render(s, n, drawScope{set: &set})
-}
-
-// in is the scope t renders in: its draw group where it names one, else its caller's.
-func (sc drawScope) in(t *template) drawScope {
-	if t.drawGroupKey != "" {
-		sc.group = t.drawGroupKey
-	}
-	return sc
-}
-
-// draws is the set's draws for sc's draw group. The session is passed in rather than
-// read out of the set: copying it from there would leak the set's maps to the heap.
-func (sc drawScope) draws(s *session) *draws {
-	if sc.group == "" {
-		return &sc.set.unnamed
-	}
-	d, drew := sc.set.named[sc.group]
-	if !drew {
-		if sc.set.named == nil {
-			sc.set.named = map[string]*draws{}
-		}
-		d = &draws{variant: map[string]node{}, value: map[string]draw{}, s: s}
-		sc.set.named[strings.Clone(sc.group)] = d // a key from sc would leak sc, and with it every render's draw set, to the heap
-	}
-	return d
-}
-
-// drawGroupOf reads a template's "drawGroup" (default ""), which a repeat cannot carry: each
-// iteration renders in no draw group.
-func drawGroupOf(m map[string]any, repeat int) (string, error) {
-	v, ok := m["drawGroup"]
-	if !ok {
-		return "", nil
-	}
-	name, ok := v.(string)
-	switch {
-	case !ok:
-		return "", fmt.Errorf("drawGroup must be a string, got %T", v)
-	case name == "":
-		return "", fmt.Errorf(`drawGroup "" is the default, so it has no effect; drop it`)
-	case repeat > 1:
-		return "", fmt.Errorf("drawGroup %q on a repeat names nothing, since each iteration is a render of its own; drop it", name)
-	}
-	return name, nil
-}
-
-// keyDrawGroup keys t's draw group by the category t sits in, "" for an inline template, so a name
-// is local to its category.
-func (t *template) keyDrawGroup(category string) {
-	if t.drawGroup != "" {
-		t.drawGroupKey = category + "/" + t.drawGroup
-	}
-}
-
-// checkNestedDrawGroup refuses a template beneath one drawing in group that names group again,
-// short of a repeat or another draw group.
-func checkNestedDrawGroup(fields map[string]node, group string) error {
-	if group == "" {
-		return nil
-	}
-	var walk func(path string, n node) error
-	walk = func(path string, n node) error {
-		t, isTemplate := n.(*template)
-		switch {
-		case isTemplate && t.drawGroup == group:
-			return fmt.Errorf("%q names drawGroup %q, the draw group this template draws in already; drop it", path, group)
-		case isTemplate && (t.drawGroup != "" || t.repeat > 1):
-			return nil
-		}
-		for _, c := range contained(n) {
-			if err := walk(join(path, c.name), c.node); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	for _, name := range sortedNames(fields) {
-		if err := walk(name, fields[name]); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// drawCheck fences each template of a scope as a render of its own, remembering which nodes read a
-// reference path.
-type drawCheck struct {
-	memo map[readsMemo]bool
-}
-
-type readsMemo struct {
-	n           node
-	stopAtGroup bool
-}
-
-// checkDrawGroup refuses a draw group that splits nothing: one whose render reads every reference
-// path inside a repeat or a nested draw group, which draw apart from it whatever it names.
-func (c *drawCheck) checkDrawGroup(path string, n node) error {
-	if t, ok := n.(*template); ok && t.drawGroup != "" && !c.splitsDraws(t) {
-		return fmt.Errorf("%s: drawGroup %q splits nothing, since nothing it renders reads a reference path outside a repeat or a nested drawGroup; drop it", path, t.drawGroup)
-	}
-	return nil
-}
-
-func (c *drawCheck) checkDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok {
-		return nil
-	}
-	if err := checkOwnFamily(t); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	if !c.readsPath(t) {
-		return nil
-	}
-	w := newDrawWalk()
-	for _, e := range renderEdges(t) {
-		w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
-	}
-	if err := w.check(); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// checkOwnFamily refuses a table's format or cell that reads, however many templates
-// away and through a repeat or a draw group too, a table of its own family: a row
-// rendered whole draws its row without pinning it, so the family would draw apart
-// from the row being rendered, whichever draws the reaching template holds.
-func checkOwnFamily(t *template) error {
-	own := t.table
-	if own == nil {
-		own = t.cellOf
-	}
-	if own == nil {
-		return nil
-	}
-	seen := map[node]bool{}
-	var find func(n node) (renderEdge, *table, bool)
-	find = func(n node) (renderEdge, *table, bool) {
-		if seen[n] {
-			return renderEdge{}, nil, false
-		}
-		seen[n] = true
-		for _, e := range renderEdges(n) {
-			if a, isRef := refRead(n, e.label); isRef {
-				if head, isTable := n.(*template).fields[a.key].(*table); isTable && head.family() == own.family() {
-					return e, head, true
-				}
-			}
-			if e, head, found := find(e.to); found {
-				return e, head, true
-			}
-		}
-		return renderEdge{}, nil, false
-	}
-	if e, head, found := find(t); found {
-		return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", e.reached(), head.category, own.category)
-	}
-	return nil
-}
-
-// checkRecordDraws fences the columns of a record — a template compiled at the top without a
-// repeat — so a load proves the record view of it as well as the string view.
-func (c *drawCheck) checkRecordDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || !t.record {
-		return nil
-	}
-	// A record-only template's format renders nothing, so weigh the columns, not the format.
-	columns := recordColumns(t)
-	reads := false
-	for _, name := range columns {
-		reads = reads || c.readsPath(t.fields[name])
-	}
-	if !reads {
-		return nil
-	}
-	if err := checkColumnDraws(t, columns); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// readsPath reports whether rendering n reads a reference path, short of a repeat, which renders
-// over draws of its own.
-func (c *drawCheck) readsPath(n node) bool { return c.reads(n, false) }
-
-// splitsDraws reports whether rendering n reads a reference path that n's own draw group answers
-// for: one outside a repeat and outside a nested draw group, which hold their own draws.
-func (c *drawCheck) splitsDraws(n node) bool { return c.reads(n, true) }
-
-// reads walks what rendering n renders for a reference path, stopping at a repeat — and at a nested
-// draw group when stopAtGroup — since each holds draws of its own.
-func (c *drawCheck) reads(n node, stopAtGroup bool) bool {
-	k := readsMemo{n, stopAtGroup}
-	if r, done := c.memo[k]; done {
-		return r
-	}
-	r := false
-	for _, e := range renderEdges(n) {
-		if a, isRef := refRead(n, e.label); (isRef && (len(a.tail) > 0 || readsTable(n, a))) || (!repeats(e.to) && !(stopAtGroup && grouped(e.to)) && c.reads(e.to, stopAtGroup)) {
-			r = true
-			break
-		}
-	}
-	if c.memo == nil {
-		c.memo = map[readsMemo]bool{}
-	}
-	c.memo[k] = r
-	return r
-}
-
-func repeats(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.repeat > 1
-}
-
-func grouped(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.drawGroupKey != ""
-}
-
-// readsTable reports whether a reference of n names a table, whose draw the render's
-// group answers for even when read whole.
-func readsTable(n node, a arm) bool {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return false
-	}
-	_, isTable := t.fields[a.key].(*table)
-	return isTable
-}
-
-// refRead is the reference an edge of n reads; false when the edge reads none.
-func refRead(n node, label string) (arm, bool) {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return arm{}, false
-	}
-	a := splitArm(label, t.refs)
-	return a, isRef(a.key)
-}
-
-// checkColumnDraws fences a record's columns as one render.
-func checkColumnDraws(t *template, columns []string) error {
-	w := newDrawWalk()
-	for _, name := range columns {
-		w.walk(t.fields[name], drawAt{group: t.drawGroupKey, route: drawRoute{spelling: fmt.Sprintf("column %q", name)}})
-	}
-	return w.check()
-}
-
-// drawWalk gathers the references one render reads, by draw group, for check to compare.
-type drawWalk struct {
-	reads []pathRead
-	read  map[drawKey]bool
-	seen  map[drawVisit]bool
-}
-
-// drawAt is where a walk stands: the draw group it draws in, how the render's root reached it, and
-// the table rows it entered.
-type drawAt struct {
-	group string
-	route drawRoute
-	alt   rowSet
-}
-
-// rowSet is the rows a walk entered, one per table: only one row of a table renders, so reads in
-// two rows of one table never meet, while reads in one row, across its columns and whatever they
-// reach, do.
-type rowSet []tablePin
-
-func (s rowSet) rowOf(t *table) (int, bool) {
-	for _, p := range s {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	return 0, false
-}
-
-// enter is s with row r of t, where t is not in it yet; the set is copied, since walks branch.
-func (s rowSet) enter(t *table, r int) rowSet {
-	if _, in := s.rowOf(t); in {
-		return s
-	}
-	out := append(append(make(rowSet, 0, len(s)+1), s...), tablePin{t, r})
-	sort.Slice(out, func(i, j int) bool { return out[i].t.category < out[j].t.category })
-	return out
-}
-
-// key spells the set for a map, by the tables' identities.
-func (s rowSet) key() string {
-	var b strings.Builder
-	for _, p := range s {
-		fmt.Fprintf(&b, "%p[%d]", p.t, p.row)
-	}
-	return b.String()
-}
-
-// alternatives reports whether two reads sit in different rows of one table.
-func alternatives(a, b drawAt) bool {
-	for _, p := range a.alt {
-		if r, in := b.alt.rowOf(p.t); in && r != p.row {
-			return true
-		}
-	}
-	return false
-}
-
-// drawRoute is how a render reaches a draw: as its author spells it, and the root edge's label.
-type drawRoute struct{ spelling, label string }
-
-// pathRead is one reference a render reads: a path, or a bare reference with no tail.
-type pathRead struct {
-	at drawAt
-	a  arm
-	tr *tableRead // set where the reference names a table
-}
-
-type drawVisit struct {
-	n     node
-	group string
-	alt   string
-}
-
-type drawKey struct {
-	group string
-	path  string
-	alt   string
-}
-
-func newDrawWalk() *drawWalk {
-	return &drawWalk{read: map[drawKey]bool{}, seen: map[drawVisit]bool{}}
-}
-
-// walk follows what rendering n renders. A repeat renders over draws of its own, so the walk stops
-// there.
-func (w *drawWalk) walk(n node, at drawAt) {
-	v := drawVisit{n, at.group, at.alt.key()}
-	if w.seen[v] {
-		return
-	}
-	w.seen[v] = true
-	if repeats(n) {
-		return
-	}
-	if t, isTemplate := n.(*template); isTemplate && t.drawGroupKey != "" {
-		at.group = t.drawGroupKey
-	}
-	for _, e := range renderEdges(n) {
-		if cell, isCell := e.to.(*template); isCell && cell.cellOf != nil {
-			if at.alt.excludes(cell) {
-				continue
-			}
-			w.edge(n, e, drawAt{at.group, at.route, at.alt.enter(cell.cellOf, cell.cellRow)})
-			continue
-		}
-		w.edge(n, e, at)
-	}
-}
-
-// excludes reports whether a cell's row cannot render with the rows entered: another
-// row of its table, or a row outside an entered ancestor's.
-func (s rowSet) excludes(cell *template) bool {
-	for _, p := range s {
-		if p.t == cell.cellOf {
-			return p.row != cell.cellRow
-		}
-		if cell.cellOf.descends(p.t) && !cell.cellOf.under(cell.cellRow, p.t, p.row) {
-			return true
-		}
-	}
-	return false
-}
-
-// edge records the reference an edge reads, then walks on with every row the read
-// pins entered, so a selected row renders only its own cells.
-func (w *drawWalk) edge(from node, e renderEdge, at drawAt) {
-	if a, reads := refRead(from, e.label); reads {
-		tr := tableReadOf(from.(*template).fields[a.key], a, e.to)
-		if k := (drawKey{at.group, a.path, at.alt.key()}); !w.read[k] {
-			w.read[k] = true
-			w.reads = append(w.reads, pathRead{at, a, tr})
-		}
-		if tr != nil {
-			tr.pins.each(func(t *table, r int) { at.alt = at.alt.enter(t, r) })
-		}
-	}
-	w.walk(e.to, at)
-}
-
-// check refuses what one draw per reference path cannot answer for: a read of a level beside a path
-// another read takes into it, and two reads of one table family that select different rows. Reads
-// are compared in path order, so which pair is reported does not vary.
-func (w *drawWalk) check() error {
-	sort.SliceStable(w.reads, func(i, j int) bool {
-		if w.reads[i].at.group != w.reads[j].at.group {
-			return w.reads[i].at.group < w.reads[j].at.group
-		}
-		return w.reads[i].a.path < w.reads[j].a.path
-	})
-	for i, level := range w.reads {
-		for _, into := range w.reads[i+1:] {
-			if into.at.group != level.at.group || alternatives(level.at, into.at) {
-				continue
-			}
-			if strings.HasPrefix(into.a.path, level.a.path+".") && !(level.tr != nil && level.tr.whole) {
-				return overlapError(level.at.route, level.a.name, into)
-			}
-		}
-	}
-	return checkFamilies(w.reads)
-}
-
-func overlapError(route drawRoute, ref string, into pathRead) error {
-	return fmt.Errorf("%s renders a level that %s reads a path into; name the fields you want instead, or draw them apart with a drawGroup", route.spelled(ref), into.at.route.spelled(into.a.name))
-}
-
-// spelled names the route, and the reference it reaches a draw by where its root edge is not that
-// reference.
-func (r drawRoute) spelled(ref string) string {
-	if ref == "" || ref == r.label {
-		return r.spelling
-	}
-	return fmt.Sprintf("%s with {%s}", r.spelling, ref)
-}

env.go

@@ -0,0 +1,113 @@
+package fejkdata
+
+import (
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// readValue is what one read drew: its text, and whether it landed on a null.
+type readValue struct {
+	text string
+	null bool
+}
+
+// frameStack is the frames of the name scopes rendering, innermost last. A render's env points
+// at it, and a repeat iteration's shares it: holding the frames in a renderEnv would move every
+// render's env to the heap.
+type frameStack struct {
+	frames []*pickFrame
+}
+
+func (st *frameStack) push(f *pickFrame) int {
+	mark := len(st.frames)
+	st.frames = append(st.frames, f)
+	return mark
+}
+
+func (st *frameStack) pop(mark int) { st.frames = st.frames[:mark] }
+
+// renderEnv is where a render reads: the frames of the name scopes rendering; and row, the row
+// of the table rendering, which its columns read. It passes by value, and what is read from it
+// reaches a map key, an interface or a func value only as a copy; else env, and with it every
+// render's frames, moves to the heap.
+type renderEnv struct {
+	frames *frameStack
+	row    renderedRow
+	// base is the depth of the frame stack when the read rendering the category entered it,
+	// below which no frame is the category's.
+	base int
+	// pick is the named pick being rendered, nil outside one; pickAt is the key in it of what renders.
+	pick   *namedPick
+	pickAt pickKey
+}
+
+type renderedRow struct {
+	t     *table
+	index int
+}
+
+// at returns the env n renders in, n being the leaf of a path whose rows sit in pins; where n
+// is a table's row or column, the env carries the row pins holds for that table.
+func (env renderEnv) at(n node, pins *pinSet) renderEnv {
+	switch n := n.(type) {
+	case *tableRow:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	case *tableColumn:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	}
+	return env
+}
+
+func (env renderEnv) rowOf(t *table) int {
+	if env.row.t != t {
+		panic(invariant.Broken("a column of %s renders in an env holding no row of it", t.rows.Segment()))
+	}
+	return env.row.index
+}
+
+// enter starts a read landing on n: the read sees no frame opened before it, and gets a frame for
+// each name scope around n, from memo where there is one, so reads sharing memo read one pick of
+// each name. It returns the mark that closes those frames.
+func (env renderEnv) enter(n node, memo *drawMemo) (renderEnv, int) {
+	env = env.entering()
+	for scope := scopeAround(n); scope != nil; scope = scope.up {
+		if len(scope.order) > 0 {
+			env.frames.push(memo.enteredFrame(scope))
+		}
+	}
+	return env, env.base
+}
+
+// entering is env as a read entering a category sees it: no frame opened before it, and no pick.
+func (env renderEnv) entering() renderEnv {
+	env.base, env.pick = len(env.frames.frames), nil
+	return env
+}
+
+// renderFrame opens a fresh frame of t's scope, where t binds names and no read entering the
+// category opened one, returning the mark that closes it, or -1.
+func (env renderEnv) renderFrame(t *template) int {
+	own := t.ownScope()
+	if own == nil || env.frameOf(own) != nil {
+		return -1
+	}
+	return env.frames.push(newPickFrame(own))
+}
+
+// frameOf is the frame of scope rendering since the read entering the category, nil where none is.
+func (env renderEnv) frameOf(scope *nameScope) *pickFrame {
+	for i := len(env.frames.frames) - 1; i >= env.base; i-- {
+		if f := env.frames.frames[i]; f.scope == scope {
+			return f
+		}
+	}
+	return nil
+}
+
+// drawRowOf draws the row a render of t reads: inside the pick's rows where t renders as part of one.
+func (env renderEnv) drawRowOf(s *drawstate.State, t *table) int {
+	if env.pick != nil {
+		return t.rows.DrawIn(s, &env.pick.pins)
+	}
+	return t.rows.Draw(s)
+}

family.go

@@ -1,319 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// tablePin is one table's pinned row.
-type tablePin struct {
-	t   *table
-	row int
-}
-
-// pinned is the row the render pinned for t, if any.
-func (d *draws) pinned(t *table) (int, bool) {
-	for _, p := range d.pins[:d.npins] {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	r, ok := d.more[t]
-	return r, ok
-}
-
-// mustRow is the row pinned for t, which the walk reaching a column pinned.
-func (d *draws) mustRow(t *table) int {
-	r, ok := d.pinned(t)
-	if !ok {
-		panic(fmt.Sprintf("fejkdata: a column of %s is rendered with no row pinned", t.category))
-	}
-	return r
-}
-
-// pin pins row r of t, and the rows of t's ancestors it links to.
-func (d *draws) pin(t *table, r int) {
-	for {
-		if _, done := d.pinned(t); done {
-			return
-		}
-		if d.npins < len(d.pins) {
-			d.pins[d.npins] = tablePin{t, r}
-			d.npins++
-		} else {
-			if d.more == nil {
-				d.more = map[*table]int{}
-			}
-			d.more[t] = r
-		}
-		if t.parentT == nil {
-			return
-		}
-		t, r = t.parentT, t.parentRow(r)
-	}
-}
-
-// each calls fn for every pinned row, in pin order, the spilled ones by table name.
-func (d *draws) each(fn func(t *table, r int)) {
-	for _, p := range d.pins[:d.npins] {
-		fn(p.t, p.row)
-	}
-	spilled := make([]*table, 0, len(d.more))
-	for t := range d.more {
-		spilled = append(spilled, t)
-	}
-	sort.Slice(spilled, func(i, j int) bool { return spilled[i].category < spilled[j].category })
-	for _, t := range spilled {
-		fn(t, d.more[t])
-	}
-}
-
-// rowOf is the render's row of t: the one pinned, else one drawn inside the
-// nearest pinned ancestor — its parent drawn inside that first where the ancestor
-// is further up — or over the whole table, and pinned with its ancestors.
-func (d *draws) rowOf(t *table) int {
-	if r, ok := d.pinned(t); ok {
-		return r
-	}
-	r := -1
-	for a := t.parentT; a != nil && r < 0; a = a.parentT {
-		if _, ok := d.pinned(a); !ok {
-			continue
-		}
-		if t.parentT != a {
-			d.rowOf(t.parentT)
-		}
-		pr, _ := d.pinned(t.parentT)
-		r = t.drawUnder(d.s, pr)
-	}
-	if r < 0 {
-		r = t.draw(d.s)
-	}
-	d.pin(t, r)
-	return r
-}
-
-// pinRow pins row r of t where it agrees with the rows pinned before it.
-func (d *draws) pinRow(t *table, r int) error {
-	if pr, ok := d.pinned(t); ok && pr != r {
-		return fmt.Errorf("%s and %s are two rows of %s", t.selectorSpelling(pr), t.selectorSpelling(r), t.path)
-	}
-	for a := t.parentT; a != nil; a = a.parentT {
-		if pa, ok := d.pinned(a); ok && !t.under(r, a, pa) {
-			return fmt.Errorf("%s is not inside %s", t.selectorSpelling(r), a.selectorSpelling(pa))
-		}
-	}
-	d.pin(t, r)
-	return nil
-}
-
-// selectRow pins the row a selector names.
-func (d *draws) selectRow(t *table, sel string) error {
-	r, err := t.find(sel, d)
-	if err != nil {
-		return err
-	}
-	return d.pinRow(t, r)
-}
-
-// inside keeps the rows of t that sit inside every pinned ancestor.
-func (d *draws) inside(t *table, rows []int) []int {
-	for a := t.parentT; a != nil; a = a.parentT {
-		pa, ok := d.pinned(a)
-		if !ok {
-			continue
-		}
-		var kept []int
-		for _, r := range rows {
-			if t.under(r, a, pa) {
-				kept = append(kept, r)
-			}
-		}
-		rows = kept
-	}
-	return rows
-}
-
-// tableRead is what a reference path reads of a table family: the table its head
-// names, the rows its selectors pin, the tables it draws — those it walks with no
-// row pinned, and their unpinned ancestors — each selector's spelling, and whether
-// it lands on a row rendered whole.
-type tableRead struct {
-	head  *table
-	pins  draws
-	drawn map[*table]bool
-	sels  []tableSel
-	whole bool
-}
-
-// tableSel is one selector on the way: the table it selects a row of, and the path
-// as written up to and including it.
-type tableSel struct {
-	t        *table
-	spelling string
-}
-
-// tableReadOf replays a reference's selectors through the walk a render uses, so
-// two paths pinning one row by different routes compare equal. checkPath proved
-// each selector names a row.
-func tableReadOf(head node, a arm, leaf node) *tableRead {
-	t, isTable := head.(*table)
-	if !isTable {
-		return nil
-	}
-	tr := &tableRead{head: t, drawn: map[*table]bool{}}
-	_, _ = walkPath(t, a.tail, pathWalk{pins: &tr.pins})
-	written := a.name[:len(a.name)-len(joinSegments(a.tail))]
-	cur := t
-	tr.draws(cur)
-	for i, seg := range a.tail {
-		switch d := cur.descendant(seg); {
-		case isSelector(seg):
-			tr.sels = append(tr.sels, tableSel{cur, written + joinSegments(a.tail[:i+1])})
-		case d != nil:
-			cur = d
-			tr.draws(cur)
-		}
-	}
-	_, tr.whole = leaf.(*row)
-	return tr
-}
-
-// draws marks t and its ancestors drawn, up to the first the read pins.
-func (r *tableRead) draws(t *table) {
-	for ; t != nil; t = t.parentT {
-		if _, pinned := r.pins.pinned(t); pinned {
-			return
-		}
-		r.drawn[t] = true
-	}
-}
-
-// joinSegments spells segments as a path: a selector attaches to the name before it.
-func joinSegments(segs []string) string {
-	var b strings.Builder
-	for _, s := range segs {
-		if b.Len() > 0 && !isSelector(s) {
-			b.WriteByte('.')
-		}
-		b.WriteString(s)
-	}
-	return b.String()
-}
-
-// family is the table a chain of parents ends at.
-func (t *table) family() *table {
-	for t.parentT != nil {
-		t = t.parentT
-	}
-	return t
-}
-
-// selected is the selector in r on t, or on the nearest ancestor of t it selects.
-func (r *tableRead) selected(t *table) (tableSel, bool) {
-	for ; t != nil; t = t.parentT {
-		for _, s := range r.sels {
-			if s.t == t {
-				return s, true
-			}
-		}
-	}
-	return tableSel{}, false
-}
-
-// checkFamilies refuses reads of one table family in one draw group that cannot
-// read one consistent draw: a table rendered whole beside a path into the family,
-// a table one read draws that another pins, and two reads pinning different rows.
-// The reads come sorted by group and path, so which pair is reported does not vary.
-func checkFamilies(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[:i] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(o.at, r.at) || o.tr.head.family() != r.tr.head.family() {
-				continue
-			}
-			if err := checkFamilyPair(o, r); err != nil {
-				return err
-			}
-		}
-	}
-	return replayPairs(reads)
-}
-
-// replayPairs replays every two reads of one draw group that can render together into
-// one draws, the earlier read first. A pin conflicts with one earlier pin, never with a
-// combination, so pairs find every conflict a full replay would.
-func replayPairs(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[i+1:] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(r.at, o.at) {
-				continue
-			}
-			var d draws
-			if err := r.tr.replay(&d); err != nil {
-				return conflict(r, err)
-			}
-			if err := o.tr.replay(&d); err != nil {
-				return conflict(o, err)
-			}
-		}
-	}
-	return nil
-}
-
-func conflict(r pathRead, err error) error {
-	return fmt.Errorf("%s: %w; select the same rows in every path into the family, or draw them apart with a drawGroup", r.at.route.spelled(r.a.name), err)
-}
-
-// replay pins the read's rows into d, where they agree with the rows pinned before.
-func (r *tableRead) replay(d *draws) error {
-	var err error
-	r.pins.each(func(t *table, row int) {
-		if err == nil {
-			err = d.pinRow(t, row)
-		}
-	})
-	return err
-}
-
-// checkFamilyPair refuses a table read whole beside a path into its family, and a
-// table one read draws that the other pins, since which token renders first would
-// then decide the row.
-func checkFamilyPair(a, b pathRead) error {
-	for _, pair := range [][2]pathRead{{a, b}, {b, a}} {
-		x, y := pair[0], pair[1]
-		if len(x.a.tail) == 0 && len(y.a.tail) > 0 {
-			return overlapError(x.at.route, x.a.name, y)
-		}
-		if drawn := x.tr.drawnOf(&y.tr.pins); drawn != nil {
-			if s, ok := y.tr.selected(x.tr.head); ok && len(x.tr.sels) == 0 {
-				tail := x.a.tail
-				if s.t != x.tr.head {
-					tail = append([]string{x.tr.head.category}, tail...)
-				}
-				return fmt.Errorf("%s draws %s, which %s selects a row of; write {%s.%s}, or draw them apart with a drawGroup",
-					x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name), s.spelling, joinSegments(tail))
-			}
-			return fmt.Errorf("%s draws %s, which %s selects a row of; select that row in both, or draw them apart with a drawGroup",
-				x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name))
-		}
-	}
-	return nil
-}
-
-// drawnOf is a table the read draws that pins holds a row of, if any.
-func (r *tableRead) drawnOf(pins *draws) *table {
-	var found *table
-	pins.each(func(t *table, _ int) {
-		if found == nil && r.drawn[t] {
-			found = t
-		}
-	})
-	return found
-}

fejkdata.go

@@ -1,81 +1,51 @@
-// Package fejkdata generates fake data from recursive JSON templates.
-//
-// Data lives in JSON, not in Go. A generator starts from the shipped data set and
-// layers any directories you add; folders and files become a dot-path namespace,
-// then generate values by path:
-//
-//	f, _ := fejkdata.New(fejkdata.WithSeed(42))
-//	f.Fake("sv_SE.address")          // "Järvedsvägen 43\n891 77 Järved"
-//	f.Fake("sv_SE.address.locality") // "Sundbyberg": a second call draws afresh
-//
-// Several sources merge in order, the last winning a name clash, so custom data
-// layers over the built-ins. The JSON template format is documented in the README.
 package fejkdata
 
 import (
-	crand "crypto/rand"
-	"embed"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io/fs"
-	"math/rand/v2"
-	"os"
 	"reflect"
 	"sort"
 	"sync"
-)
 
-//go:embed data
+	"github.com/larvit/fejkdata/internal/datafiles"
-var shippedFS embed.FS
+	"github.com/larvit/fejkdata/internal/drawstate"
+)
 
 // MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
 // any one path; the CLI's --repeat shares it.
 const MaxRepeat = 1 << 20
 
-var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions
+// Fails to compile where uint is narrower than 64 bits: docs/decisions.md#64-bit-targets-only
+var _ [^uint(0)>>63 - 1]struct{}
 
-// ErrNoData is returned by New when no source is loaded at all.
+// Generator generates fake data from a namespace tree. Create one with [New].
-var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS")
+// It is safe for concurrent use; a sequence is reproducible only when drawn
-
+// from one goroutine.
-// Generator generates fake data from a loaded namespace tree. Create one with [New].
-// It is safe for concurrent use; a seeded sequence is reproducible only when drawn
-// from one goroutine. The compiled tree is immutable after [New], so List reads it
-// without a lock; mu guards the rest, which every other entry point takes.
 type Generator struct {
-	mu         sync.Mutex
+	// mu guards drawState, structs and root, which gains an indexed category on the
-	rand       *session
+	// first call reaching it.
-	categories map[string]node
+	mu        sync.Mutex
-	root       folder  // the categories as the node a path walks from, owned here so a walk allocates none
+	drawState *drawstate.State
-	set        drawSet // one Fake's draws, owned here so a walk pinning rows keeps them off the heap
+	root      folder // the categories as the node a path walks from, owned here so a walk allocates none
-	records    map[node]recordShape
+	structs   map[reflect.Type]structResult
-	structs    map[reflect.Type]structResult
-}
-
-// session is one generator's mutable render state: the seeded rng plus the {seq()}
-// counters.
-type session struct {
-	*rand.Rand
-	counters map[string]uint64
-}
-
-func (s *session) next(key string) uint64 {
-	s.counters[key]++
-	return s.counters[key]
 }
 
 type config struct {
-	seed    uint64
+	seed       uint64
-	seeded  bool
+	seeded     bool
-	shipped bool
+	sources    []datafiles.Source
-	sources []dataSource
+	nilModules error
+	// dataGiven is whether a data option was passed, even WithDataFS(), which loads none.
+	dataGiven bool
 }
 
 // Option configures a [Generator].
 type Option func(*config)
 
-// WithSeed makes output reproducible: two generators with the same seed and data
+// WithSeed seeds the generator, as [New] requires: two generators with the same seed,
-// emit identical sequences.
+// version and data emit identical sequences. For a new seed each run, set
+// seed := rand.Uint64() (math/rand/v2), and log it to replay the run.
 func WithSeed(seed uint64) Option {
 	return func(c *config) { c.seed, c.seeded = seed, true }
 }
@@ -84,175 +54,85 @@ func WithSeed(seed uint64) Option {
 // layer several; the last wins a name clash.
 func WithDataPath(dir string) Option {
 	return func(c *config) {
-		c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir})
+		c.sources, c.dataGiven = append(c.sources, datafiles.Dir(dir)), true
 	}
 }
 
-// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own.
+// WithDataFS layers modules, each a data tree held in an [fs.FS], over what is loaded
-func WithDataFS(fsys fs.FS) Option {
+// before them, in order. A module may be a data package's FS, such as one of
-	return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) }
+// github.com/larvit/fejkdata/data's Modules, or an embed.FS of your own. WithDataFS()
+// loads no data.
+func WithDataFS(modules ...fs.FS) Option {
+	return func(c *config) {
+		for i, fsys := range modules {
+			if fsys == nil {
+				c.nilModules = errors.Join(c.nilModules, fmt.Errorf("fejkdata: WithDataFS module %d is nil", i+1))
+				continue
+			}
+			c.sources = append(c.sources, datafiles.FS(fsys))
+		}
+		c.dataGiven = true
+	}
 }
 
-// WithoutShippedData leaves the shipped data set out, so only the sources given
+var errNoSeed = errors.New(`fejkdata: no seed: add fejkdata.WithSeed(n), where the same n repeats a run's output; for a new seed each run, set n := rand.Uint64() (import "math/rand/v2") and log n to replay the run`)
-// with [WithDataPath] and [WithDataFS] load.
-func WithoutShippedData() Option {
-	return func(c *config) { c.shipped = false }
-}
 
-// New builds a generator from the shipped data set and the options' sources, merged
+var errNoData = errors.New(`fejkdata: no data: add fejkdata.WithDataFS(data.Modules()...) (import "github.com/larvit/fejkdata/data") for all shipped data, WithDataFS/WithDataPath for your own, or WithDataFS() for none`)
-// in order with the last winning a name clash. Each JSON file becomes a category
+
-// named after the file (address.json -> "address") and each subdirectory a
+// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
-// namespace segment. It errors on a missing directory, invalid JSON, invalid data,
+// of an indexed source.
-// or no data at all.
+var ErrLoad = errors.New("data fails to load")
+
+// loadError is a load's error, unchanged in its text, matching ErrLoad.
+type loadError struct{ error }
+
+func (e loadError) Unwrap() error      { return e.error }
+func (loadError) Is(target error) bool { return target == ErrLoad }
+
+// New builds a generator from the options' sources, merged in order with the last
+// winning a name clash; it fails where no [WithSeed] gives the seed, or no [WithDataFS]
+// or [WithDataPath] names the data to load. Each JSON file becomes a category named
+// after the file (address.json -> "address") and each subdirectory a namespace segment.
+// It errors on a missing directory, invalid JSON or invalid data. Given only
+// WithDataFS() with no modules, New loads no data, and the generator renders templates
+// that read none. A source whose .fejkdata.json carries an index, as each shipped
+// module's does, loads each category on the first call reaching it. Data failing to
+// load, in New or on that first call, gives an error matching [ErrLoad].
 func New(opts ...Option) (*Generator, error) {
-	c := config{shipped: true}
+	var c config
 	for _, opt := range opts {
 		opt(&c)
 	}
-	var sources []dataSource
+	if err := c.misuse(); err != nil {
-	if c.shipped {
+		return nil, err
-		sources = append(sources, dataSource{fsys: shippedFS, root: "data"})
 	}
-	sources = append(sources, c.sources...)
+	root, err := loadSources(c.sources)
-	if len(sources) == 0 {
-		return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
-	}
-	cats, err := loadData(sources)
 	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
+		return nil, fmt.Errorf("fejkdata: %w", loadError{err})
 	}
-	rng, err := newRand(c.seed, c.seeded)
+	return &Generator{drawState: drawstate.New(c.seed), root: root}, nil
-	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
-	}
-	return &Generator{rand: rng, categories: cats}, nil
 }
 
-// List returns the sorted dotted paths Fake can render: every category, the dotted
+// List returns the sorted dotted paths Fake renders: each category and every field,
-// fields within a template, and folder segments. A choice consumes no segment, so a
+// column and linked table below it, a direct descent at a time, listing a level
-// path continues through one only where every variant carries it, which is the
+// carrying a repeat but nothing below it. A choice consumes no segment, so a path
-// rule Fake applies too: List is the set of paths Fake accepts.
+// continues through one only where every variant carries it.
 func (f *Generator) List() []string {
-	var out []string
+	f.mu.Lock()
-	for _, name := range sortedNames(f.categories) {
+	defer f.mu.Unlock()
-		for _, p := range paths(f.categories[name]) {
+	out := paths(&f.root, false)
-			out = append(out, join(name, p))
-		}
-	}
 	sort.Strings(out)
 	return out
 }
 
-// paths lists the dot paths addressable from n, relative to it, where "" is n
+// misuse names every mistake in the options, before any data loads.
-// itself. A folder has no value of its own, so it contributes only its children's.
+func (c config) misuse() error {
-func paths(n node) []string {
+	var err error
-	switch n := n.(type) {
+	if !c.dataGiven {
-	case *folder:
+		err = errNoData
-		var out []string
-		for _, name := range sortedNames(n.children) {
-			for _, p := range paths(n.children[name]) {
-				out = append(out, join(name, p))
-			}
-		}
-		return out
-	case *template:
-		out := []string{""}
-		for _, name := range sortedNames(n.fields) {
-			if isRef(name) { // a binding, not a path segment
-				continue
-			}
-			for _, p := range paths(n.fields[name]) {
-				out = append(out, join(name, p))
-			}
-		}
-		return out
-	case *null:
-		return []string{""}
-	case *table:
-		return tablePaths(n)
-	case *column, *row:
-		return []string{""}
-	case *choice:
-		out := []string{""}
-		for p := range n.shared {
-			out = append(out, p)
-		}
-		return out
 	}
-	return nil
+	if !c.seeded {
-}
+		err = errors.Join(err, errNoSeed)
-
+	}
-// tablePaths is a table's columns, then each table linked to it under its name: the
+	return errors.Join(err, c.nilModules)
-// direct descents, a step at a time.
-func tablePaths(t *table) []string {
-	out := append([]string{""}, t.columns...)
-	sort.Strings(out[1:])
-	children := make([]string, 0, len(t.children))
-	for name := range t.children {
-		children = append(children, name)
-	}
-	sort.Strings(children)
-	for _, name := range children {
-		for _, p := range paths(t.children[name]) {
-			out = append(out, join(name, p))
-		}
-	}
-	return out
-}
-
-// sharedPaths is the sub-paths every item carries — the only ones a path may step
-// through a choice to reach. It intersects, bailing as soon as the set
-// is empty, which is immediate for a choice of plain strings.
-func sharedPaths(items []node) map[string]bool {
-	shared := subPaths(items[0])
-	for _, it := range items[1:] {
-		if len(shared) == 0 {
-			return nil
-		}
-		next := subPaths(it)
-		for p := range shared {
-			if !next[p] {
-				delete(shared, p)
-			}
-		}
-	}
-	return shared
-}
-
-// subPaths is paths(n) as a set, without the empty path that means n itself.
-func subPaths(n node) map[string]bool {
-	out := map[string]bool{}
-	for _, p := range paths(n) {
-		if p != "" {
-			out[p] = true
-		}
-	}
-	return out
-}
-
-func join(prefix, name string) string {
-	switch {
-	case prefix == "":
-		return name
-	case name == "":
-		return prefix
-	}
-	return prefix + "." + name
-}
-
-// randomBytes seeds an unseeded generator.
-var randomBytes = crand.Read
-
-func newRand(seed uint64, seeded bool) (*session, error) {
-	var r *rand.Rand
-	if seeded {
-		r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
-	} else {
-		var b [16]byte
-		if _, err := randomBytes(b[:]); err != nil {
-			return nil, fmt.Errorf("seeding from crypto/rand: %w", err)
-		}
-		r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:])))
-	}
-	return &session{Rand: r, counters: map[string]uint64{}}, nil
 }

fejkdata_test.go

@@ -3,11 +3,18 @@ package fejkdata
 import (
 	"errors"
 	"io/fs"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
-	"testing/fstest"
+
+	"github.com/larvit/fejkdata/data"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
+func withShipped() Option { return WithDataFS(data.Modules()...) }
+
 // newGenerator creates a generator over a single data directory, failing on
 // error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
 // on conflicts).
@@ -18,17 +25,28 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator {
 
 func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
 	t.Helper()
-	all := []Option{WithoutShippedData()}
+	var all []Option
 	for _, dir := range dirs {
 		all = append(all, WithDataPath(dir))
 	}
-	f, err := New(append(all, opts...)...)
+	f, err := New(append(append(all, WithSeed(1)), opts...)...)
 	if err != nil {
 		t.Fatalf("New(%q): %v", dirs, err)
 	}
 	return f
 }
 
+// newBareModule loads the shipped module in data/<module> without its manifest, so its
+// categories sit at the root, failing on error.
+func newBareModule(t *testing.T, module string, opts ...Option) *Generator {
+	t.Helper()
+	f, err := New(append([]Option{WithDataFS(withoutManifest{os.DirFS(filepath.Join("data", module))}), WithSeed(1)}, opts...)...)
+	if err != nil {
+		t.Fatalf("New(%s without its manifest): %v", module, err)
+	}
+	return f
+}
+
 // fake generates a value, failing the test on error.
 func fake(t *testing.T, f *Generator, path string) string {
 	t.Helper()
@@ -40,31 +58,32 @@ func fake(t *testing.T, f *Generator, path string) string {
 }
 
 func TestNewMissingDirectory(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath("data/de_DE"))
+	_, err := New(WithSeed(1), WithDataPath("data/de_DE"))
 	if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
 		t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
 	}
 }
 
 func TestNewRejectsAnEmptyDataPath(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath(""))
+	_, err := New(WithSeed(1), WithDataPath(""))
 	if err == nil || !strings.Contains(err.Error(), "empty") {
 		t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
 	}
 }
 
-func TestNewReportsAnEntropyFailure(t *testing.T) {
+func TestNewWithoutASeedFailsNamingWithSeed(t *testing.T) {
-	saved := randomBytes
+	for name, opts := range map[string][]Option{"no option": nil, "only data": {WithDataFS()}} {
-	randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") }
+		if _, err := New(opts...); err == nil || !strings.Contains(err.Error(), "WithSeed(") {
-	defer func() { randomBytes = saved }()
+			t.Errorf("New(%s) = %v, want it refused, naming WithSeed", name, err)
-	_, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}}))
+		}
-	if err == nil || !strings.Contains(err.Error(), "no entropy") {
+	}
-		t.Fatalf("New() without entropy = %v, want the failure reported", err)
+	if _, err := New(WithDataFS(nil)); err == nil || !strings.Contains(err.Error(), "WithSeed(") || !strings.Contains(err.Error(), "module 1 is nil") {
+		t.Errorf("New(WithDataFS(nil)) = %v, want both the missing seed and the nil module named", err)
 	}
 }
 
 func TestWithSeedIsDeterministic(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(42)), newGenerator(t, "data", WithSeed(42))
+	a, b := shipped(t, WithSeed(42)), shipped(t, WithSeed(42))
 	for i := 0; i < 50; i++ {
 		if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
 			t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
@@ -73,7 +92,7 @@ func TestWithSeedIsDeterministic(t *testing.T) {
 }
 
 func TestDifferentSeedsDiffer(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(1)), newGenerator(t, "data", WithSeed(2))
+	a, b := shipped(t, WithSeed(1)), shipped(t, WithSeed(2))
 	for i := 0; i < 50; i++ {
 		if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
 			return
@@ -92,3 +111,43 @@ func fakeTemplate(t *testing.T, f *Generator, s string) string {
 	}
 	return got
 }
+
+// engine builds a seeded generator with no loaded categories, for rendering tests.
+func engine(seed uint64) *Generator {
+	return &Generator{drawState: drawstate.New(seed)}
+}
+
+// parse unmarshals a JSON template fragment into its dynamic form.
+func parse(t *testing.T, s string) jsonvalue.Value {
+	t.Helper()
+	v, err := jsonvalue.Decode([]byte(s))
+	if err != nil {
+		t.Fatalf("parse %q: %v", s, err)
+	}
+	return v
+}
+
+// resolved compiles a JSON fragment and resolves it against no data.
+func resolved(t *testing.T, s string) (node, error) {
+	t.Helper()
+	n, err := compile(parse(t, s))
+	if err != nil {
+		return nil, err
+	}
+	return n, resolveInlineTemplates(inlineNodes(n, "template"), nil)
+}
+
+// compiled parses, compiles and resolves a JSON fragment into a node.
+func compiled(t *testing.T, s string) node {
+	t.Helper()
+	n, err := resolved(t, s)
+	if err != nil {
+		t.Fatalf("compile %q: %v", s, err)
+	}
+	return n
+}
+
+func mustRender(t *testing.T, f *Generator, s string) string {
+	t.Helper()
+	return renderOnce(f.drawState, compiled(t, s))
+}

format.go

@@ -0,0 +1,205 @@
+package fejkdata
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// checkFunc validates a call at compile time: a known builtin, its args, and, for calc, the
+// operands held in fields.
+func checkFunc(tok grammar.Token, fields map[string]node) error {
+	err := builtinfunc.Check(tok.Fn, tok.Args)
+	if err == nil && tok.Fn == builtinfunc.CalcName {
+		err = checkCalcFields(tok.Args, fields)
+	}
+	if err != nil {
+		return fmt.Errorf("token {%s}: %w", tok.Body, err)
+	}
+	return nil
+}
+
+func parseChecked(format string, fields map[string]node) ([]grammar.Token, []unboundRead, error) {
+	toks, err := grammar.ParseFormat(format)
+	if err != nil {
+		return nil, nil, err
+	}
+	unbound, err := checkTokens(toks, fields)
+	return toks, unbound, err
+}
+
+// unboundRead is a token reading a head no field holds, which only a name bound around the
+// template can answer; err is the refusal where none does, and body is the token's.
+type unboundRead struct {
+	head, body string
+	err        error
+}
+
+// checkTokens proves every token's grammar, functions and field paths, and returns the reads
+// whose head no field holds, for bindNames to look up among the names.
+func checkTokens(toks []grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	var unbound []unboundRead
+	for _, t := range toks {
+		u, err := checkToken(t, fields)
+		if err != nil {
+			return nil, err
+		}
+		unbound = append(unbound, u...)
+	}
+	return unbound, nil
+}
+
+func checkToken(t grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	switch t.Kind {
+	case grammar.BuiltinCall:
+		if err := checkFunc(t, fields); err != nil {
+			return nil, err
+		}
+		return checkReads(t, fields, true)
+	case grammar.NameBind:
+		return nil, checkBind(t, fields)
+	case grammar.PathRead:
+		return checkReads(t, fields, false)
+	}
+	return nil, nil
+}
+
+// checkReads proves each name t reads, an arm or an operand, is a reference or a path into a
+// field, returning those whose head no field holds.
+func checkReads(t grammar.Token, fields map[string]node, operands bool) ([]unboundRead, error) {
+	var unbound []unboundRead
+	names := tokenReads(t)
+	for _, name := range names {
+		if grammar.IsRef(name) {
+			if _, _, err := grammar.RefShape(name); err != nil {
+				return nil, fmt.Errorf("token {%s}: %w", t.Body, err)
+			}
+			continue // resolveRefs checks its target once the tree is assembled
+		}
+		missing, err := checkArm(name, fields, !operands && len(names) == 1)
+		if err == nil {
+			continue
+		}
+		err = fmt.Errorf("token {%s}: %w", t.Body, err)
+		if !missing {
+			return nil, err
+		}
+		unbound = append(unbound, unboundRead{head: splitArm(name, nil).head, body: t.Body, err: err})
+	}
+	return unbound, nil
+}
+
+// checkBind proves a {x as name} token is spelled once, binds a reference or a path into a
+// field, and names a valid name that is no option.
+func checkBind(t grammar.Token, fields map[string]node) error {
+	ref, name := t.BoundRef, t.Bound
+	if trimmed := strings.TrimSpace(ref) + grammar.AsWord + strings.TrimSpace(name); trimmed != t.Body {
+		return fmt.Errorf("token {%s}: write {%s}", t.Body, trimmed)
+	}
+	if err := checkBound(ref, fields); err != nil {
+		return fmt.Errorf("token {%s}: %w", t.Body, err)
+	}
+	if name == "" {
+		return fmt.Errorf("token {%s}: a binding names nothing; write {%s as n}", t.Body, ref)
+	}
+	if isOption(name) {
+		return fmt.Errorf("token {%s}: %q is an option and can never be a name; rename it", t.Body, name)
+	}
+	if err := grammar.CheckIdentifier(name); err != nil {
+		return fmt.Errorf("token {%s}: name %w", t.Body, err)
+	}
+	return nil
+}
+
+// checkBound proves what a binding binds is a reference or a path into a field of its binder.
+func checkBound(ref string, fields map[string]node) error {
+	if grammar.IsRef(ref) {
+		_, _, err := grammar.RefShape(ref)
+		return err
+	}
+	missing, err := checkArm(ref, fields, false)
+	if missing {
+		return fmt.Errorf("%w; a binding names a field, or a reference, which starts with /, . or ..", err)
+	}
+	return err
+}
+
+// checkArm validates one sibling name or path against a template's fields.
+// wholeToken says the name is the token's entire body, so {/name} would render
+// the same value and can be offered as the reference spelling. missing says no
+// field holds the head, which a name may yet answer.
+func checkArm(name string, fields map[string]node, wholeToken bool) (missing bool, err error) {
+	a := splitArm(name, nil)
+	if err := checkSegments(a); err != nil {
+		return false, err
+	}
+	field, ok := fields[a.head]
+	if !ok {
+		if a.head == "" {
+			return false, fmt.Errorf("a name is never empty, so this token can name no field")
+		}
+		if isOption(a.head) {
+			return false, fmt.Errorf("%q is an option and can never be a field", a.head)
+		}
+		if len(fields) == 0 {
+			hint := ""
+			if wholeToken && hintableRef(name) {
+				hint = fmt.Sprintf(" — write {/%s} to reference the data", name)
+			}
+			return true, fmt.Errorf("no field %q; a token names a sibling field, and this template has none%s", a.head, hint)
+		}
+		return true, fmt.Errorf("no field %q", a.head)
+	}
+	if err := provePath(field, a.tail, a.head, 0); err != nil {
+		return false, fmt.Errorf("field %q: %w", a.head, err)
+	}
+	return false, nil
+}
+
+// hintableRef reports whether {/name} is a reference the grammar accepts, so the
+// hint never names a spelling that fails too.
+func hintableRef(name string) bool {
+	return grammar.CheckPathIdentifiers(name) == nil
+}
+
+// tokenReads lists the names t reads: a read's arms, or the operands a call's builtin reads.
+func tokenReads(t grammar.Token) []string {
+	if t.Kind == grammar.BuiltinCall {
+		return builtinfunc.Operands(t.Fn, t.Args)
+	}
+	return t.Arms
+}
+
+// formatOps is a compiled format: its ops, and the size of its literal text.
+type formatOps struct {
+	ops  []op
+	grow int
+}
+
+// compileFormat compiles t's parsed format against its resolved references and its names'
+// targets; a {ref as n} token compiles to no op. checkTokens proved every token valid.
+func compileFormat(t *template, targets map[*nameBinding]nameTarget) (formatOps, error) {
+	var c formatOps
+	for _, tok := range t.tokens {
+		o := op{Token: tok}
+		var err error
+		switch tok.Kind {
+		case grammar.LiteralRun:
+			c.grow += len(tok.Lit)
+		case grammar.BuiltinCall:
+			o.call = builtinfunc.Prep(tok.Fn, tok.Args)
+			o.operands, err = t.compileArms(builtinfunc.Operands(tok.Fn, tok.Args), targets)
+		case grammar.PathRead:
+			o.arms, err = t.compileArms(tok.Arms, targets)
+		case grammar.NameBind:
+			continue
+		}
+		if err != nil {
+			return c, fmt.Errorf("token {%s}: %w", tok.Body, err)
+		}
+		c.ops = append(c.ops, o)
+	}
+	return c, nil
+}

format_stability_test.go

@@ -0,0 +1,54 @@
+package fejkdata
+
+import "testing"
+
+// TestSeededOutputIsStable pins the exact bytes a seeded generator emits for each
+// piece of the token grammar, so a change to how a format is scanned or rendered
+// cannot quietly shift the rng stream that reproducibility depends on.
+func TestSeededOutputIsStable(t *testing.T) {
+	dir := writeFiles(t, map[string]string{
+		"alt.json":          `{"format":"{a|b}","a":"A","b":"B"}`,
+		"calc.json":         `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"classes.json":      `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`,
+		"escapes.json":      `{"format":"01Aa#{x}","x":"!"}`,
+		"funcs.json":        `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`,
+		"municipality.json": `{"format":"{name}","rows":"municipality.tsv","key":"code","name":"name","parent":"region"}`,
+		"municipality.tsv":  "code\tname\tregion\n0180\tStockholm\t01\n0184\tSolna\t01\n1280\tMalmö\t12\n1281\tLund\t12\n",
+		"named.json":        `{"format":"{p as q}{q.a}{q.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"nested.json":       `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`,
+		"paths.json":        `{"format":"{p.a}{p.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"ref.json":          `"see {/alt}"`,
+		"refpath.json":      `"{/region.municipality.name} in {/region.name}"`,
+		"region.json":       `{"format":"{name}","rows":"region.tsv","key":"code","name":"name","weight":"people"}`,
+		"region.tsv":        "code\tname\tpeople\n01\tStockholms län\t3\n12\tSkåne län\t1\n",
+		"repeat.json":       `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`,
+		"sums.json":         `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`,
+		"weights.json":      `[{"format":"big","weight":9},"tiny"]`,
+	})
+	want := map[string][]string{
+		"alt":                            {"A", "A", "B", "A"},
+		"calc":                           {"100.00 x 3 = 700.00", "5.00 x 7 = 35.00", "5.00 x 3 = 35.00", "100.00 x 2 = 139.93"},
+		"classes":                        {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"},
+		"escapes":                        {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"},
+		"funcs":                          {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"},
+		"named":                          {"ab", "ab", "AB", "AB"},
+		"nested":                         {"i-73", "i-23", "i-67", "i-85"},
+		"paths":                          {"aB", "aB", "ab", "AB"},
+		"paths.p.a":                      {"A", "a", "a", "A"},
+		"ref":                            {"see A", "see A", "see A", "see B"},
+		"refpath":                        {"Solna in Skåne län", "Solna in Stockholms län", "Solna in Stockholms län", "Stockholm in Stockholms län"},
+		"region.municipality[0184].name": {"Solna", "Solna", "Solna", "Solna"},
+		"region[12].municipality.name":   {"Malmö", "Malmö", "Lund", "Malmö"},
+		"repeat":                         {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"},
+		"sums":                           {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"},
+		"weights":                        {"big", "big", "big", "big"},
+	}
+	for path := range want {
+		f := newGenerator(t, dir, WithSeed(42))
+		for i, expect := range want[path] {
+			if got := fake(t, f, path); got != expect {
+				t.Errorf("%s draw %d = %q, want %q", path, i, got, expect)
+			}
+		}
+	}
+}

format_test.go

@@ -0,0 +1,132 @@
+package fejkdata
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func TestStringIsAFormat(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`"Malmö"`:                  "Malmö",
+		`"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0",
+		`"{{x}}"`:                  "{x}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) {
+		t.Errorf(`render("{digits(3)}") = %q, want three digits`, got)
+	}
+	if _, err := resolved(t, `"{x}"`); err == nil || !strings.Contains(err.Error(), `no field "x"`) {
+		t.Errorf(`compile("{x}") = %v, want a no-field error`, err)
+	}
+}
+
+func TestTextIsLiteral(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" {
+		t.Fatalf("text = %q, want it verbatim", got)
+	}
+}
+
+func TestBraceEscapes(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`{"format":"{{","x":"v"}`:         "{",
+		`{"format":"}}","x":"v"}`:         "}",
+		`{"format":"{{x}}","x":"v"}`:      "{x}",
+		`{"format":"{{{x}}}","x":"v"}`:    "{v}",
+		`{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	for src, want := range map[string]string{
+		`"x}y"`:                      "}}",
+		`"}"`:                        "}}",
+		`{"format":"{a{b}","a":"Q"}`: "'{'",
+		`"{x"`:                       "unterminated",
+	} {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want)
+		}
+	}
+}
+
+func TestTokenSubstitution(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" {
+		t.Fatalf("token = %q, want Eriksson", got)
+	}
+}
+
+func TestAlternationDrawsOneField(t *testing.T) {
+	f := engine(3)
+	seen := map[string]bool{}
+	for i := 0; i < 100; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || len(seen) != 2 {
+		t.Fatalf("alternation produced %v, want both A and B", seen)
+	}
+}
+
+func TestAlternationThreeWay(t *testing.T) {
+	f := engine(4)
+	seen := map[string]bool{}
+	for i := 0; i < 200; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 {
+		t.Fatalf("3-way alternation produced %v, want A, B and C", seen)
+	}
+}
+
+func TestHashIsLiteral(t *testing.T) {
+	f := engine(1)
+	cases := map[string]string{
+		`{"format":"","x":"v"}`:         "",
+		`{"format":"#","x":"v"}`:        "#",
+		`{"format":"##","x":"v"}`:       "##",
+		`{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a",
+		`{"format":"#{x}","x":"v"}`:     "#v",
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Errorf("render(%s) = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestMultibyteFormat(t *testing.T) {
+	// Scanning is rune-aware: multibyte literals coexist with class chars and
+	// tokens without corrupting indices.
+	got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`)
+	if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) {
+		t.Fatalf("multibyte format = %q", got)
+	}
+}
+
+func TestFormatCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{x"`,                       // unterminated brace
+		`"x}y"`,                      // a lone } must be written }}
+		`{"format":"{a{b}","a":"Q"}`, // a brace inside a token
+		`"{x}"`,                      // a bare string has no fields to name
+		`{"format":"{y}","x":"Q"}`,   // token names a missing field
+		`"{}"`,                       // empty token name
+		`{"format":"{a|}","a":"Q"}`,  // empty alternation segment
+		`"{luhn(}"`,                  // malformed function token
+		`"{lower()}"`,                // wrong arity
+		`"{nope()}"`,                 // unknown function
+		`"{luhn(x)}"`,                // function given args it takes none of
+		`"{int(1)}"`,                 // wrong arity
+		`"{float(1,2)}"`,             // wrong arity
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}

go.mod

@@ -1,3 +1,3 @@
-module gitea.larvit.se/larvit/fejkdata
+module github.com/larvit/fejkdata
 
 go 1.22

graph.go

@@ -2,128 +2,99 @@ package fejkdata
 
 import (
 	"fmt"
-	"sort"
+	"slices"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// walkNodes calls fn once per contained node, passing the dot path that reaches it,
+// eachNode visits n and every node inside it once, and passes fn each node with the label an
-// visiting keys in sorted order so which of several broken nodes gets reported does
+// error uses for it: the dot path reaching it, then ", line N" for a table cell. It never
-// not depend on map iteration.
+// crosses a reference edge — a {/path} reference is skipped — so a single inline node is
-func walkNodes(root map[string]node, fn func(path string, n node) error) error {
+// walked on its own.
-	for _, name := range sortedNames(root) {
+func eachNode(n node, label string, fn func(label string, n node) error) error {
-		if err := eachNode(root[name], name, fn); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// eachNode visits n and every node contained within it once, passing the dot path
-// that reaches each. It never crosses a reference edge — a bound {/path} field is
-// skipped — so a single inline node is walked on its own.
-func eachNode(n node, path string, fn func(path string, n node) error) error {
 	seen := map[node]bool{}
 	var visit func(string, node) error
-	visit = func(path string, m node) error {
+	visit = func(label string, m node) error {
 		if m == nil || seen[m] {
 			return nil
 		}
 		seen[m] = true
-		if err := fn(path, m); err != nil {
+		if err := fn(label, m); err != nil {
 			return err
 		}
 		for _, c := range contained(m) {
-			if err := visit(join(path, c.name), c.node); err != nil {
+			if err := visit(c.labelIn(label), c.node); err != nil {
 				return err
 			}
 		}
 		return nil
 	}
-	return visit(path, n)
+	return visit(label, n)
 }
 
-// namedNode is a contained child and the segment reaching it; a choice's items carry
+// containedNode is a contained child and the segment reaching it; a choice's items carry
 // no segment, matching how a dot path steps over a choice.
-type namedNode struct {
+type containedNode struct {
 	name string
 	node node
+	line int // a table cell's line in its rows file, else 0
 }
 
-func contained(n node) []namedNode {
+func (c containedNode) labelIn(label string) string {
+	if c.line > 0 {
+		return fmt.Sprintf("%s, line %d", label, c.line)
+	}
+	return join(label, c.name)
+}
+
+func contained(n node) []containedNode {
 	switch n := n.(type) {
 	case *folder:
-		return named(n.children)
+		return containedByName(n.children)
 	case *choice:
-		out := make([]namedNode, len(n.items))
+		out := make([]containedNode, len(n.items))
 		for i, it := range n.items {
-			out[i] = namedNode{node: it}
+			out[i] = containedNode{node: it}
 		}
 		return out
 	case *template:
-		return named(n.fields)
+		return containedByName(n.fields)
 	case *table:
-		return append([]namedNode{{node: n.format}}, named(n.fields)...)
+		return append([]containedNode{{node: n.formatTemplate}}, containedByName(n.formatTemplate.fields)...)
-	case *column:
+	case *tableColumn:
-		if len(n.t.tokens) == 0 {
+		if len(n.t.cellTemplates) == 0 {
 			return nil
 		}
-		var out []namedNode
+		var out []containedNode
-		for r := 0; r < n.t.rows(); r++ {
+		for r := 0; r < n.t.rows.Len(); r++ {
-			if cell := n.t.cellNode(r, n.i); cell != nil {
+			if cell := n.t.cellTemplate(r, n.i); cell != nil {
-				out = append(out, namedNode{node: cell})
+				out = append(out, containedNode{node: cell, line: r + 2})
 			}
 		}
 		return out
-	default:
+	case *nullItem, *tableRow:
 		return nil
+	default:
+		panic(invariant.Broken("contained has no case for node %T", n))
 	}
 }
 
-// named skips a bound {/path} key: it is a render edge, not containment, so using
+func containedByName(m map[string]node) []containedNode {
-// it as a path segment would report a node under a path that does not reach it. Only
+	out := make([]containedNode, 0, len(m))
-// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a
-// folder's children never carry the prefix — so this one skip serves both.
-func named(m map[string]node) []namedNode {
-	out := make([]namedNode, 0, len(m))
 	for _, name := range sortedNames(m) {
-		if isRef(name) {
+		out = append(out, containedNode{name: name, node: m[name]})
-			continue
-		}
-		out = append(out, namedNode{name: name, node: m[name]})
 	}
 	return out
 }
 
-func sortedNames(m map[string]node) []string {
-	names := make([]string, 0, len(m))
-	for name := range m {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return names
-}
-
 // renderEdge is a child a node renders into, labelled by what reaches it (a field
-// name, reference, or choice index) for a readable cycle report. operand names
+// name, reference, or choice index) for a readable cycle report.
-// the builtin when the label is its operand rather than a token, so an error can
-// name it the way the author wrote it.
 type renderEdge struct {
-	to      node
+	to    node
-	label   string
+	label string
-	operand string
 }
 
-// reached names an edge as the author spelled it, the vocabulary boundReaders uses
+// renderEdges lists the children rendering n recurses into, mirroring render.
-// for the sibling fence.
-func (e renderEdge) reached() string {
-	if e.operand != "" {
-		return fmt.Sprintf("%s operand %q", e.operand, e.label)
-	}
-	return "{" + e.label + "}"
-}
-
-// renderEdges lists the children rendering n recurses into, mirroring expand: a
-// choice's items, and a template's field/reference tokens plus its operands. A
-// folder renders nothing, so it has no edges.
 func renderEdges(n node) []renderEdge {
 	switch n := n.(type) {
 	case *choice:
@@ -134,120 +105,40 @@ func renderEdges(n node) []renderEdge {
 		return es
 	case *template:
 		var es []renderEdge
-		add := func(name, operand string) {
+		for _, o := range n.compiled.ops {
-			a := splitArm(name, n.refs)
+			for _, a := range slices.Concat(o.operands, o.arms) {
-			c, ok := n.fields[a.key]
+				for _, leaf := range a.leaves {
-			if !ok {
+					es = append(es, renderEdge{leaf, a.spelling})
-				return
+				}
-			}
-			for _, leaf := range pathLeaves(c, a.tail) {
-				es = append(es, renderEdge{leaf, name, operand})
 			}
 		}
-		_ = eachToken(n.format, func(t ftoken) error {
-			if t.kind != 'b' {
-				return nil
-			}
-			if fn, _, isFunc := funcCall(t.body); isFunc {
-				for _, operand := range tokenOperands(t.body) {
-					add(operand, fn)
-				}
-				return nil
-			}
-			for _, name := range splitOutside(t.body, '|') {
-				add(name, "")
-			}
-			return nil
-		})
 		return es
 	case *table:
-		return []renderEdge{{to: n.format, label: "format"}}
+		return []renderEdge{{to: n.formatTemplate, label: "format"}}
-	case *row:
+	case *tableRow:
-		return []renderEdge{{to: n.t.format, label: "format"}}
+		return []renderEdge{{to: n.t.formatTemplate, label: "format"}}
-	case *column:
+	case *tableColumn:
 		var es []renderEdge
 		for _, c := range contained(n) {
-			es = append(es, renderEdge{to: c.node, label: n.t.columns[n.i]})
+			es = append(es, renderEdge{to: c.node, label: n.name()})
 		}
 		return es
-	default:
+	case *folder, *nullItem:
 		return nil
+	default:
+		panic(invariant.Broken("renderEdges has no case for node %T", n))
 	}
 }
 
-// pathLeaves lists what a token's dotted tail renders: a choice on the way
+type renderCounts map[node]int
-// contributes every variant, since any of them may be the one drawn. checkPath has
-// already proved the tail resolves in every variant.
-func pathLeaves(n node, tail []string) []node {
-	var out []node
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil },
-		leaf:   func(n node) error { out = append(out, n); return nil },
-	})
-	return out
-}
 
-// nodeScope is the set of nodes one validation pass covers: a whole loaded tree,
+func (m renderCounts) renderCount(n node) int {
-// or a single inline node.
-type nodeScope func(fn func(path string, n node) error) error
-
-func treeScope(root map[string]node) nodeScope {
-	return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
-}
-
-func inlineScope(n node, label string) nodeScope {
-	return func(fn func(path string, m node) error) error { return eachNode(n, label, fn) }
-}
-
-func checkScope(s nodeScope) error {
-	if err := checkColumns(s); err != nil {
-		return err
-	}
-	return checkRenders(s)
-}
-
-// checkRenders runs the per-node fences over a scope, each over the whole scope
-// before the next, so which of several broken nodes is reported does not depend on
-// the walk. It runs after checkNoCycles, whose guarantee is what lets the walks
-// terminate.
-func checkRenders(s nodeScope) error {
-	mem := reachMemo{}
-	if err := s(func(path string, n node) error { return repeatCheck(path, n, mem) }); err != nil {
-		return err
-	}
-	if err := s(heldCheck); err != nil {
-		return err
-	}
-	fence := &drawCheck{}
-	refs := false
-	if err := s(func(path string, n node) error {
-		if t, ok := n.(*template); ok && len(t.refs) > 0 {
-			refs = true
-		}
-		return fence.checkDrawGroup(path, n)
-	}); err != nil {
-		return err
-	}
-	if refs {
-		if err := s(fence.checkDraws); err != nil {
-			return err
-		}
-		if err := s(fence.checkRecordDraws); err != nil {
-			return err
-		}
-	}
-	return s((&valueProof{}).checkDatatype)
-}
-
-type reachMemo map[node]int
-
-func (m reachMemo) of(n node) int {
 	if r, done := m[n]; done {
 		return r
 	}
 	r := 1
 	for _, e := range renderEdges(n) {
-		if c := m.of(e.to); c > r {
+		if c := m.renderCount(e.to); c > r {
 			r = c
 		}
 	}
@@ -260,9 +151,10 @@ func (m reachMemo) of(n node) int {
 
 // repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
 // path, so nested repeats cannot build what one repeat may not.
-func repeatCheck(path string, n node, mem reachMemo) error {
+// docs/decisions.md#the-repeat-cap-bounds-renders-not-bytes
-	if t, ok := n.(*template); ok && t.repeat > 1 && mem.of(n) > MaxRepeat {
+func repeatCheck(label string, n node, mem renderCounts) error {
-		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, mem.of(n), MaxRepeat)
+	if t, ok := n.(*template); ok && t.repeat > 1 && mem.renderCount(n) > MaxRepeat {
+		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", label, t.repeat, mem.renderCount(n), MaxRepeat)
 	}
 	return nil
 }
@@ -270,32 +162,42 @@ func repeatCheck(path string, n node, mem reachMemo) error {
 // checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
 // — directly, mutually, or through a chain — never terminates, so it must fail at
 // New rather than stack-overflow at render. It is a depth-first walk of the render
-// graph (renderEdges); grey marks nodes on the current path so a back-edge to one
+// graph (renderEdges). Every node is a root: a field its parent's format never
-// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped.
+// renders is still reachable by dot path, so a cycle in one would otherwise reach
-// Every node is a root: a field its parent's format never renders is still reachable
+// render and be fatal there.
-// by dot path, so a cycle in one would otherwise reach render and be fatal there.
+func checkNoCycles(s nodeSet) error {
-func checkNoCycles(root map[string]node) error {
 	const (
 		grey  = 1
 		black = 2
 	)
+	inSet := map[node]bool{}
+	_ = s(func(_ string, n node) error {
+		inSet[n] = true
+		if t, isTable := n.(*table); isTable {
+			inSet[t.rowNode] = true // a selector lands on it, and contained lists it nowhere
+		}
+		return nil
+	})
 	color := map[node]int{}
-	var visit func(n node, path string) error
+	var visit func(n node, label string) error
-	visit = func(n node, path string) error {
+	visit = func(n node, label string) error {
+		if !inSet[n] {
+			return nil // loaded before these nodes and proven then, so it never reaches back into it
+		}
 		switch color[n] {
 		case grey:
-			return fmt.Errorf("reference cycle: %s", path)
+			return fmt.Errorf("reference cycle: %s", label)
 		case black:
 			return nil
 		}
 		color[n] = grey
 		for _, e := range renderEdges(n) {
-			if err := visit(e.to, path+" -> "+e.label); err != nil {
+			if err := visit(e.to, label+" -> "+e.label); err != nil {
 				return err
 			}
 		}
 		color[n] = black
 		return nil
 	}
-	return walkNodes(root, func(path string, n node) error { return visit(n, path) })
+	return s(func(label string, n node) error { return visit(n, label) })
 }

hold.go

@@ -1,358 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// heldCheck rejects every route to a held sibling name except the ones that read its
-// draw. An expansion holds one draw of that name; anything else that renders it draws
-// again, and the two disagree. checkNoOverlap settles the spellings within one format
-// (a token, an operand); this settles the rest — a reference, whether it sits in that
-// format or in anything the format renders, however deep. A reference path is held
-// for the whole render instead, which drawCheck fences.
-func heldCheck(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || len(t.held) == 0 {
-		return nil
-	}
-	readers := boundReaders(t.format, t.bound, t.refs)
-	for _, head := range heldHeads(t) {
-		if _, isPath := t.bound[head]; isPath && isRef(head) {
-			continue
-		}
-		if err := checkHeadHeld(t, head, readers); err != nil {
-			return fmt.Errorf("%s: %w", path, err)
-		}
-	}
-	return nil
-}
-
-// heldHeads lists a template's held names, operand heads first, then paths, each
-// in name order: a level read both ways is reported by the operand's fence, and
-// which overlap is reported does not vary.
-func heldHeads(t *template) []string {
-	heads := make([]string, 0, len(t.held))
-	for head := range t.held {
-		heads = append(heads, head)
-	}
-	sort.Slice(heads, func(i, j int) bool {
-		_, pi := t.bound[heads[i]]
-		_, pj := t.bound[heads[j]]
-		if pi != pj {
-			return !pi
-		}
-		return heads[i] < heads[j]
-	})
-	return heads
-}
-
-// pinned collects what the hold of head answers for: a path pins the levels it
-// passes through and the leaf it lands on, an operand exactly the value its render
-// produces.
-func pinned(t *template, head string, readers []reader) map[node]bool {
-	held := map[node]bool{}
-	if _, isPath := t.bound[head]; !isPath {
-		operandDraw(t.fields[head], held)
-		return held
-	}
-	for _, r := range readers {
-		if a := splitArm(r.name, t.refs); a.key == head {
-			coverPath(t.fields[head], a.tail, held)
-		}
-	}
-	return held
-}
-
-// checkHeadHeld rejects every route to what head's hold pins except the readers
-// holding it. One seen set across the edges: a node that cannot reach the level
-// cannot reach it by another route either, so it is walked once.
-func checkHeadHeld(t *template, head string, readers []reader) error {
-	held := pinned(t, head, readers)
-	if len(held) == 0 {
-		return nil // a fixed head holds nothing to reach
-	}
-	reader, isPath := t.bound[head]
-	seen := map[node]bool{}
-	for _, e := range renderEdges(t) {
-		if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) {
-			continue
-		}
-		if isPath {
-			return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader)
-		}
-		return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head))
-	}
-	return nil
-}
-
-// operandReader names the builtin whose operand holds head.
-func operandReader(t *template, head string) string {
-	fn := ""
-	_ = eachToken(t.format, func(tok ftoken) error {
-		if tok.kind != 'b' || fn != "" {
-			return nil
-		}
-		if name, _, isFunc := funcCall(tok.body); isFunc {
-			for _, operand := range tokenOperands(tok.body) {
-				if splitArm(operand, t.refs).key == head {
-					fn = name
-				}
-			}
-		}
-		return nil
-	})
-	return fn
-}
-
-// coverPath collects what holding one path pins: every choice level the path
-// passes through, whole, and the leaf it renders.
-func coverPath(n node, tail []string, into map[node]bool) {
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil },
-		leaf:   func(n node) error { cover(n, into, false); return nil },
-	})
-}
-
-// cover collects a level and everything contained in it. A fixed string outside a
-// choice is left out — it cannot disagree with itself — but inside one each
-// variant carries its own, so there it counts.
-func cover(n node, into map[node]bool, inChoice bool) {
-	if isFixed(n) && !inChoice {
-		return
-	}
-	into[n] = true
-	_, isChoice := n.(*choice)
-	for _, c := range contained(n) {
-		cover(c.node, into, inChoice || isChoice)
-	}
-}
-
-// operandDraw collects what one held draw of an operand answers for: the operand
-// and what rendering it settles inside itself. The builtin renders its operand
-// whole, so that draw fixes every value the render produced, and a second route to
-// any of them disagrees with it.
-//
-// The walk stops at a reference edge, which is where the operand's own value ends
-// and a shared source begins: two names referencing one category are two draws, the
-// same rule {word} {word} follows.
-func operandDraw(n node, into map[node]bool) {
-	if isFixed(n) {
-		return
-	}
-	if into[n] {
-		return
-	}
-	into[n] = true
-	for _, e := range renderEdges(n) {
-		if isRef(e.label) {
-			continue
-		}
-		operandDraw(e.to, into)
-	}
-}
-
-// isFixed is a string that varies nothing: fixed text with no fields to read into.
-func isFixed(n node) bool {
-	t, ok := n.(*template)
-	return ok && t.fixed && len(t.fields) == 0
-}
-
-// renders reports whether rendering n can reach anything in want, following the
-// same edges expand does. seen keeps a node shared by several routes from being
-// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk
-// ends.
-func renders(n node, want, seen map[node]bool) bool {
-	if want[n] {
-		return true
-	}
-	if seen[n] {
-		return false
-	}
-	seen[n] = true
-	for _, e := range renderEdges(n) {
-		if renders(e.to, want, seen) {
-			return true
-		}
-	}
-	return false
-}
-
-// checkNoOverlap rejects a format that both renders a sibling level and reads a path
-// into it — {p} beside {p.first}, {p.addr} beside {p.addr.city}. The path reads the
-// level's held draw while rendering the level expands it afresh, so their values would
-// disagree. Reads are compared in sorted order, so which pair is reported does not
-// depend on where the tokens sit.
-func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
-	names := boundReaders(format, bound, refs)
-	// Stable over one format-order scan, so two readers of one name (a token and a
-	// calc operand both naming "p") are reported as the format writes them.
-	sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
-	for i, level := range names {
-		for _, path := range names[i+1:] {
-			if strings.HasPrefix(path.path, level.path+".") {
-				return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name)
-			}
-		}
-	}
-	return nil
-}
-
-// reader is one way a format reaches a bound field: as written, by its one
-// spelling, and how to name it.
-type reader struct{ name, path, label string }
-
-// boundReaders lists every way a format reaches a bound sibling field, in the order the
-// format writes them. An operand renders its field, so it names a level exactly
-// as a token does; one scan finds both, which is what puts them in one order.
-func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
-	var names []reader
-	_ = eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if fn, _, isFunc := funcCall(t.body); isFunc {
-			for _, operand := range tokenOperands(t.body) {
-				a := splitArm(operand, refs)
-				if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-					names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
-				}
-			}
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-				names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
-			}
-		}
-		return nil
-	})
-	return names
-}
-
-// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside
-// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
-// error names the single-token spelling.
-func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
-	count := map[string]int{}
-	return eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if _, _, isFunc := funcCall(t.body); isFunc {
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if len(a.tail) > 0 || !c.held[a.key] {
-				continue
-			}
-			if count[a.key]++; count[a.key] > 1 {
-				return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
-			}
-		}
-		return nil
-	})
-}
-
-// draws is what has already been drawn for held names: the variant each was drawn as,
-// so every path under it reads one row, and the draw each read made, by its one
-// spelling, so the same read written twice reads one draw. An expansion keeps one for
-// its sibling names, and a render one per group for its reference paths.
-type draws struct {
-	variant map[string]node
-	value   map[string]draw
-	pins    [8]tablePin // the rows pinned, inline so a render over a country's five-deep geo tree stays off the heap
-	npins   int
-	more    map[*table]int // the rows pinned past the inline eight
-	s       *session       // what draws a row; nil where a walk only proves selectors
-}
-
-// draw is what one read drew: its text, and whether it landed on a null.
-type draw struct {
-	text string
-	null bool
-}
-
-// readField renders one arm of a token. An arm's key is a sibling field or a
-// reference linkRefs bound into fields. A name the expansion holds — a level some
-// token addresses by dotted path, or a field an operand reads — is drawn once and
-// kept, so {place.postal-code} and {place.locality} read one row, either read twice
-// gives one value, and a shown operand is the operand computed. Every other name is
-// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
-// linkRefs prove every step, so the walk cannot fail.
-func readField(s *session, t *template, held *draws, sc drawScope, a arm) draw {
-	if !t.held[a.key] {
-		if len(a.tail) > 0 {
-			panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
-		}
-		return draw{text: render(s, t.fields[a.key], sc)}
-	}
-	d := readScope(s, held, sc, a)
-	if r, done := d.value[a.path]; done {
-		return r
-	}
-	leaf, err := walkPath(t.fields[a.key], a.tail, pathWalk{
-		// Hold the draw at every level passed through, so two paths sharing a
-		// prefix share it.
-		choice: func(c *choice, rest []string) ([]node, error) {
-			key := a.key
-			if consumed := len(a.tail) - len(rest); consumed > 0 {
-				key = a.steps[consumed-1]
-			}
-			n, drew := d.variant[key]
-			if !drew {
-				n = drawn(s, c)
-				if d.variant == nil {
-					d.variant = map[string]node{}
-				}
-				d.variant[key] = n
-			}
-			return []node{n}, nil
-		},
-		pins: d,
-	})
-	if err != nil {
-		panic(fmt.Sprintf("fejkdata: %q: %v; a fence should have refused this at New", a.name, err))
-	}
-	r := renderLeaf(s, leaf, sc)
-	if d.value == nil {
-		d.value = map[string]draw{}
-	}
-	d.value[a.path] = r
-	return r
-}
-
-// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
-// the render's draws for its group, so its draw spans the render; for a sibling, or a
-// reference read whole, held.
-func readScope(s *session, held *draws, sc drawScope, a arm) *draws {
-	if isRef(a.key) && len(a.tail) > 0 {
-		return sc.draws(s)
-	}
-	return held
-}
-
-// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
-// reference alone whose read drew null.
-func renderLeaf(s *session, n node, sc drawScope) draw {
-	n = drawn(s, n)
-	if _, isNull := n.(*null); isNull {
-		return draw{null: true}
-	}
-	r := draw{text: render(s, n, sc)}
-	if t, _ := n.(*template); t != nil && t.readsColumn != nil {
-		r.null = sc.in(t).draws(s).value[t.readsColumn.a.path].null
-	}
-	return r
-}
-
-// drawn resolves a choice to one variant, so a bound head is a concrete node the
-// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not
-// another set to pick from.
-func drawn(s *session, n node) node {
-	for c, ok := n.(*choice); ok; c, ok = n.(*choice) {
-		n = pick(s, c)
-	}
-	return n
-}

hold_test.go

@@ -1,771 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-)
-
-// This file pins the one-draw rule for dotted sibling tokens. A format that
-// addresses a sibling by path ({place.locality}) draws that sibling once per
-// expansion, so every {place.*} token in the same format reads the same variant.
-// That is what makes a correlated pair — a locality and a postal code that agree
-// — expressible in data, without the row having to be the whole node.
-
-// swedishPlaces is a two-variant sibling whose variants pair a locality with the
-// postal-code prefix that really belongs to it.
-const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`
-
-// agree reports whether a rendered "postcode locality" pair is a real pairing.
-var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`)
-
-func places(format string) string {
-	return strings.Replace(swedishPlaces, "%s", format, 1)
-}
-
-func TestDottedTokenCorrelatesSiblingFields(t *testing.T) {
-	f := engine(1)
-	seen := map[string]bool{}
-	for i := 0; i < 500; i++ {
-		got := mustRender(t, f, places("{place.postal-code} {place.locality}"))
-		if !agree.MatchString(got) {
-			t.Fatalf("draw %d = %q, want a postcode and locality that pair", i, got)
-		}
-		seen[got[len(got)-4:]] = true
-	}
-	if len(seen) < 2 {
-		t.Fatalf("500 draws produced only %v, want both localities", seen)
-	}
-}
-
-func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) {
-	// One draw, one spelling. A token that renders a level and one that reads a
-	// path into it name the same draw two ways, and only the path reads what the
-	// other rendered — so the pair is a load error rather than a shape whose two
-	// halves can disagree.
-	rejected := map[string]string{
-		"bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-		"prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` +
-			`{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`,
-		"either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-	}
-	for name, file := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), "reads a path into") {
-			t.Errorf("%s: New = %v, want it to name the overlap", name, err)
-		}
-	}
-	// The same path twice is one spelling, so it stays legal.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-	}))); err != nil {
-		t.Errorf("New = %v, want one path read twice accepted", err)
-	}
-}
-
-func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) {
-	// A calc operand renders its field, so naming a bound level in one is the same
-	// overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a
-	// path into, and the two disagree.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` +
-			`{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-		t.Fatalf("New = %v, want the calc operand rejected as an overlap", err)
-	}
-	// Arithmetic over fields no path names is untouched.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`,
-	}))); err != nil {
-		t.Errorf("New = %v, want plain arithmetic accepted", err)
-	}
-}
-
-func TestReferenceNamingABoundLevelIsRejected(t *testing.T) {
-	// A reference can name a bound level from the data root, which renders it
-	// afresh beside the path that reads its held draw — the same overlap by
-	// another spelling.
-	// A category never references itself, so each reference back into cat sits in a
-	// second category cat renders.
-	rejected := map[string]map[string]string{
-		"reference names the head": {
-			"cat": `{"format":"{p.first}|{/hop}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-			"hop": `"{/cat.p}"`,
-		},
-		"reference names the leaf": {
-			"cat": `{"format":"{p.addr}|{/hop}","p":{"format":"x","addr":["A","B","C","D"]}}`,
-			"hop": `"{/cat.p.addr}"`,
-		},
-		// The reference need not sit in the format that binds: any field it renders
-		// reaches the level just the same, however deep.
-		"reference from a sibling field": {
-			"cat": `{"format":"{p.first}|{inner}","p":[` +
-				`{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` +
-				`"inner":"{/hop}"}`,
-			"hop": `"{/cat.p}"`,
-		},
-	}
-	for name, files := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
-		if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-			t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err)
-		}
-	}
-	// A reference to anything this format does not bind is untouched.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":     `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
-		"surname": `["Eriksson","Lindqvist"]`,
-	}))); err != nil {
-		t.Errorf("New = %v, want a reference outside the bound level accepted", err)
-	}
-}
-
-func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) {
-	// A path token renders what it lands on, not the level it started from, so the
-	// cycle walk has to follow it there. A head whose own format names nothing
-	// would otherwise hide the cycle until render, where it is fatal.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"a": `{"format":"{p.x}","p":{"format":"static","x":"{/b}"}}`,
-		"b": `"{/a}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reference cycle") {
-		t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
-	}
-}
-
-func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) {
-	// {p.a} renders q, so it is a route to the level {q.x} holds — even though p's
-	// own format names nothing.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/hop}"},` +
-			`"q":{"format":"{x}","x":["1","2"]}}`,
-		"hop": `"{/thing.q}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reads a path into") {
-		t.Fatalf("New = %v, want the second route to q rejected", err)
-	}
-}
-
-// TestACalcOperandIsHeldAgainstEveryRoute pins what a {calc()} operand's hold
-// fences. A calc renders its operand whole, so the draw it holds is that one node's
-// value: another route conflicts only by naming that same node, and then the value
-// shown is not the value computed. Two names that merely draw from one source are
-// two draws, as {word} {word} is — the rule everywhere else in the engine.
-func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) {
-	rejected := map[string]struct {
-		files map[string]string
-		want  string // the route the error names, spelled as the author wrote it
-	}{
-		// A category never references itself, so each route back into cat sits in a
-		// second category cat renders.
-		"a reference beside the operand": {
-			map[string]string{
-				"cat": `{"format":"{/hop} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
-				"hop": `"{/cat.net}"`,
-			},
-			`{/hop} renders "net"`,
-		},
-		"a reference one level down": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.net}"`,
-			},
-			`{q} renders "net"`,
-		},
-		// A reference ending at the operand binds the choice itself, not a variant,
-		// so wrapping the operand in one changes nothing.
-		"a reference to an operand wrapped in a choice": {
-			map[string]string{
-				"cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.n}"`,
-			},
-			`{q} renders "n"`,
-		},
-		// The reaching route can be another operand, which is not a token — so the
-		// error must name it as one rather than invent a {b} the format never wrote.
-		"an operand reaching another operand": {
-			map[string]string{
-				"cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` +
-					`"b":"{/hop}"}`,
-				"hop": `"{/cat.a}"`,
-			},
-			`calc operand "b" renders "a"`,
-		},
-		// Reaching *into* the operand is the same shape as naming it: the draw the
-		// calc holds settled that value too. The token spelling {net.v} is already
-		// rejected, so the reference spelling has to be.
-		"a reference into the operand": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)}|{/hop}",` +
-					`"net":{"format":"{v}","v":["10.00","20.00"]}}`,
-				"hop": `"{/cat.net.v}"`,
-			},
-			`{/hop} renders "net"`,
-		},
-		"a reference into the operand one level down": {
-			map[string]string{
-				"cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` +
-					`"q":"{/hop}"}`,
-				"hop": `"{/cat.net.v}"`,
-			},
-			`{q} renders "net"`,
-		},
-		// Each head is walked against its own seen set. Sharing one across heads
-		// would let the first head's walk mark the only route to the second's draw,
-		// and this violation sits on the later head in sorted order.
-		"a violation on the second of two held heads": {
-			map[string]string{
-				"cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` +
-					`"b":{"format":"{y}","y":["3","4"]},"w":"{/hop}"}`,
-				"hop": `"{/cat.b}"`,
-			},
-			`{w} renders "b"`,
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
-		if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") {
-			t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err)
-			continue
-		}
-		if !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to name %q", name, err, c.want)
-		}
-	}
-
-	accepted := map[string]map[string]string{
-		// The spellings that read the one draw: the operand itself, and a bare token.
-		"only the operand and a bare token": {
-			"cat": `{"format":"{net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
-		},
-		// A sibling the operand's format never renders is no part of its value. A
-		// path head differs — a path may read into anything the level contains —
-		// which is why that half of the fence covers containment.
-		"a reference to a sibling the operand never renders": {
-			"cat": `{"format":"{calc(net * 2, 2)} {unit}",` +
-				`"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` +
-				`"unit":"{/hop}"}`,
-			"hop": `"{/cat.net.spare}"`,
-		},
-		// Two operands drawing from one source are two names, so two draws: each is
-		// held under its own name and shown once, and neither can disagree.
-		"two operands sharing one source": {
-			"die": `["1","2","3","4","5","6"]`,
-			"cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` +
-				`"d2":"{/die}"}`,
-		},
-		// Likewise a reference drawing from what the operand draws from: {/common}
-		// and net are two names, not two spellings of one field.
-		"a reference to what an operand renders through": {
-			"common": `["1","2"]`,
-			"cat":    `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`,
-		},
-		// A fixed string cannot disagree with itself, so it needs no fence.
-		"a literal operand named twice": {
-			"cat": `{"format":"{calc(n * 2, 0)} {/hop}","n":"5"}`,
-			"hop": `"{/cat.n}"`,
-		},
-		// One node reached twice while walking the operand: the walk must not
-		// revisit it, and the repeat is not a second route to anything.
-		"an operand that renders one field twice": {
-			"cat": `{"format":"{calc(v * 2, 0)}","v":{"format":"{a}{a}","a":["1","2"]}}`,
-		},
-	}
-	for name, files := range accepted {
-		if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-			t.Errorf("%s: New = %v, want it accepted", name, err)
-		}
-	}
-}
-
-func TestAPathReachesEveryVariantItMightDraw(t *testing.T) {
-	// A path through a choice may land in any variant, so both walks have to see
-	// all of them. Reaching only the first leaves whatever hides in a later
-	// variant to be found at render — a cycle there is fatal, and a second route
-	// to a held level disagrees silently.
-	rejected := map[string]struct {
-		files map[string]string
-		want  string
-	}{
-		"a cycle in a later variant": {
-			map[string]string{
-				"cat": `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}]}`,
-				"hop": `"{/cat}"`,
-			},
-			"reference cycle",
-		},
-		"a second route in a later variant": {
-			map[string]string{
-				"cat": `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}],` +
-					`"q":{"format":"{y}","y":["1","2"]}}`,
-				"hop": `"{/cat.q}"`,
-			},
-			"reads a path into",
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
-		if err == nil || !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
-		}
-	}
-}
-
-func TestALevelAPathNeverRendersIsAccepted(t *testing.T) {
-	// A path token does not expand its head's format, so a reference sitting in
-	// that format is not a second route to anything: it is never rendered by the
-	// path at all. Both orders must load. p and q sit one level down, so they are
-	// one column of thing rather than two that would read one another.
-	accepted := map[string]map[string]string{
-		"reference in the head's own format": {
-			"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
-				`"p":{"format":"{first} {/hop}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}}`,
-			"hop": `"{/thing.inner.q}"`,
-		},
-		"the mirror shape": {
-			"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
-				`"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/hop}","a":["1","2"]}}}`,
-			"hop": `"{/thing.inner.p}"`,
-		},
-	}
-	for name, files := range accepted {
-		if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-			t.Errorf("%s: New = %v, want it accepted", name, err)
-		}
-	}
-}
-
-func TestADeepDiamondChainLoads(t *testing.T) {
-	// A diamond chain is 2^n routes through n nodes. The search past a bound level
-	// must walk each node once, not once per route, or a deep chain never loads.
-	files := map[string]string{"l0": `"x"`}
-	for i := 1; i <= 30; i++ {
-		files[fmt.Sprintf("l%d", i)] = fmt.Sprintf(
-			`{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1)
-	}
-	files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}`
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
-		t.Fatalf("New = %v, want a deep diamond chain to load", err)
-	}
-}
-
-func TestASharedNodeIsWalkedOnce(t *testing.T) {
-	// Two fields reaching one node make the render graph a diamond, not a tree.
-	// The search past a bound level must take that in its stride rather than walk
-	// the shared node once per route.
-	f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` +
-			`"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`,
-		"shared": `"x"`,
-	})), WithSeed(1))
-	if err != nil {
-		t.Fatalf("New = %v, want a shared node accepted", err)
-	}
-	if got, err := f.Fake("cat"); err != nil || !strings.HasSuffix(got, "|xx") {
-		t.Fatalf("Fake(cat) = %q, %v, want it to end in xx", got, err)
-	}
-}
-
-func TestTheEarlierReaderIsNamed(t *testing.T) {
-	// A token and a calc operand can name one level. The one the format writes
-	// first is the one reported, so the error points at the same place a reader
-	// looking at the format would start.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) {
-		t.Fatalf("New = %v, want the calc operand named, being written first", err)
-	}
-}
-
-func TestReferenceToAMatchingStringIsAccepted(t *testing.T) {
-	// A literal renders one fixed string, so no draw of it can disagree with a
-	// held one. Two unrelated literals that merely spell the same text must not
-	// read as the same node — the trap when comparing a value type.
-	if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":   `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`,
-		"other": `{"format":"x","tag":"Stockholm"}`,
-	}))); err != nil {
-		t.Fatalf("New = %v, want a reference to a matching string accepted", err)
-	}
-}
-
-func TestNestedChoiceDrawsOneVariant(t *testing.T) {
-	// A choice item may itself be a choice, so drawing a bound head unwraps until
-	// it reaches a value. Stopping at one level leaves a choice where the walk
-	// expects a template.
-	f := engine(21)
-	seen := map[string]bool{}
-	for i := 0; i < 200; i++ {
-		seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true
-	}
-	if !seen["1"] || !seen["2"] || len(seen) != 2 {
-		t.Fatalf("200 draws produced %v, want 1 and 2", seen)
-	}
-}
-
-func TestRepeatingLevelIsNamedInTheError(t *testing.T) {
-	// The level carrying the repeat is the one to fix, so the error names it
-	// rather than the head the path started from.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `"p.a"`) {
-		t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err)
-	}
-}
-
-func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) {
-	// A choice of rows is the shape this feature is for, so the repeat rule has to
-	// reach inside one — otherwise the direct spelling is a load error and the same
-	// mistake behind a choice silently drops the repeat.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "repeat") {
-		t.Fatalf("New = %v, want the repeat behind a choice rejected", err)
-	}
-}
-
-func TestPathIntoARepeatingLevelIsRejected(t *testing.T) {
-	// A path reads one level's draw, so it can never apply that level's repeat.
-	// The engine rejects options that cannot take effect, and this is one.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "repeat") {
-		t.Fatalf("New = %v, want a path into a repeating level rejected", err)
-	}
-	_, err = New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat":  `{"format":"[{p.a}]","p":{"format":"{a}","a":"{/word.w}","drawGroup":"g"}}`,
-		"word": `{"format":"{w}","w":["x","y"]}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `the level "p" carries a drawGroup`) {
-		t.Fatalf("New = %v, want a path into a level carrying a group rejected", err)
-	}
-}
-
-func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) {
-	// A head that is one template, not a choice, reports the missing segment
-	// directly — there are no variants to compare.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), `no field "nope"`) {
-		t.Fatalf("New = %v, want it to name the missing field", err)
-	}
-}
-
-func TestFieldWithoutAPathTokenStillDrawsEachTime(t *testing.T) {
-	// A format with no dotted token keeps today's behaviour: every {word} is an
-	// independent draw.
-	f := engine(3)
-	varied := false
-	for i := 0; i < 200 && !varied; i++ {
-		got := mustRender(t, f, `{"format":"{w}{w}{w}","w":["a","b","c","d","e"]}`)
-		varied = got[0] != got[1] || got[1] != got[2]
-	}
-	if !varied {
-		t.Fatal("200 draws of {w}{w}{w} never varied, want independent draws")
-	}
-}
-
-func TestTwoBoundHeadsDrawIndependently(t *testing.T) {
-	// Binding is per head, so two correlated sets in one format compose instead
-	// of needing a cross product of variants.
-	f := engine(4)
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, `{"format":"{p.v}{q.v}",
-			"p":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
-			"q":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}`)] = true
-	}
-	for _, want := range []string{"A1", "A2", "B1", "B2"} {
-		if !seen[want] {
-			t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
-		}
-	}
-}
-
-func TestRepeatRedrawsTheBinding(t *testing.T) {
-	// The binding lives for one expansion, so each repetition is a fresh draw —
-	// and each repetition is internally consistent.
-	f := engine(5)
-	tmpl := strings.Replace(places("{place.postal-code} {place.locality}"), `"format":"{place.postal-code} {place.locality}"`,
-		`"format":"{place.postal-code} {place.locality}","repeat":6,"separator":"\n"`, 1)
-	varied := false
-	for i := 0; i < 40; i++ {
-		got := mustRender(t, f, tmpl)
-		lines := strings.Split(got, "\n")
-		if len(lines) != 6 {
-			t.Fatalf("repeat 6 produced %d lines", len(lines))
-		}
-		for _, line := range lines {
-			if !agree.MatchString(line) {
-				t.Fatalf("repetition %q does not pair", line)
-			}
-		}
-		if lines[0] != lines[1] || lines[1] != lines[2] {
-			varied = true
-		}
-	}
-	if !varied {
-		t.Fatal("40 renders of repeat 6 never varied within a render, want a redraw per repetition")
-	}
-}
-
-func TestNestedTemplateKeepsItsOwnBinding(t *testing.T) {
-	// A binding belongs to the expansion that made it: an inner template's own
-	// place is not the outer one's.
-	f := engine(6)
-	tmpl := `{"format":"{place.v}{inner}",
-		"place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
-		"inner":{"format":"{place.v}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}}`
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, tmpl)] = true
-	}
-	if !seen["AB"] && !seen["BA"] {
-		t.Fatalf("400 draws produced %v, want the inner binding to be independent", seen)
-	}
-}
-
-func TestBoundDrawIsSeedStable(t *testing.T) {
-	tmpl := places("{place.postal-code} {place.locality}")
-	a := mustRender(t, engine(42), tmpl)
-	b := mustRender(t, engine(42), tmpl)
-	if a != b {
-		t.Fatalf("same seed gave %q and %q, want an identical draw", a, b)
-	}
-}
-
-func TestDottedTokenErrors(t *testing.T) {
-	// A path token is validated where every other token is: at New, naming what
-	// is wrong, so a typo can never reach a render.
-	rejected := map[string]struct {
-		file string
-		want string
-	}{
-		"unknown head": {
-			`{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`,
-			`no field "nope"`,
-		},
-		"unknown tail": {
-			`{"format":"{place.nope}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}`,
-			`"nope"`,
-		},
-		"tail only some variants carry": {
-			`{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`,
-			`locality`,
-		},
-		"path into a literal": {
-			`{"format":"{x.y}","x":"plain"}`,
-			`"y"`,
-		},
-		"dotted field key": {
-			`{"format":"[{a.b}]","a.b":"V"}`,
-			`contains "."`,
-		},
-	}
-	for name, c := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want it rejected at load", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), c.want) {
-			t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
-		}
-	}
-}
-
-func TestSamePathReadTwiceReadsOneValue(t *testing.T) {
-	// A path is drawn once per expansion, so reading it twice reads one value —
-	// what a name shown in a display form and again in an address needs.
-	f := engine(7)
-	for i := 0; i < 300; i++ {
-		got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`)
-		parts := strings.Split(got, "|")
-		if parts[0] != parts[1] {
-			t.Fatalf("draw %d = %q, want one value read twice", i, got)
-		}
-	}
-}
-
-func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) {
-	// Two tails of one head stay in the same variant, each keeping its own value.
-	f := engine(8)
-	for i := 0; i < 300; i++ {
-		got := mustRender(t, f, `{"format":"{p.a}{p.b}",
-			"p":[{"format":"x","a":"A","b":"1"},{"format":"y","a":"B","b":"2"}]}`)
-		if got != "A1" && got != "B2" {
-			t.Fatalf("draw %d = %q, want a row's own pair", i, got)
-		}
-	}
-}
-
-// deepPlaces nests the correlated facts a level down: the row holds an addr, and
-// the addr holds the city and the zip that belongs to it.
-const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}`
-
-func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) {
-	// A choice below the head is drawn once too, so two paths sharing a prefix
-	// share every level of it — not just the row they started from.
-	f := engine(11)
-	seen := map[string]bool{}
-	for i := 0; i < 500; i++ {
-		got := mustRender(t, f, strings.Replace(deepPlaces, "%s", "{p.addr.city}/{p.addr.zip}", 1))
-		if got != "Stockholm/11111" && got != "Kiruna/98100" {
-			t.Fatalf("draw %d = %q, want a city and the zip that belongs to it", i, got)
-		}
-		seen[got] = true
-	}
-	if len(seen) < 2 {
-		t.Fatalf("500 draws produced only %v, want both", seen)
-	}
-}
-
-func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) {
-	// The rule does not run out at two: every level a path passes through is held.
-	f := engine(12)
-	tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}`
-	want := map[string]bool{
-		"Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true,
-		"Malmö/21100 Skåne": true, "Lund/22100 Skåne": true,
-	}
-	seen := map[string]bool{}
-	for i := 0; i < 600; i++ {
-		got := mustRender(t, f, tmpl)
-		if !want[got] {
-			t.Fatalf("draw %d = %q, want a town with its own zip and region", i, got)
-		}
-		seen[got] = true
-	}
-	if len(seen) != len(want) {
-		t.Fatalf("600 draws produced %v, want all four towns", seen)
-	}
-}
-
-func TestPathWalksVariantsOfDifferentShape(t *testing.T) {
-	// One variant holds addr as a plain template where another holds a choice.
-	// Both carry the path, so both must walk — the walk reads what it drew, never
-	// assuming a shape.
-	f := engine(13)
-	tmpl := `{"format":"{p.addr.city}/{p.addr.zip}","p":[
-		{"format":"{addr}","addr":{"format":"{city}","city":"Kiruna","zip":"98100"}},
-		{"format":"{addr}","addr":[
-			{"format":"{city}","city":"Malmö","zip":"21100"},
-			{"format":"{city}","city":"Lund","zip":"22100"}]}]}`
-	for i := 0; i < 400; i++ {
-		got := mustRender(t, f, tmpl)
-		if got != "Kiruna/98100" && got != "Malmö/21100" && got != "Lund/22100" {
-			t.Fatalf("draw %d = %q, want a city with its own zip", i, got)
-		}
-	}
-}
-
-func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) {
-	// Two paths share only what they actually share: a common prefix is one draw,
-	// and each keeps its own draw past the point they part.
-	f := engine(14)
-	tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}`
-	seen := map[string]bool{}
-	for i := 0; i < 400; i++ {
-		seen[mustRender(t, f, tmpl)] = true
-	}
-	for _, want := range []string{"11", "12", "21", "22"} {
-		if !seen[want] {
-			t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
-		}
-	}
-}
-
-func TestEmptyPathSegmentIsRejected(t *testing.T) {
-	// "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is
-	// reported as that rather than as a missing field. An empty name is rejected
-	// where it is authored, so no data can make these resolve.
-	rejected := map[string]string{
-		"trailing dot": `{"format":"[{a.}]","a":"x"}`,
-		"double dot":   `{"format":"[{a..b}]","a":"x"}`,
-	}
-	for name, file := range rejected {
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
-		if err == nil {
-			t.Errorf("%s: New = nil error, want the unfinished path rejected", name)
-			continue
-		}
-		if !strings.Contains(err.Error(), "empty segment") {
-			t.Errorf("%s: New = %v, want it to name the empty segment", name, err)
-		}
-	}
-}
-
-func TestCycleThroughAPathTokenIsRejected(t *testing.T) {
-	// A path token is a render edge like any other, so a cycle routed through one
-	// must be caught at New. Reaching render would be fatal: the recursion never
-	// terminates, and a stack overflow cannot be recovered.
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
-		"a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/b}"}}`,
-		"b": `"{/a}"`,
-	})))
-	if err == nil || !strings.Contains(err.Error(), "reference cycle") {
-		t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
-	}
-}
-
-func TestBoundPathIsReachableByFake(t *testing.T) {
-	// Binding changes how a format reads a sibling, not what List and Fake offer:
-	// the sub-fields stay addressable on their own.
-	dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")})
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9))
-	if err != nil {
-		t.Fatalf("New = %v", err)
-	}
-	for _, path := range []string{"address", "address.place", "address.place.locality", "address.place.postal-code"} {
-		if _, err := f.Fake(path); err != nil {
-			t.Errorf("Fake(%q) = %v, want it to render", path, err)
-		}
-	}
-}
-
-func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) {
-	for src, want := range map[string]string{
-		`{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`:                                    "write {w} once",
-		`{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`:                    "write {w} once",
-		`{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`:                                 "write {n} once",
-		`{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once",
-	} {
-		_, err := compile(parse(t, src))
-		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") {
-			t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want)
-		}
-	}
-	for _, ok := range []string{
-		`{"format":"{w} {w}","w":["a","b"]}`,
-		`{"format":"{w} {uppercase(w)}","w":["a","b"]}`,
-		`{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`,
-		`{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`,
-	} {
-		if _, err := compile(parse(t, ok)); err != nil {
-			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
-		}
-	}
-}
-
-func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
-	tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template)
-	if !ok {
-		t.Fatal("not a template")
-	}
-	mustPanic(t, "unheld arm with a path", func() {
-		readField(engine(1).rand, tm, nil, drawScope{}, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
-	})
-}

imports_test.go

@@ -0,0 +1,120 @@
+package fejkdata
+
+import (
+	"errors"
+	"go/build"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// TestPackageImports holds each package in the module to the module packages it may import.
+func TestPackageImports(t *testing.T) {
+	mod, err := os.ReadFile("go.mod")
+	if err != nil {
+		t.Fatal(err)
+	}
+	module := ""
+	for _, line := range strings.Split(string(mod), "\n") {
+		if f := strings.Fields(line); len(f) > 1 && f[0] == "module" {
+			module = strings.Trim(f[1], `"`)
+			break
+		}
+	}
+	if module == "" {
+		t.Fatal("go.mod names no module")
+	}
+	checked := 0
+	allowed := map[string][]string{
+		".":                    {"internal/builtinfunc", "internal/datafiles", "internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/jsonvalue", "internal/proven", "internal/rows"},
+		"cli":                  {"."},
+		"cmd/fejkdata":         {"cli", "data"},
+		"data":                 {"data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
+		"data/en_US":           {"data/geo/US"},
+		"data/geo/SE":          nil,
+		"data/geo/US":          nil,
+		"data/misc":            nil,
+		"data/sv_SE":           {"data/geo/SE"},
+		"internal/builtinfunc": {"internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/proven"},
+		"internal/datafiles":   {"internal/grammar", "internal/jsonvalue"},
+		"internal/datatype":    nil,
+		"internal/drawstate":   nil,
+		"internal/grammar":     nil,
+		"internal/invariant":   nil,
+		"internal/jsonvalue":   nil,
+		"internal/proven":      {"internal/datatype", "internal/grammar", "internal/invariant"},
+		"internal/rows":        {"internal/drawstate", "internal/grammar", "internal/invariant"},
+	}
+	testsAlso := map[string][]string{
+		".":           {"data", "data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
+		"cli":         {"internal/grammar"},
+		"data/en_US":  {"."},
+		"data/geo/SE": {"."},
+		"data/geo/US": {"."},
+		"data/misc":   {"."},
+		"data/sv_SE":  {"."},
+	}
+	seen := map[string]bool{}
+	eachPackage(t, func(dir string, pkg *build.Package) {
+		seen[dir] = true
+		may, listed := allowed[dir]
+		if !listed {
+			t.Errorf("package %s is not listed: add the module packages it may import", dir)
+			return
+		}
+		for _, imp := range slices.Concat(pkg.Imports, pkg.TestImports, pkg.XTestImports) {
+			may := may
+			if !slices.Contains(pkg.Imports, imp) {
+				may = slices.Concat(may, testsAlso[dir])
+			}
+			if imp != module && !strings.HasPrefix(imp, module+"/") {
+				continue
+			}
+			checked++
+			rel := strings.TrimPrefix(strings.TrimPrefix(imp, module), "/")
+			if rel == "" {
+				rel = "."
+			}
+			if rel != dir && !slices.Contains(may, rel) {
+				t.Errorf("package %s imports %s, which its list does not allow", dir, rel)
+			}
+		}
+	})
+	if checked == 0 {
+		t.Fatalf("no package imports a package of module %s, which go.mod names", module)
+	}
+	for dir := range allowed {
+		if !seen[dir] {
+			t.Errorf("listed package %s does not exist", dir)
+		}
+	}
+}
+
+// eachPackage calls fn with every Go package in the module and its slash-separated directory.
+func eachPackage(t *testing.T, fn func(dir string, pkg *build.Package)) {
+	t.Helper()
+	err := filepath.WalkDir(".", func(dir string, d fs.DirEntry, err error) error {
+		if err != nil || !d.IsDir() {
+			return err
+		}
+		if dir != "." && (strings.HasPrefix(d.Name(), ".") || d.Name() == "testdata") {
+			return filepath.SkipDir
+		}
+		pkg, err := build.ImportDir(dir, 0)
+		var noGo *build.NoGoError
+		if errors.As(err, &noGo) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		fn(filepath.ToSlash(dir), pkg)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

index.go

@@ -0,0 +1,188 @@
+package fejkdata
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"slices"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// indexed is a category an index names and no call has reached yet, and the source holding it.
+type indexed struct {
+	entry datafiles.IndexEntry
+	src   datafiles.Source
+}
+
+// placeIndex sets every category the index names as unloaded in root, in place of what
+// root holds at its path.
+func placeIndex(root *folder, src datafiles.Source, index map[string]datafiles.IndexEntry) {
+	for _, p := range sortedNames(index) {
+		segs := strings.Split(p, ".")
+		madeFolder(root, segs[:len(segs)-1]).putUnloaded(segs[len(segs)-1], &indexed{entry: index[p], src: src})
+	}
+}
+
+// categoryAt is where a category, loaded or not, sits: the folder holding it, that
+// folder's path, and its name.
+type categoryAt struct {
+	dir  []string
+	in   *folder
+	name string
+}
+
+func (c categoryAt) unloaded() bool { return c.in.unloaded[c.name] != nil }
+
+// categoryOn is the category, loaded or not, that a path from root names or passes.
+func categoryOn(root *folder, segs []string) (categoryAt, bool) {
+	g, i := folderAt(root, segs)
+	if i == len(segs) {
+		return categoryAt{}, false
+	}
+	c := categoryAt{dir: segs[:i:i], in: g, name: segs[i]}
+	return c, c.unloaded() || g.children[c.name] != nil
+}
+
+// loadCallerPath strips the / a caller's path may start with, splits the path, and loads the
+// unloaded category it names or passes. The caller walks it next, and a walk loads nothing.
+func (f *Generator) loadCallerPath(path string) (string, []string, error) {
+	path, err := grammar.CallerPath(path)
+	if err != nil {
+		return "", nil, fmt.Errorf("fejkdata: %w", err)
+	}
+	segs, err := grammar.SplitPath(path)
+	if err != nil {
+		return "", nil, fmt.Errorf("fejkdata: %w", err)
+	}
+	if c, ok := categoryOn(&f.root, segs); ok && c.unloaded() {
+		if err := loadReached(&f.root, nil, []categoryAt{c}); err != nil {
+			return "", nil, fmt.Errorf("fejkdata: %w", loadError{err})
+		}
+	}
+	return path, segs, nil
+}
+
+// referenced is every category, loaded or not, that the templates of nodes reference, each
+// reference read from the folder dir.
+func referenced(root *folder, dir []string, nodes nodeSet) []categoryAt {
+	var out []categoryAt
+	_ = nodes(func(_ string, n node) error {
+		t, isTemplate := n.(*template)
+		if !isTemplate {
+			return nil
+		}
+		for _, name := range refTokens(t.tokens) {
+			segs, err := refSegments(name, dir)
+			if err != nil {
+				continue // resolveRefs reports it
+			}
+			if c, ok := categoryOn(root, segs); ok {
+				out = append(out, c)
+			}
+		}
+		return nil
+	})
+	return out
+}
+
+// needs is every category, loaded or not, that s references, the table beside s that its
+// parent column names, and every unloaded table beside s whose parent s is.
+func needs(root *folder, s categorySite) []categoryAt {
+	out := referenced(root, s.dir, siteNodes([]categorySite{s}))
+	if parent := parentOf(s.n); parent != "" {
+		out = append(out, categoryAt{dir: s.dir, in: s.in, name: parent})
+	}
+	for _, name := range sortedNames(s.in.unloaded) {
+		if s.in.unloaded[name].entry.Parent == s.name {
+			out = append(out, categoryAt{dir: s.dir, in: s.in, name: name})
+		}
+	}
+	return out
+}
+
+// loadReached loads the unloaded categories among wanted and every unloaded category they
+// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an
+// error it puts back what it loaded, so the next call fails the same way.
+func loadReached(root *folder, sites []categorySite, wanted []categoryAt) error {
+	queue := wanted
+	for _, s := range sites {
+		queue = append(queue, needs(root, s)...)
+	}
+	var b batch
+	err := b.load(root, queue)
+	if err == nil {
+		err = b.run(root, sites)
+	}
+	if err != nil {
+		b.putBack()
+	}
+	return err
+}
+
+// batch is the categories one loadReached loaded, each beside the entry it was loaded from.
+type batch []loaded
+
+type loaded struct {
+	site categorySite
+	was  *indexed
+}
+
+// load loads each unloaded category in queue and what it needs. It stops at a table whose
+// parent column disagrees with its entry. Otherwise a table could load without its parent,
+// or an earlier table could link to one that putBack then removes.
+func (b *batch) load(root *folder, queue []categoryAt) error {
+	for ; len(queue) > 0; queue = queue[1:] {
+		c := queue[0]
+		x := c.in.unloaded[c.name]
+		if x == nil {
+			continue
+		}
+		if err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })); err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				err = fmt.Errorf("%s indexes %s: %w", x.src.ManifestPath(), categoryPath(c.dir, c.name), err)
+			}
+			return err
+		}
+		s := siteIn(c.dir, c.in, c.name)
+		*b = append(*b, loaded{site: s, was: x})
+		if parent := parentOf(s.n); parent != x.entry.Parent {
+			return fmt.Errorf("%s: the index entry for %s %s, and its table %s", x.src.ManifestPath(), s.path, namesParent(x.entry.Parent), namesParent(parent))
+		}
+		queue = append(queue, needs(root, s)...)
+	}
+	return nil
+}
+
+func (b batch) run(root *folder, sites []categorySite) error {
+	sites = slices.Clip(sites)
+	for _, l := range b {
+		sites = append(sites, l.site)
+	}
+	if len(sites) == 0 {
+		return nil
+	}
+	return categoryPipeline(sites, root.children).run()
+}
+
+func (b batch) putBack() {
+	for _, l := range b {
+		l.site.in.putUnloaded(l.site.name, l.was)
+	}
+}
+
+func namesParent(parent string) string {
+	if parent == "" {
+		return "names no parent"
+	}
+	return fmt.Sprintf("names parent %q", parent)
+}
+
+func parentOf(n node) string {
+	if t, isTable := n.(*table); isTable {
+		return t.rows.Options().Parent
+	}
+	return ""
+}

inline.go

@@ -3,28 +3,30 @@ package fejkdata
 import (
 	"encoding/json"
 	"fmt"
-	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
 // Template is an inline template compiled, referenced and validated against a
 // generator's loaded data once, ready to render many times with [Template.Fake].
 // It is safe for concurrent use: Fake serializes on its generator's lock, so a
-// seeded sequence is reproducible only when a generator — and its templates — are
+// sequence is reproducible only when one goroutine makes every draw from a generator
-// drawn from one goroutine.
+// and its templates.
 type Template struct {
 	g *Generator
 	n node
 }
 
-// Fake renders the template with one draw.
+// Fake renders the template once; each call draws anew.
 func (t *Template) Fake() string {
 	t.g.mu.Lock()
 	defer t.g.mu.Unlock()
-	return renderOnce(t.g.rand, t.n)
+	return renderOnce(t.g.drawState, t.n)
 }
 
 // NewTemplate compiles an inline template — a format string or a JSON value — and
-// binds its references against the loaded tree, so repeated renders pay the
+// resolves its references against the loaded tree, so repeated renders pay the
 // compile and validation once. It shares [New]'s guarantees: a bad template errors
 // here, and rendering cannot fail.
 func (f *Generator) NewTemplate(input string) (*Template, error) {
@@ -32,8 +34,10 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
-	if err := bindInline(n, "template", f.categories, checkScope); err != nil {
+	f.mu.Lock()
-		return nil, fmt.Errorf("fejkdata: %w", err)
+	defer f.mu.Unlock()
+	if err := loadInline(&f.root, n, "template", false); err != nil {
+		return nil, f.noDataNote(fmt.Errorf("fejkdata: %w", err))
 	}
 	return &Template{g: f, n: n}, nil
 }
@@ -49,127 +53,70 @@ func (f *Generator) FakeTemplate(input string) (string, error) {
 	return t.Fake(), nil
 }
 
-// IsTemplate reports whether arg is an inline template rather than a path, by its shape: a {
+// IsTemplate reports whether arg is an inline template rather than a path, by its shape: an
-// token, or a JSON object, array or string, is a template, and anything else is a path. A
+// arg holding a {, or a valid JSON array or string, is a template, and anything else is a
-// name never holds a bracket, a brace or a quote, so an arg holding one that is not valid
+// path. A path errors where it holds a } or a quote, starts with [, or is no valid path. A path
-// JSON names neither, and errors; so do a template of one reference alone, which is a path
+// may start with /, as a reference does, and every call taking a path drops it.
-// written as a template, and a path written with a leading /.
 func IsTemplate(arg string) (bool, error) {
-	inline, err := isTemplate(arg)
+	inline, err := grammar.IsTemplate(arg)
 	if err != nil {
 		return false, fmt.Errorf("fejkdata: %w", err)
 	}
 	return inline, nil
 }
 
-func isTemplate(arg string) (bool, error) {
-	if strings.ContainsRune(arg, '{') || (isJSONStart(strings.TrimSpace(arg)) && json.Valid([]byte(arg))) {
-		if path, lone := loneReference(arg); lone {
-			return false, pathAdvice(path, fmt.Sprintf("%s is the path %s written as a template", arg, path))
-		}
-		return true, nil
-	}
-	if i := strings.IndexAny(arg, `}"`); i >= 0 {
-		return false, fmt.Errorf("%q holds a %q, which no path may, and it is not valid JSON, so it names no template either", arg, arg[i:i+1])
-	}
-	if strings.HasPrefix(strings.TrimSpace(arg), "[") {
-		return false, fmt.Errorf(`%q starts with "[" and is not valid JSON, so it names no template; a path starts with a name`, arg)
-	}
-	if _, err := splitPath(arg); err != nil {
-		return false, err
-	}
-	if path := strings.TrimLeft(arg, "/"); path != arg && path != "" {
-		return false, pathAdvice(path, fmt.Sprintf("path %s starts with /, and every path starts at the root already", arg))
-	}
-	return false, nil
-}
-
-func isJSONStart(arg string) bool {
-	return strings.HasPrefix(arg, "[") || strings.HasPrefix(arg, `"`)
-}
-
-// pathAdvice refuses a spelling of path, naming path to write, or why no name can spell it.
-func pathAdvice(path, refusal string) error {
-	if err := checkPathNames(path); err != nil {
-		return err
-	}
-	return fmt.Errorf("%s; write %s", refusal, path)
-}
-
-// loneReference is the path a template spells when the value it holds — a format string, a
-// JSON string, or an object holding only a format — is one reference token and nothing else.
-func loneReference(arg string) (string, bool) {
-	var raw any
-	if json.Unmarshal([]byte(arg), &raw) != nil {
-		raw = arg
-	}
-	if m, isObject := raw.(map[string]any); isObject && len(m) == 1 {
-		raw = m["format"]
-	}
-	format, isString := raw.(string)
-	body, lone := loneRef(format)
-	if !isString || !lone {
-		return "", false
-	}
-	if strings.HasPrefix(body, "/") {
-		body = "/" + strings.TrimLeft(body, "/")
-	}
-	_, path, err := refShape(body)
-	return path, err == nil
-}
-
 func compileInput(input string) (node, error) {
-	v, err := inputValue(input)
+	return compile(inputValue(input))
+}
+
+// inputValue reads an inline template as the value compile takes: the JSON object, array, string
+// or null it holds, the whitespace around it dropped, or else the input itself as a format string,
+// a bare number or boolean among them.
+func inputValue(input string) jsonvalue.Value {
+	raw, err := jsonvalue.Decode([]byte(input))
 	if err != nil {
-		return nil, err
+		return jsonvalue.String(input)
 	}
-	return compile(v)
+	switch raw.Any().(type) {
+	case json.Number, bool:
+		return jsonvalue.String(input)
+	}
+	return raw
 }
 
-// inputValue reads an inline template as the value compile takes: the JSON value it holds, or
+// loadInline loads the unloaded categories n reads, then runs n through the pipeline against root.
-// the input itself as a format string when it is not JSON.
+func loadInline(root *folder, n node, label string, typedByGo bool) error {
-func inputValue(input string) (any, error) {
+	nodes := inlineNodes(n, label)
-	var raw any
+	if err := refuseFolderRefs(nodes); err != nil {
-	if err := json.Unmarshal([]byte(input), &raw); err != nil {
-		return input, nil
-	}
-	if trimmed := strings.TrimSpace(input); trimmed != input {
-		return nil, fmt.Errorf("a JSON template may not be padded with spaces, which a format string would render; write %s", trimmed)
-	}
-	return raw, nil
-}
-
-// bindInline links an inline node's references against root and runs check over it, naming its
-// nodes from label.
-func bindInline(n node, label string, root map[string]node, check func(nodeScope) error) error {
-	if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
-		return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)
-	}
-	scope := inlineScope(n, label)
-	if err := linkNodeRefs(scope, root); err != nil {
 		return err
 	}
-	return check(scope)
+	if err := loadReached(root, nil, referenced(root, nil, nodes)); err != nil {
+		return loadError{err}
+	}
+	return pipeline{
+		nodes:     nodes,
+		resolve:   func() error { return resolveInlineTemplates(nodes, root.children) },
+		typedByGo: typedByGo,
+	}.run()
 }
 
-// linkNodeRefs binds the references in an inline node's templates against the
+// refuseFolderRefs refuses each reference in nodes not written from the root, {/x}: an
-// loaded tree.
+// inline node sits in no folder.
-func linkNodeRefs(scope nodeScope, root map[string]node) error {
+func refuseFolderRefs(nodes nodeSet) error {
-	return scope(func(path string, m node) error {
+	return nodes(func(label string, m node) error {
 		t, ok := m.(*template)
 		if !ok {
 			return nil
 		}
-		t.keyDrawGroup("")
+		for _, name := range refTokens(t.tokens) {
-		for _, name := range refTokens(t.format) {
+			sigil, rest, err := grammar.RefShape(name)
-			sigil, rest, err := refShape(name)
 			if err != nil {
-				return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
+				return fmt.Errorf("%s: reference {%s}: %w", label, name, err)
 			}
 			if sigil != "/" {
-				return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", path, name, rest)
+				return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", label, name, rest)
 			}
 		}
-		return linkTemplateRefs(nil, path, "", t, root)
+		return nil
 	})
 }

inline_test.go

@@ -16,7 +16,7 @@ func tmpl(t *testing.T, f *Generator, input string) string {
 
 func shipped(t *testing.T, opts ...Option) *Generator {
 	t.Helper()
-	f, err := New(append([]Option{WithSeed(1)}, opts...)...)
+	f, err := New(append([]Option{withShipped(), WithSeed(1)}, opts...)...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -53,13 +53,13 @@ func TestFakeTemplateJSONArray(t *testing.T) {
 	}
 }
 
-func TestFakeTemplateCorrelatedReferences(t *testing.T) {
+func TestFakeTemplateNamedReference(t *testing.T) {
 	dir := writeData(t, map[string]string{
 		"person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`,
 	})
 	f := newGenerator(t, dir, WithSeed(1))
 	for i := 0; i < 100; i++ {
-		got := tmpl(t, f, "{/person.first} {/person.last}")
+		got := tmpl(t, f, "{/person as p}{p.first} {p.last}")
 		if got != "Ada Lovelace" && got != "Bo Ek" {
 			t.Fatalf("correlated references = %q, want one person's first and last", got)
 		}
@@ -104,13 +104,9 @@ func TestFakeTemplateErrors(t *testing.T) {
 	}{
 		{`{"x":"Q"}`, "missing string \"format\""},
 		{`"{x}"`, `no field "x"`},
-		{`"{digits(0)}"`, "must be positive"},
+		{`"{digits(-1)}"`, "must not be negative"},
 		{`name: {/no.such.path}`, "no entry"},
 		{`name: {..nope}`, "write {/nope}"},
-		{`{"format":"x"}`, "is a string"},
-		{`{/misc.territory} {/misc.territory.alpha2}`, "renders a level that {/misc.territory.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"},
-		{`{"format":"{/misc.territory.alpha2} {x}","x":"{/misc.territory}"}`, "reads a path into"},
-		{`{"format":"{/misc.territory.alpha2}","drawGroup":"g"}`, "nothing can reference"},
 	} {
 		_, err := f.FakeTemplate(c.input)
 		if err == nil || !strings.Contains(err.Error(), c.want) {
@@ -127,12 +123,15 @@ func TestFakeTemplateJSONString(t *testing.T) {
 	}
 }
 
-func TestPaddedJSONIsRejected(t *testing.T) {
+func TestPaddedJSONIsTheJSON(t *testing.T) {
 	f := shipped(t)
 	in := `{"format":"{x}","x":["a","b"]}`
-	_, err := f.NewTemplate("  " + in + "  ")
+	padded, err := f.NewTemplate(" \t" + in + "\n ")
-	if err == nil || !strings.Contains(err.Error(), "write "+in) {
+	if err != nil {
-		t.Fatalf("NewTemplate(padded JSON) = %v, want an error naming the unpadded spelling", err)
+		t.Fatalf("NewTemplate(padded JSON) = %v, want the JSON it pads", err)
+	}
+	if got := padded.Fake(); got != "a" && got != "b" {
+		t.Fatalf("padded JSON rendered %q, want a or b with no padding", got)
 	}
 }
 
@@ -153,46 +152,45 @@ func TestNewTemplateReusable(t *testing.T) {
 
 func TestIsTemplate(t *testing.T) {
 	for arg, want := range map[string]bool{
-		"sv_SE.person":    false,
+		"sv_SE.person":                           false,
-		"person.last":     false,
+		"person.last":                            false,
-		"name: {x}":       true,
+		"name: {x}":                              true,
-		`{"format":"x"}`:  true,
+		`{"format":"x"}`:                         true,
-		`["a","b"]`:       true,
+		`["a","b"]`:                              true,
-		`[1, 2]`:          true,
+		`[1, 2]`:                                 true,
-		` ["a","b"]`:      true, // padding is the template's own error, not a shape verdict
+		` ["a","b"]`:                             true, // padding is the template's own error, not a shape verdict
-		`"hello"`:         true,
+		`"hello"`:                                true,
-		"{/a}{/b}":        true,
+		"{/a}{/b}":                               true,
-		"{/a|/b}":         true,
+		"{/a|/b}":                                true,
-		"{uppercase(/a)}": true,
+		"{uppercase(/a)}":                        true,
-		"{{/a}}":          true,
+		"{{/a}}":                                 true,
-		`"{/a} x"`:        true,
+		`"{/a} x"`:                               true,
-		"x[1]":            false, // a row selected by key or name
+		"x[1]":                                   false, // a row selected by key or name
-		"x[St. Louis].y":  false,
+		"x[St. Louis].y":                         false,
+		"{/sv_SE.person.last}":                   true,
+		`"{/sv_SE.person}"`:                      true,
+		"{.person.last}":                         true,
+		` "{/sv_SE.person}"`:                     true,
+		`{"format":"{/sv_SE.person}"}`:           true,
+		"{//sv_SE.person}":                       true,
+		"{/misc.currency[US Dollar (Next day)]}": true,
+		"/sv_SE.person":                          false,
 	} {
 		if got, err := IsTemplate(arg); err != nil || got != want {
 			t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want)
 		}
 	}
 	for arg, want := range map[string]string{
-		"[abc]":                        `starts with "["`,
+		"[abc]":          `starts with "["`,
-		"[abc].field":                  `starts with "["`,
+		"[abc].field":    `starts with "["`,
-		"x[1]y":                        `"]"`,
+		"x[1]y":          `"]"`,
-		"x[[1]]":                       `"["`,
+		"x[[1]]":         `"["`,
-		"x[]":                          "empty",
+		"x[]":            "empty",
-		"a]b":                          `holds a "]"`,
+		"//sv_SE.person": "write /sv_SE.person",
-		"a}b":                          `holds a "}"`,
+		"a]b":            `holds a "]"`,
-		`"abc`:                         `holds a "\""`,
+		"a}b":            `holds a "}"`,
-		`"a]b`:                         `holds a "\""`, // the opener the reader typed, not the bracket behind it
+		`"abc`:           `holds a "\""`,
-		"{/sv_SE.person.last}":         "{/sv_SE.person.last} is the path sv_SE.person.last written as a template; write sv_SE.person.last",
+		`"a]b`:           `holds a "\""`, // the opener the reader typed, not the bracket behind it
-		`"{/sv_SE.person}"`:            "write sv_SE.person",
-		"{.person.last}":               "write person.last",
-		"/sv_SE.person":                "write sv_SE.person",
-		` "{/sv_SE.person}"`:           "write sv_SE.person",
-		`{"format":"{/sv_SE.person}"}`: "write sv_SE.person",
-		"//sv_SE.person":               "write sv_SE.person",
-		"{//sv_SE.person}":             "{//sv_SE.person} is the path sv_SE.person written as a template; write sv_SE.person",
-		"/sv_SE/person":                `path "sv_SE/person" contains "/"`,
-		"{/-}":                         `path "-" is reserved`,
 	} {
 		if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want)

internal/builtinfunc/builtinfunc.go

@@ -0,0 +1,119 @@
+// Package builtinfunc holds the {name(args)} functions a format calls: their checks, the
+// draws they make, and the numbers their text is proven to read as.
+package builtinfunc
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/proven"
+)
+
+// Call is a builtin prepared for one call site: its args already parsed. It reads the
+// output emitted so far in the current expansion (a derivation's payload) and the
+// values of the operands it named, which the caller read for it.
+type Call func(s *drawstate.State, emitted string, operands []string) string
+
+// builtin is a format-string function invoked as {name(args)}. Its Call stays pure over its
+// inputs, so seeded output is reproducible. arity is the exact arg count, or -1 for variadic,
+// where checkArgs does all the validation.
+type builtin struct {
+	arity int
+	// prep parses validated args once, at compile time, into the closure a render calls.
+	prep      func(args []string) Call
+	checkArgs func(args []string) error
+	// operands names the fields the call reads, which a render reads for it; nil
+	// for a builtin that reads none.
+	operands func(args []string) []string
+	// proveNumber bounds the number a call's text reads as, token its body, and says which
+	// datatypes that text is not; set it where every render reads as a finite number, which
+	// makes the call a calc operand, and leave it nil otherwise.
+	proveNumber func(token string, prints datatype.DataType, args []string) proven.Facts
+	// prints is the datatype a call's text is, handed to proveNumber: datatype.String where it
+	// reads as a number no column should type, as digits' leading zeros; unset where
+	// proveNumber is nil.
+	prints datatype.DataType
+}
+
+// CalcName is the builtin whose operands package fejkdata checks and proves, since only it
+// holds the nodes they read.
+const CalcName = "calc"
+
+// rng is the randomness a builtin sample draws from, which *drawstate.State satisfies.
+type rng interface {
+	IntN(n int) int
+	Float64() float64
+}
+
+// Check validates a call at compile time: naming a known builtin, with the arg count that
+// builtin takes and args its check accepts.
+func Check(name string, args []string) error {
+	b, known := builtins[name]
+	if !known {
+		return fmt.Errorf("unknown function %q", name)
+	}
+	if b.arity >= 0 && len(args) != b.arity {
+		return fmt.Errorf("%s takes %d argument%s, got %d", name, b.arity, plural(b.arity), len(args))
+	}
+	if b.checkArgs != nil {
+		return b.checkArgs(args)
+	}
+	return nil
+}
+
+// Prep parses a call's args, which Check accepted, into the Call a render makes.
+func Prep(name string, args []string) Call { return builtins[name].prep(args) }
+
+// Operands lists the fields a call reads as operands, empty for a builtin that reads none
+// or is unknown.
+func Operands(name string, args []string) []string {
+	b, known := builtins[name]
+	if !known || b.operands == nil {
+		return nil
+	}
+	return b.operands(args)
+}
+
+// IsTransform reports a builtin that rewrites the text of its one operand.
+func IsTransform(name string) bool {
+	_, isTransform := transforms[name]
+	return isTransform
+}
+
+// ProveNumber proves a call whose every render reads as a finite number, token its body;
+// false for any other call, calc included: proven.OfCalc proves a calc from its operands,
+// which only the caller holds.
+func ProveNumber(name, token string, args []string) (proven.Facts, bool) {
+	b := builtins[name]
+	if b.proveNumber == nil {
+		return proven.Facts{}, false
+	}
+	return b.proveNumber(token, b.prints, args), true
+}
+
+// TypedCalls lists the calls whose text a typed column may hold, as an error names them.
+func TypedCalls() string { return numberCalls(false) }
+
+// OperandCalls lists the calls whose text a calc reads as a number, as an error names them.
+func OperandCalls() string { return numberCalls(true) }
+
+func numberCalls(text bool) string {
+	var calls []string
+	for name, b := range builtins {
+		if b.proveNumber != nil && (text || b.prints != datatype.String) {
+			calls = append(calls, "{"+name+"()}")
+		}
+	}
+	slices.Sort(calls)
+	return strings.Join(calls, ", ") + " or {" + CalcName + "()}"
+}
+
+func plural(n int) string {
+	if n == 1 {
+		return ""
+	}
+	return "s"
+}

internal/builtinfunc/builtinfunc_test.go

@@ -0,0 +1,68 @@
+package builtinfunc
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// TestRegistryShapes pins the builtin contract Prep relies on: every entry supplies prep,
+// and args parsed at compile only behind a check.
+func TestRegistryShapes(t *testing.T) {
+	for name, b := range builtins {
+		if b.prep == nil {
+			t.Errorf("builtin %q has no prep: Prep would call a nil func", name)
+		}
+		if b.arity != 0 && b.checkArgs == nil {
+			t.Errorf("builtin %q parses args in prep with no check", name)
+		}
+	}
+}
+
+func TestCheckNamesWhatACallGetsWrong(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		args []string
+		want string
+	}{
+		{"nope", nil, `unknown function "nope"`},
+		{"luhn", []string{"1"}, "luhn takes 0 arguments, got 1"},
+		{"hex", nil, "hex takes 1 argument, got 0"},
+		{"calc", []string{"a +"}, `calc("a +"): `},
+		{"calc", []string{"a", "-1"}, "calc decimals "},
+		{"calc", nil, "calc takes an expression and an optional decimals count, got 0 args"},
+	} {
+		if err := Check(c.name, c.args); err == nil || !strings.HasPrefix(err.Error(), c.want) {
+			t.Errorf("Check(%s, %q) = %v, want it to start %q", c.name, c.args, err, c.want)
+		}
+	}
+	if err := Check("calc", []string{"a / b", "2"}); err != nil {
+		t.Errorf("Check(calc) = %v, want an expression over any operand accepted", err)
+	}
+}
+
+// TestArgGuardsPanic pins the guards that report a builtin arg its check should have
+// rejected. No data reaches them, since Check runs before Prep, so they are called directly.
+func TestArgGuardsPanic(t *testing.T) {
+	for name, call := range map[string]func(){
+		"atoi on an unvalidated arg":             func() { atoi("nope") },
+		"atof on an unvalidated arg":             func() { atof("nope") },
+		"prep on an expression that won't parse": func() { calcPrep([]string{"1 +"}) },
+	} {
+		mustPanic(t, name, call)
+	}
+}
+
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
+// separates a reported invariant break from a silently wrong value.
+func mustPanic(t *testing.T, name string, call func()) {
+	t.Helper()
+	defer func() {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
+		}
+	}()
+	call()
+}

internal/builtinfunc/builtins.go

@@ -1,4 +1,4 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"encoding/base64"
@@ -7,6 +7,11 @@ import (
 	"math"
 	"strconv"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/proven"
 )
 
 // maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
@@ -22,88 +27,88 @@ const (
 // emitted so far in the current expansion (place them after their payload);
 // samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
 // from the rng, not the wall clock, so seeded output stays reproducible.
-var builtins = map[string]builtin{
+var builtins = withTransforms(map[string]builtin{
 	"luhn":   {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
 	"mod11":  {arity: 0, prep: derive(mod11Check)},
 	"ean":    {arity: 0, prep: derive(eanCheck)},
 	"uuid":   {arity: 0, prep: sample(uuidV7)},
 	"ulid":   {arity: 0, prep: sample(ulid)},
-	"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)},
+	"nanoid": {arity: 1, checkArgs: countArg, prep: chars(nanoidAlphabet)},
-	"hex":    {arity: 1, check: posIntArg, prep: chars(hexDigits)},
+	"hex":    {arity: 1, checkArgs: countArg, prep: chars(hexDigits)},
-	"digits": {arity: 1, check: posIntArg, prep: chars("0123456789"), number: func(token string, a []string) proven {
+	"digits": {arity: 1, checkArgs: countArg, prep: chars("0123456789"), prints: datatype.String, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
-		return printing(token, DataTypeString, bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
+		if atoi(a[0]) == 0 {
+			return proven.Unproven("{" + token + "} prints nothing, which is no number")
+		}
+		return proven.Printing(token, prints, proven.Bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
 	}},
-	"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
+	"upper": {arity: 1, checkArgs: countArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
-	"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
+	"lower": {arity: 1, checkArgs: countArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
-	"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
+	"base64": {arity: 1, checkArgs: countArg, prep: func(a []string) Call {
 		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
 			return base64.StdEncoding.EncodeToString(randBytes(s, n))
 		}
 	}},
-	"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
+	"int": {arity: 2, checkArgs: intRangeArgs, prep: func(a []string) Call {
 		lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
-		return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
+		return func(s *drawstate.State, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
-	}, number: func(token string, a []string) proven {
+	}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
-		return printing(token, DataTypeInteger, bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
+		return proven.Printing(token, prints, proven.Bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
 	}},
-	"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
+	"float": {arity: 3, checkArgs: floatArgs, prep: func(a []string) Call {
 		lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
 			return formatFloat(lo+s.Float64()*(hi-lo), dp)
 		}
-	}, number: func(token string, a []string) proven {
+	}, prints: datatype.Number, proveNumber: func(token string, _ datatype.DataType, a []string) proven.Facts {
-		return printedNumber(token, bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
+		return proven.PrintedNumber(token, proven.Bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
 	}},
-	"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn {
+	"iban": {arity: 1, checkArgs: ibanArg, prep: func(a []string) Call {
 		cc := a[0]
-		return func(s *session, _ string, _ []string) string { return iban(s, cc) }
+		return func(s *drawstate.State, _ string, _ []string) string { return iban(s, cc) }
 	}},
-	"date":      {arity: -1, check: dateArgs, prep: datePrep},
+	"date":   {arity: -1, checkArgs: dateArgs, prep: datePrep},
-	"time":      {arity: -1, check: timeArg, prep: timePrep},
+	"time":   {arity: -1, checkArgs: timeArg, prep: timePrep},
-	"calc":      {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands},
+	CalcName: {arity: -1, checkArgs: checkCalc, prep: calcPrep, operands: calcOperands},
-	"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand},
+	// seq is the one stateful builtin: a per-generator counter from 1, advancing on
-	"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
-	"ascii":     {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
-	// seq is the one stateful builtin: a per-session counter from 1, advancing on
 	// each call. An optional name selects an independent counter; no name uses the
 	// default one. Deterministic by construction, so seeded output stays stable.
-	"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
+	"seq": {arity: -1, checkArgs: seqArg, prep: func(a []string) Call {
 		key := ""
 		if len(a) == 1 {
 			key = a[0]
 		}
-		return func(s *session, _ string, _ []string) string {
+		return func(s *drawstate.State, _ string, _ []string) string {
-			return strconv.FormatUint(s.next(key), 10)
+			return strconv.FormatUint(s.Seq(key), 10)
 		}
-	}, number: func(token string, _ []string) proven {
+	}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, _ []string) proven.Facts {
-		return printing(token, DataTypeInteger, bounded(1, math.MaxInt64, true))
+		return proven.Printing(token, prints, proven.Bounded(1, math.MaxInt64, true))
 	}},
-}
+})
 
 // derive and sample are the two argument-free builtin shapes: a derivation reads
 // the output emitted so far, a sample reads only the rng. chars is the shape of a
 // sample of n characters drawn from an alphabet.
-func derive(f func(emitted string) string) func([]string) callFn {
+func derive(f func(emitted string) string) func([]string) Call {
-	return func([]string) callFn {
+	return func([]string) Call {
-		return func(_ *session, emitted string, _ []string) string { return f(emitted) }
+		return func(_ *drawstate.State, emitted string, _ []string) string { return f(emitted) }
 	}
 }
-func sample(f func(rng) string) func([]string) callFn {
+func sample(f func(rng) string) func([]string) Call {
-	return func([]string) callFn {
+	return func([]string) Call {
-		return func(s *session, _ string, _ []string) string { return f(s) }
+		return func(s *drawstate.State, _ string, _ []string) string { return f(s) }
 	}
 }
-func chars(alphabet string) func([]string) callFn {
+func chars(alphabet string) func([]string) Call {
-	return func(a []string) callFn {
+	return func(a []string) Call {
 		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) }
+		return func(s *drawstate.State, _ string, _ []string) string { return randChars(s, n, alphabet) }
 	}
 }
 
 const hexDigits = "0123456789abcdef"
 
-// formatFloat prints v to dp decimals, -1 for the shortest form, and a zero unsigned.
+// formatFloat prints v to dp decimals, or proven.ShortestDecimals, and a zero unsigned.
 func formatFloat(v float64, dp int) string {
 	s := strconv.FormatFloat(v, 'f', dp, 64)
 	if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
@@ -118,7 +123,7 @@ func formatFloat(v float64, dp int) string {
 func atoi(s string) int {
 	n, err := strconv.Atoi(s)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
 	}
 	return n
 }
@@ -127,7 +132,7 @@ func atoi(s string) int {
 func atof(s string) float64 {
 	f, err := strconv.ParseFloat(s, 64)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
 	}
 	return f
 }
@@ -146,8 +151,7 @@ func randChars(r rng, n int, alphabet string) string {
 	return string(b)
 }
 
-// plainInt parses an integer arg written the one way: no sign, no leading zero.
+func intArg(s string) (int, error) {
-func plainInt(s string) (int, error) {
 	n, err := strconv.Atoi(s)
 	if errors.Is(err, strconv.ErrRange) {
 		return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
@@ -155,54 +159,48 @@ func plainInt(s string) (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("%q is not an integer", s)
 	}
-	if strconv.Itoa(n) != s {
-		return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n)
-	}
 	return n, nil
 }
-func posIntArg(_ map[string]node, a []string) error {
+func countArg(a []string) error {
-	n, err := plainInt(a[0])
+	n, err := intArg(a[0])
 	if errors.Is(err, strconv.ErrRange) {
 		return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
 	}
 	if err != nil {
 		return fmt.Errorf("count %w", err)
 	}
-	if n < 1 {
+	if n < 0 {
-		return fmt.Errorf("count %q must be positive", a[0])
+		return fmt.Errorf("count %q must not be negative", a[0])
 	}
 	if n > maxLen {
 		return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
 	}
 	return nil
 }
-func intRangeArgs(_ map[string]node, a []string) error {
+func intRangeArgs(a []string) error {
-	lo, err := plainInt(a[0])
+	lo, err := intArg(a[0])
 	if err != nil {
 		return fmt.Errorf("int(min,max): min %w", err)
 	}
-	hi, err := plainInt(a[1])
+	hi, err := intArg(a[1])
 	if err != nil {
 		return fmt.Errorf("int(min,max): max %w", err)
 	}
 	if lo > hi {
 		return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
 	}
-	if lo == hi {
-		return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo)
-	}
 	if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
 		return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
 	}
 	return nil
 }
-func floatArgs(_ map[string]node, a []string) error {
+func floatArgs(a []string) error {
 	lo, e1 := strconv.ParseFloat(a[0], 64)
 	hi, e2 := strconv.ParseFloat(a[1], 64)
 	if e1 != nil || e2 != nil {
 		return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
 	}
-	dp, err := plainInt(a[2])
+	dp, err := intArg(a[2])
 	if err != nil {
 		return fmt.Errorf("float(min,max,dp): decimals %w", err)
 	}
@@ -215,15 +213,12 @@ func floatArgs(_ map[string]node, a []string) error {
 	if dp < 0 || dp > maxDecimals {
 		return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
 	}
-	if lo == hi {
-		return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64))
-	}
 	if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
 		return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
 	}
 	return nil
 }
-func seqArg(_ map[string]node, a []string) error {
+func seqArg(a []string) error {
 	if len(a) > 1 {
 		return fmt.Errorf("seq takes at most one name, got %d", len(a))
 	}

internal/builtinfunc/calc.go

@@ -0,0 +1,109 @@
+package builtinfunc
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/proven"
+)
+
+// evalCalc evaluates an expression over the operand values a render already read, so it
+// draws nothing.
+func evalCalc(n grammar.CalcNode, operands []string) float64 {
+	switch n := n.(type) {
+	case grammar.CalcNum:
+		return float64(n)
+	case grammar.CalcVar:
+		v, err := strconv.ParseFloat(strings.TrimSpace(operands[n.At]), 64)
+		if err != nil {
+			return math.NaN() // a non-numeric operand stays visible, never an error
+		}
+		return v
+	case grammar.CalcNeg:
+		return -evalCalc(n.X, operands)
+	case grammar.CalcBin:
+		return Arith(n.Operator, evalCalc(n.L, operands), evalCalc(n.R, operands))
+	}
+	panic(invariant.Broken("calc node %#v has no evaluation", n))
+}
+
+// Arith applies a calc operator.
+func Arith(operator byte, l, r float64) float64 {
+	switch operator {
+	case '+':
+		return l + r
+	case '-':
+		return l - r
+	case '*':
+		return l * r
+	default: // '/'
+		return l / r
+	}
+}
+
+// checkCalc validates a calc token's own args: a parseable expression and an optional
+// non-negative integer dp. What its operands hold is the caller's to check.
+func checkCalc(args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
+	}
+	if _, err := grammar.ParseCalc(args[0]); err != nil {
+		return fmt.Errorf("calc(%q): %w", args[0], err)
+	}
+	if len(args) == 2 {
+		dp, err := intArg(args[1])
+		if err != nil {
+			return fmt.Errorf("calc decimals %w", err)
+		}
+		if dp < 0 || dp > maxDecimals {
+			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
+		}
+	}
+	return nil
+}
+
+// calcPrep parses the expression and decimals once, at compile time. checkCalc proved
+// both args valid, so no step here can fail.
+func calcPrep(args []string) Call {
+	expr := ParsedCalc(args[0]).Expr
+	dp := CalcDecimals(args)
+	return func(_ *drawstate.State, _ string, operands []string) string {
+		return formatFloat(evalCalc(expr, operands), dp)
+	}
+}
+
+// ParsedCalc parses an expression Check accepted.
+func ParsedCalc(expr string) grammar.Calc {
+	c, err := grammar.ParseCalc(expr)
+	if err != nil {
+		panic(invariant.Broken("calc(%q) passed its check unparsed: %v", expr, err))
+	}
+	return c
+}
+
+// CalcDecimals is the decimals count of a calc Check accepted, or proven.ShortestDecimals
+// where it names none.
+func CalcDecimals(args []string) int {
+	if len(args) == 2 {
+		return atoi(args[1])
+	}
+	return proven.ShortestDecimals
+}
+
+// calcOperands lists the operands a calc's args read, fields or names. checkCalc reports
+// an expression that does not parse, so one that does not simply names nothing.
+func calcOperands(args []string) []string {
+	if len(args) == 0 {
+		return nil
+	}
+	c, err := grammar.ParseCalc(args[0])
+	if err != nil {
+		return nil
+	}
+	return c.Operands
+}

internal/builtinfunc/checksum.go

@@ -1,4 +1,4 @@
-package fejkdata
+package builtinfunc
 
 import "fmt"
 
@@ -65,16 +65,14 @@ func eanCheck(s string) string {
 // a left-to-right derivation; it generates the whole value instead.
 var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
 
-func ibanArg(_ map[string]node, a []string) error {
+func ibanArg(a []string) error {
 	if _, ok := ibanLen[a[0]]; !ok {
 		return fmt.Errorf("iban(%q): unsupported country code", a[0])
 	}
 	return nil
 }
 
-// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right
+// iban generates an IBAN for cc: a numeric BBAN of the right length, then mod-97 check digits.
-// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
-// the result passes length and checksum validation, which is what fake data needs.
 func iban(r rng, cc string) string {
 	bban := make([]byte, ibanLen[cc]-4)
 	for i := range bban {

internal/builtinfunc/layout.go

@@ -1,39 +1,23 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"fmt"
 	"strings"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 const dayLayout = "2006-01-02"
 
-// The instants a layout is proved against: layoutProbe2 is alike in no field, while
-// layoutDay differs from layoutProbe in its date fields alone and layoutClock in its
-// clock fields alone, so formatting two of them tells which kind a layout names.
-var (
-	layoutProbe  = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
-	layoutProbe2 = time.Date(2010, 11, 12, 13, 14, 15, 0, time.UTC)
-	layoutDay    = time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
-	layoutClock  = time.Date(2001, 2, 3, 13, 14, 15, 0, time.UTC)
-)
-
-func namesAField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutProbe2.Format(layout)
-}
-func namesADateField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutDay.Format(layout)
-}
-func namesAClockField(layout string) bool {
-	return layoutProbe.Format(layout) != layoutClock.Format(layout)
-}
-
 // quotedLayout reports whether an arg carries the single quotes a layout is written in.
 func quotedLayout(a string) bool {
 	return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\''
 }
 
-// layoutArg is the Go layout a quoted arg holds, refused when unquoted or constant.
+// layoutArg is the Go layout a quoted arg holds, refused when unquoted.
+// docs/decisions.md#a-layout-is-always-quoted
 func layoutArg(a string) (string, error) {
 	if !quotedLayout(a) {
 		bare := strings.Trim(a, `'"`)
@@ -42,18 +26,14 @@ func layoutArg(a string) (string, error) {
 		}
 		return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare)
 	}
-	layout := a[1 : len(a)-1]
+	return a[1 : len(a)-1], nil
-	if !namesAField(layout) {
-		return "", fmt.Errorf("layout '%s' names no field, so it is the constant %q; write it as text", layout, layout)
-	}
-	return layout, nil
 }
 
 // layoutOf is layoutArg for an arg a check already validated.
 func layoutOf(a string) string {
 	layout, err := layoutArg(a)
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a, err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a, err))
 	}
 	return layout
 }
@@ -81,7 +61,7 @@ func holdsQuotedLayout(a []string) bool {
 	}
 	return false
 }
-func dateArgs(_ map[string]node, a []string) error {
+func dateArgs(a []string) error {
 	if err := layoutArity("date", 3, a); err != nil {
 		return err
 	}
@@ -96,51 +76,45 @@ func dateArgs(_ map[string]node, a []string) error {
 	if to.Before(from) {
 		return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1])
 	}
-	layout, err := layoutArg(a[2])
+	_, err = layoutArg(a[2])
-	if err != nil {
+	return err
-		return err
-	}
-	if !namesADateField(layout) {
-		return fmt.Errorf("date(from,to,layout): '%s' names no date field; write time('%s')", layout, layout)
-	}
-	if from.Equal(to) && !namesAClockField(layout) {
-		return fmt.Errorf("date(%s,%s,'%s') is the constant %q; write it as text", a[0], a[1], layout, from.Format(layout))
-	}
-	return nil
 }
-func timeArg(_ map[string]node, a []string) error {
+func timeArg(a []string) error {
 	if err := layoutArity("time", 1, a); err != nil {
 		return err
 	}
 	layout, err := layoutArg(a[0])
-	if err != nil {
+	if err == nil && namesADateField(layout) {
-		return err
+		return fmt.Errorf("time(layout): '%s' names a date field, and time draws no date; write date(from,to,layout)", layout)
 	}
-	if namesADateField(layout) {
+	return err
-		return fmt.Errorf("time(layout): '%s' names a date field; write date(from,to,layout)", layout)
+}
-	}
+
-	return nil
+// namesADateField reports a layout that prints two instants differing in their date alone apart.
+func namesADateField(layout string) bool {
+	day, nextYear := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC), time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
+	return day.Format(layout) != nextYear.Format(layout)
 }
 
 // datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted
 // in seconds, since a Duration overflows past 292 years.
-func datePrep(a []string) callFn {
+func datePrep(a []string) Call {
 	from, err := time.Parse(dayLayout, a[0])
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[0], err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[0], err))
 	}
 	to, err := time.Parse(dayLayout, a[1])
 	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[1], err))
+		panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[1], err))
 	}
 	layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400
-	return func(s *session, _ string, _ []string) string {
+	return func(s *drawstate.State, _ string, _ []string) string {
 		return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout)
 	}
 }
-func timePrep(a []string) callFn {
+func timePrep(a []string) Call {
 	layout := layoutOf(a[0])
-	return func(s *session, _ string, _ []string) string {
+	return func(s *drawstate.State, _ string, _ []string) string {
 		return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout)
 	}
 }

internal/builtinfunc/transform.go

@@ -1,9 +1,13 @@
-package fejkdata
+package builtinfunc
 
 import (
 	"fmt"
 	"strings"
 	"unicode"
+
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 // transforms are the builtins that rewrite one operand's value; they nest, so
@@ -14,11 +18,21 @@ var transforms = map[string]func(string) string{
 	"uppercase": strings.ToUpper,
 }
 
+func withTransforms(calls map[string]builtin) map[string]builtin {
+	for name, fn := range transforms {
+		if _, clash := calls[name]; clash {
+			panic(invariant.Broken("%s is registered as a builtin and a transform", name))
+		}
+		calls[name] = builtin{arity: 1, checkArgs: transformArg, prep: transformPrep(fn), operands: transformOperand}
+	}
+	return calls
+}
+
 // unwrapTransform peels nested transform calls off an operand arg, returning the
 // field it finally names and the transforms to apply, innermost last.
 func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
 	for {
-		name, args, isCall := funcCall(arg)
+		name, args, isCall := grammar.FuncCall(arg)
 		if !isCall {
 			return arg, chain, nil
 		}
@@ -33,31 +47,29 @@ func unwrapTransform(arg string) (leaf string, chain []func(string) string, err
 		arg = args[0]
 	}
 }
-func transformArg(fields map[string]node, a []string) error {
+
-	leaf, _, err := unwrapTransform(a[0])
+// transformArg checks that each call nested in the arg is a transform taking one arg.
-	if err != nil {
+func transformArg(a []string) error {
-		return err
+	_, _, err := unwrapTransform(a[0])
-	}
+	return err
-	if isRef(leaf) {
-		_, _, err := refShape(leaf)
-		return err
-	}
-	return checkArm(leaf, fields, false)
 }
 func transformOperand(a []string) []string {
+	if len(a) != 1 {
+		return nil
+	}
 	leaf, _, err := unwrapTransform(a[0])
 	if err != nil {
 		return nil
 	}
 	return []string{leaf}
 }
-func transformPrep(outer func(string) string) func([]string) callFn {
+func transformPrep(outer func(string) string) func([]string) Call {
-	return func(a []string) callFn {
+	return func(a []string) Call {
 		_, chain, err := unwrapTransform(a[0])
 		if err != nil {
-			panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err))
+			panic(invariant.Broken("transform arg %q reached prep unvalidated: %v", a[0], err))
 		}
-		return func(_ *session, _ string, operands []string) string {
+		return func(_ *drawstate.State, _ string, operands []string) string {
 			v := operands[0]
 			for i := len(chain) - 1; i >= 0; i-- {
 				v = chain[i](v)

internal/datafiles/datafiles.go

@@ -0,0 +1,215 @@
+// Package datafiles walks a data tree and reads its category files and their rows.
+package datafiles
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
+)
+
+// Source is one tree to load, from the root of an fs.FS. label prefixes file names in
+// errors; onDisk marks label as a directory that must exist; at is the folder fsys mounts the
+// tree at, "" for its root.
+type Source struct {
+	fsys   fs.FS
+	label  string
+	onDisk bool
+	at     string
+}
+
+// Dir is the tree in the directory path, its errors labelled with it.
+func Dir(path string) Source {
+	return Source{fsys: os.DirFS(path), label: path, onDisk: true}
+}
+
+// FS is the tree in fsys.
+func FS(fsys fs.FS) Source { return Source{fsys: fsys} }
+
+// Mounted is s with its tree under the folders the dot path namespace spells, "" for none.
+func (s Source) Mounted(namespace string) Source {
+	if namespace != "" && s.at == "" {
+		s.at = strings.ReplaceAll(namespace, ".", "/")
+		s.fsys = mounted{fsys: s.fsys, at: s.at}
+	}
+	return s
+}
+
+// Category is one category file: the folders above it from the source's root, its name,
+// and its parsed JSON.
+type Category struct {
+	Folders []string
+	Name    string
+	JSON    jsonvalue.Value
+	rows    *rowsFiles
+}
+
+// ReadRows reads a rows file beside the category.
+func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) }
+
+// labelled is the file p as errors name it: a labelled source's as its own tree spells it,
+// below the mount.
+func (s Source) labelled(p string) string {
+	if s.label == "" {
+		return p
+	}
+	if inner, below := underMount(s.at, p); below {
+		p = inner
+	}
+	return path.Join(s.label, p)
+}
+
+// dirPath is the folder that dir names, as fsys spells it.
+func (s Source) dirPath(dir []string) string {
+	if p := path.Join(dir...); p != "" {
+		return p
+	}
+	return "."
+}
+
+// Walk hands compile every category of the tree, folders and files in name order. A
+// folder holding no category anywhere below it is skipped; a hidden file or folder is no
+// category, though a category may name a hidden rows file beside it.
+func (s Source) Walk(compile func(Category) error) error {
+	if err := s.check(); err != nil {
+		return err
+	}
+	_, err := s.walkDir(nil, compile)
+	return err
+}
+
+// check refuses a data path that is empty or names no directory.
+func (s Source) check() error {
+	if !s.onDisk {
+		return nil
+	}
+	if s.label == "" {
+		return fmt.Errorf("a data path is empty")
+	}
+	info, err := os.Stat(s.label)
+	if errors.Is(err, fs.ErrNotExist) {
+		return noDirectory(s.label)
+	}
+	if err != nil {
+		return fmt.Errorf("data path %s: %w", s.label, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("data path %s: not a directory", s.label)
+	}
+	return nil
+}
+
+// noDirectory is a data path naming nothing.
+type noDirectory string
+
+func (d noDirectory) Error() string { return "data path " + string(d) + ": no such directory" }
+
+func (noDirectory) Unwrap() error { return fs.ErrNotExist }
+
+// walkDir walks the folder that dir names, and reports whether it handed over any category.
+func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) {
+	full := s.dirPath(dir)
+	entries, err := fs.ReadDir(s.fsys, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	rows := newRowsFiles(s, full, entries)
+	handed := false
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".") {
+			continue
+		}
+		var holds bool
+		if e.IsDir() {
+			holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile)
+		} else {
+			holds, err = s.compileFile(dir, e.Name(), rows, compile)
+		}
+		if err != nil {
+			return false, err
+		}
+		handed = handed || holds
+	}
+	return handed, nil
+}
+
+// walkFolder walks a subfolder, and refuses its name if it holds a category.
+func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) {
+	holds, err := s.walkDir(dir, compile)
+	if err != nil || !holds {
+		return false, err
+	}
+	if err := grammar.CheckIdentifier(dir[len(dir)-1]); err != nil {
+		return false, fmt.Errorf("%s: folder %w", s.labelled(s.dirPath(dir)), err)
+	}
+	return true, nil
+}
+
+// Load hands compile the category file name.json in the folder dir.
+func (s Source) Load(dir []string, name string, compile func(Category) error) error {
+	full := s.dirPath(dir)
+	entries, err := fs.ReadDir(s.fsys, full)
+	if err != nil {
+		return fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	_, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile)
+	return err
+}
+
+// compileFile hands compile a *.json file as a category named after it, and reports
+// whether it was one; any other file is skipped.
+func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) {
+	if !strings.HasSuffix(file, ".json") {
+		return false, nil
+	}
+	full := path.Join(rows.dir, file)
+	name := strings.TrimSuffix(file, ".json")
+	if err := grammar.CheckIdentifier(name); err != nil {
+		return false, fmt.Errorf("%s: category %w", s.labelled(full), err)
+	}
+	b, err := fs.ReadFile(s.fsys, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	raw, err := jsonvalue.Decode(b)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	if err := compile(Category{Folders: dir, Name: name, JSON: raw, rows: rows}); err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	return true, nil
+}
+
+// rowsFiles is what a category may name beside itself: the rows files of its folder.
+type rowsFiles struct {
+	src Source
+	dir string
+	tsv map[string]bool
+}
+
+func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles {
+	files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
+			files.tsv[e.Name()] = true
+		}
+	}
+	return files
+}
+
+func (r *rowsFiles) read(name string) (string, error) {
+	if !r.tsv[name] {
+		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir))
+	}
+	b, err := fs.ReadFile(r.src.fsys, path.Join(r.dir, name))
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err)
+	}
+	return string(b), nil
+}

internal/datafiles/datafiles_test.go

@@ -0,0 +1,217 @@
+package datafiles
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+type handed struct {
+	dir  []string
+	name string
+	json any
+}
+
+func walk(t *testing.T, src Source) ([]handed, error) {
+	t.Helper()
+	var got []handed
+	err := src.Walk(func(c Category) error {
+		got = append(got, handed{c.Folders, c.Name, c.JSON.Any()})
+		return nil
+	})
+	return got, err
+}
+
+func TestWalkHandsEachCategoryInOrder(t *testing.T) {
+	got, err := walk(t, Source{fsys: fstest.MapFS{
+		"a.json":          {Data: []byte(`{"x":"1"}`)},
+		"sub/b.json":      {Data: []byte(`"y"`)},
+		"sub/notes.txt":   {Data: []byte("not data")},
+		"empty/x.txt":     {Data: []byte("not data")},
+		".git/c.json":     {Data: []byte(`"hidden"`)},
+		"sub/.d.json":     {Data: []byte(`"hidden"`)},
+		"sub/deep/e.json": {Data: []byte(`"z"`)},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []handed{
+		{nil, "a", map[string]any{"x": "1"}},
+		{[]string{"sub"}, "b", "y"},
+		{[]string{"sub", "deep"}, "e", "z"},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Walk handed %#v, want %#v", got, want)
+	}
+}
+
+func TestReadRowsReadsOnlyAFileBesideTheCategory(t *testing.T) {
+	src := Source{fsys: fstest.MapFS{
+		"t.json":     {Data: []byte(`{"rows":"t.tsv"}`)},
+		"t.tsv":      {Data: []byte("key\nA\n")},
+		"sub/u.tsv":  {Data: []byte("key\nB\n")},
+		"sub/u.json": {Data: []byte(`{"rows":"u.tsv"}`)},
+	}, label: "lbl"}
+	rows := map[string]string{}
+	err := src.Walk(func(c Category) error {
+		data, err := c.ReadRows(c.Name + ".tsv")
+		rows[c.Name] = data
+		if err != nil {
+			return err
+		}
+		if _, err := c.ReadRows("x.tsv"); err == nil || err.Error() != "rows names x.tsv, which is not beside it in "+map[string]string{"t": "lbl", "u": "lbl/sub"}[c.Name] {
+			t.Errorf("ReadRows(x.tsv) = %v, want an error saying x.tsv is not beside %s", err, c.Name)
+		}
+		return nil
+	})
+	if err != nil || rows["t"] != "key\nA\n" || rows["u"] != "key\nB\n" {
+		t.Errorf("Walk = %v, read %q", err, rows)
+	}
+}
+
+func TestWalkIgnoresARowsFileNoCategoryNames(t *testing.T) {
+	if _, err := walk(t, Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "a.tsv": {Data: []byte("key\n")}}, label: "lbl"}); err != nil {
+		t.Errorf("Walk = %v, want a.tsv ignored", err)
+	}
+}
+
+func TestARowsFileMayBeHidden(t *testing.T) {
+	var rows string
+	err := Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, ".a.tsv": {Data: []byte("key\nA\n")}}}.Walk(func(c Category) error {
+		var err error
+		rows, err = c.ReadRows(".a.tsv")
+		return err
+	})
+	if err != nil || rows != "key\nA\n" {
+		t.Errorf("Walk = %v, read %q, want the hidden rows file read", err, rows)
+	}
+}
+
+func TestWalkLabelsWhatACategoryFileGetsWrong(t *testing.T) {
+	for _, c := range []struct {
+		name    string
+		fs      fstest.MapFS
+		compile error
+		want    string
+	}{
+		{"its compile", fstest.MapFS{"a.json": {Data: []byte(`"x"`)}}, errors.New("bad"), "lbl/a.json: bad"},
+		{"its JSON", fstest.MapFS{"a.json": {Data: []byte(`{`)}}, nil, "lbl/a.json: line 1, column 1: unexpected end of JSON input"},
+		{"its name", fstest.MapFS{"a.b.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b.json: category "},
+		{"its folder's name", fstest.MapFS{"a.b/c.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b: folder "},
+	} {
+		err := Source{fsys: c.fs, label: "lbl"}.Walk(func(Category) error { return c.compile })
+		if err == nil || !strings.HasPrefix(err.Error(), c.want) {
+			t.Errorf("%s: Walk = %v, want it to start %q", c.name, err, c.want)
+		}
+	}
+}
+
+func TestWalkChecksAFolderNameOnlyWhenItHoldsData(t *testing.T) {
+	if _, err := walk(t, Source{fsys: fstest.MapFS{"a.b/notes.txt": {Data: []byte("x")}, "c.json": {Data: []byte(`"x"`)}}}); err != nil {
+		t.Errorf("Walk = %v, want a folder holding no data skipped whatever its name", err)
+	}
+}
+
+func TestWalkRefusesADiskPathThatIsNoDirectory(t *testing.T) {
+	dir := t.TempDir()
+	for _, c := range []struct{ path, want string }{
+		{"", "a data path is empty"},
+		{filepath.Join(dir, "none"), "data path " + filepath.Join(dir, "none") + ": no such directory"},
+	} {
+		_, err := walk(t, Dir(c.path))
+		if err == nil || err.Error() != c.want {
+			t.Errorf("Walk(%q) = %v, want %q", c.path, err, c.want)
+		}
+	}
+}
+
+func TestLoadHandsOneCategoryAndItsRows(t *testing.T) {
+	src := Source{fsys: fstest.MapFS{
+		"sub/t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
+		"sub/t.tsv":  {Data: []byte("key\nA\n")},
+		"sub/u.json": {Data: []byte(`{`)},
+	}}
+	var got handed
+	var rows string
+	err := src.Load([]string{"sub"}, "t", func(c Category) error {
+		got = handed{c.Folders, c.Name, c.JSON.Any()}
+		var err error
+		rows, err = c.ReadRows("t.tsv")
+		return err
+	})
+	if err != nil || got.name != "t" || !reflect.DeepEqual(got.dir, []string{"sub"}) || rows != "key\nA\n" {
+		t.Errorf("Load = %v, handed %#v with rows %q", err, got, rows)
+	}
+}
+
+func TestDirReadsTheDirectoryAndLabelsItsErrorsWithIt(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "a.json"), []byte(`{`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := walk(t, Dir(dir))
+	if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "a.json"))+": ") {
+		t.Errorf("Walk = %v, want the error labelled with the directory", err)
+	}
+	if _, err := walk(t, Dir(filepath.Join(dir, "a.json"))); err == nil || err.Error() != "data path "+filepath.Join(dir, "a.json")+": not a directory" {
+		t.Errorf("Walk = %v, want a file refused as no directory", err)
+	}
+}
+
+func TestAMountedDirNamesItsFilesWhereTheyAreOnDisk(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "sub", "a.json"), []byte(`{`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := walk(t, Dir(dir).Mounted("my.data"))
+	if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "sub", "a.json"))+": ") {
+		t.Errorf("Walk = %v, want the error to name the file on disk", err)
+	}
+}
+
+func TestAMountedTreeListsAndReadsUnderItsMount(t *testing.T) {
+	m := mounted{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "sub/b.json": {Data: []byte(`"y"`)}}, at: "my/data"}
+	for dir, want := range map[string][]string{".": {"my"}, "my": {"data"}, "my/data": {"a.json", "sub"}, "my/data/sub": {"b.json"}} {
+		entries, err := fs.ReadDir(m, dir)
+		var got []string
+		for _, e := range entries {
+			got = append(got, e.Name())
+		}
+		if err != nil || !reflect.DeepEqual(got, want) {
+			t.Errorf("ReadDir(%q) = %v, %v, want %v", dir, got, err, want)
+		}
+	}
+	if b, err := fs.ReadFile(m, "my/data/sub/b.json"); err != nil || string(b) != `"y"` {
+		t.Errorf(`ReadFile("my/data/sub/b.json") = %q, %v, want "y"`, b, err)
+	}
+	if _, err := fs.ReadFile(m, "other/a.json"); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf(`ReadFile("other/a.json") = %v, want it not to exist`, err)
+	}
+}
+
+func TestAMountedSourceNamesItsManifestBesideItsFiles(t *testing.T) {
+	dir := t.TempDir()
+	for name, c := range map[string]struct {
+		src  Source
+		want string
+	}{
+		"an fs.FS":    {FS(fstest.MapFS{}).Mounted("my.data"), "my/data/.fejkdata.json"},
+		"a data path": {Dir(dir).Mounted("my.data"), filepath.ToSlash(filepath.Join(dir, ".fejkdata.json"))},
+	} {
+		if got := c.src.ManifestPath(); got != c.want {
+			t.Errorf("%s: ManifestPath() = %q, want %q", name, got, c.want)
+		}
+	}
+	m, err := FS(fstest.MapFS{ManifestFile: {Data: []byte(`{"namespace": "my.data"}`)}}).Mounted("my.data").Manifest()
+	if err != nil || m.Namespace != "my.data" {
+		t.Errorf("Manifest() after Mounted = %+v, %v, want the manifest read from beside the files", m, err)
+	}
+}

internal/datafiles/manifest.go

@@ -0,0 +1,246 @@
+package datafiles
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
+)
+
+// ManifestFile is the file at a source's root that describes the source.
+const ManifestFile = ".fejkdata.json"
+
+// Manifest is what a source's manifest says of it.
+type Manifest struct {
+	// Index is every category of the source by dot path; nil where the manifest has none.
+	Index map[string]IndexEntry
+	// Namespace is the dot path the source's root sits under, "" for the namespace's root.
+	Namespace string
+}
+
+// IndexEntry is what an index says of a category: its parent table, "" for none, and the
+// paths List advertises below it.
+type IndexEntry struct {
+	Parent string   `json:"parent,omitempty"`
+	Paths  []string `json:"paths"`
+}
+
+// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
+func (s Source) Manifest() (Manifest, error) {
+	if err := s.check(); err != nil {
+		return Manifest{}, err
+	}
+	b, err := fs.ReadFile(s.fsys, path.Join(s.at, ManifestFile))
+	if errors.Is(err, fs.ErrNotExist) {
+		return Manifest{}, nil
+	}
+	if err != nil {
+		return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
+	}
+	m, err := decodeManifest(b)
+	if err != nil {
+		return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
+	}
+	return m, nil
+}
+
+// ManifestPath is the manifest's path as errors name it.
+func (s Source) ManifestPath() string { return s.labelled(path.Join(s.at, ManifestFile)) }
+
+// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
+// reports every mistake: first each entry's own, in path order, then each parent naming
+// no entry.
+func decodeManifest(b []byte) (Manifest, error) {
+	raw, err := jsonvalue.Decode(b)
+	if err != nil {
+		return Manifest{}, err
+	}
+	top, isObject := raw.Any().(map[string]any)
+	if !isObject {
+		return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any()))
+	}
+	var m Manifest
+	var errs []error
+	for _, k := range sortedKeys(top) {
+		switch k {
+		case "index":
+			var indexErrs []error
+			m.Index, indexErrs = decodeIndex(top[k])
+			errs = append(errs, indexErrs...)
+		case "namespace":
+			var err error
+			m.Namespace, err = decodeNamespace(top[k])
+			errs = append(errs, err)
+		default:
+			errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index and namespace", k))
+		}
+	}
+	for _, p := range sortedKeys(m.Index) {
+		if m.Namespace != "" && !strings.HasPrefix(p, m.Namespace+".") {
+			errs = append(errs, fmt.Errorf("index entry %q lies outside the namespace %q", p, m.Namespace))
+		}
+	}
+	return m, errors.Join(errs...)
+}
+
+func decodeNamespace(v any) (string, error) {
+	namespace, isString := v.(string)
+	if !isString {
+		return "", fmt.Errorf("namespace must be a string, not %s", jsonvalue.Kind(v))
+	}
+	if namespace == "" {
+		return "", nil
+	}
+	if err := checkCategoryPath(namespace); err != nil {
+		return "", fmt.Errorf("namespace %w", err)
+	}
+	return namespace, nil
+}
+
+func decodeIndex(v any) (map[string]IndexEntry, []error) {
+	entries, isObject := v.(map[string]any)
+	if !isObject {
+		return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))}
+	}
+	index := map[string]IndexEntry{}
+	var errs []error
+	for _, p := range sortedKeys(entries) {
+		e, err := decodeEntry(p, entries[p])
+		if err != nil {
+			errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
+		}
+		index[p] = e
+	}
+	for _, p := range sortedKeys(index) {
+		for _, err := range []error{checkOutside(index, p), checkParent(index, p)} {
+			if err != nil {
+				errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
+			}
+		}
+	}
+	return index, errs
+}
+
+// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no
+// mistake twice.
+func decodeEntry(p string, v any) (IndexEntry, error) {
+	var e IndexEntry
+	errs := []error{checkCategoryPath(p)}
+	fields, isObject := v.(map[string]any)
+	if !isObject {
+		errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v)))
+		return e, errors.Join(errs...)
+	}
+	if _, has := fields["paths"]; !has {
+		errs = append(errs, fmt.Errorf("paths is missing"))
+	}
+	for _, k := range sortedKeys(fields) {
+		switch k {
+		case "parent":
+			parent, err := decodeParent(fields[k])
+			e.Parent = parent
+			errs = append(errs, err)
+		case "paths":
+			var err error
+			e.Paths, err = decodePaths(fields[k])
+			errs = append(errs, err)
+		default:
+			errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k))
+		}
+	}
+	return e, errors.Join(errs...)
+}
+
+func decodeParent(v any) (string, error) {
+	parent, isString := v.(string)
+	if !isString {
+		return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v))
+	}
+	if err := grammar.CheckIdentifier(parent); err != nil {
+		return "", fmt.Errorf("parent %w", err)
+	}
+	return parent, nil
+}
+
+// decodePaths decodes a list of paths below a category, "" being the category itself.
+func decodePaths(v any) ([]string, error) {
+	items, isList := v.([]any)
+	if !isList {
+		return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v))
+	}
+	out := make([]string, 0, len(items))
+	seen := map[string]bool{}
+	var errs []error
+	for i, item := range items {
+		q, isString := item.(string)
+		if !isString {
+			errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item)))
+			continue
+		}
+		if q != "" {
+			if err := checkCategoryPath(q); err != nil {
+				errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err))
+				continue
+			}
+		}
+		if seen[q] {
+			errs = append(errs, fmt.Errorf("paths item %d repeats %q", i+1, q))
+			continue
+		}
+		seen[q] = true
+		out = append(out, q)
+	}
+	return out, errors.Join(errs...)
+}
+
+// checkOutside refuses an entry whose path passes through another entry's category.
+func checkOutside(index map[string]IndexEntry, p string) error {
+	segs := strings.Split(p, ".")
+	for n := 1; n < len(segs); n++ {
+		outer := strings.Join(segs[:n], ".")
+		if _, isEntry := index[outer]; isEntry {
+			return fmt.Errorf("%q is a category, so it holds no other", outer)
+		}
+	}
+	return nil
+}
+
+// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
+// from one source.
+func checkParent(index map[string]IndexEntry, p string) error {
+	parent := index[p].Parent
+	if parent == "" {
+		return nil
+	}
+	sibling := parent
+	if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
+		sibling = p[:dot+1] + parent
+	}
+	if _, ok := index[sibling]; !ok {
+		return fmt.Errorf("parent %q names no entry in this entry's folder; a table in an indexed source links only to a parent in the same index, so to link under another source's table, drop this index, and the source loads whole at start", parent)
+	}
+	return nil
+}
+
+func checkCategoryPath(p string) error {
+	for _, seg := range strings.Split(p, ".") {
+		if err := grammar.CheckIdentifier(seg); err != nil {
+			return fmt.Errorf("%q: %w", p, err)
+		}
+	}
+	return nil
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

internal/datafiles/mount.go

@@ -0,0 +1,73 @@
+package datafiles
+
+import (
+	"io/fs"
+	"strings"
+)
+
+// mounted is fsys with its root moved down to the folder at, so a file f of fsys is at/f. It
+// serves what a walk asks of a tree, a folder's entries and a file's bytes: the folders above
+// at list only the next folder towards it, and opening one fails.
+type mounted struct {
+	fsys fs.FS
+	at   string
+}
+
+func (m mounted) Open(name string) (fs.File, error) {
+	if inner, below := m.inner(name); below {
+		return m.fsys.Open(inner)
+	}
+	return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+}
+
+func (m mounted) ReadFile(name string) ([]byte, error) {
+	if inner, below := m.inner(name); below {
+		return fs.ReadFile(m.fsys, inner)
+	}
+	return nil, &fs.PathError{Op: "read", Path: name, Err: fs.ErrNotExist}
+}
+
+func (m mounted) ReadDir(name string) ([]fs.DirEntry, error) {
+	if inner, below := m.inner(name); below {
+		return fs.ReadDir(m.fsys, inner)
+	}
+	rest, above := strings.CutPrefix(m.at, name+"/")
+	if name == "." {
+		rest, above = m.at, true
+	}
+	if !above {
+		return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
+	}
+	next, _, _ := strings.Cut(rest, "/")
+	return []fs.DirEntry{folderEntry(next)}, nil
+}
+
+// inner is name as fsys spells it, where name lies at or below at.
+func (m mounted) inner(name string) (string, bool) {
+	inner, below := underMount(m.at, name)
+	return inner, below && fs.ValidPath(name)
+}
+
+// underMount is name as the tree mounted at at spells it, where name lies at or below at.
+func underMount(at, name string) (string, bool) {
+	switch name {
+	case at:
+		return ".", true
+	case "":
+		return name, false
+	}
+	if at == "" {
+		return name, true
+	}
+	return strings.CutPrefix(name, at+"/")
+}
+
+// folderEntry is a folder above the mount, as a folder's entries list it.
+type folderEntry string
+
+func (e folderEntry) Name() string      { return string(e) }
+func (e folderEntry) IsDir() bool       { return true }
+func (e folderEntry) Type() fs.FileMode { return fs.ModeDir }
+func (e folderEntry) Info() (fs.FileInfo, error) {
+	return nil, &fs.PathError{Op: "stat", Path: string(e), Err: fs.ErrNotExist}
+}

internal/datatype/datatype.go

@@ -0,0 +1,39 @@
+// Package datatype names what a record column holds.
+package datatype
+
+import "fmt"
+
+// DataType is what a record column holds, which decides how a record writes its value.
+type DataType int
+
+// The datatypes a column declares with "datatype"; a column without one is a string.
+const (
+	String DataType = iota
+	Integer
+	Number
+	Boolean
+)
+
+// Count is the number of datatypes, for an array indexed by one.
+const Count = Boolean + 1
+
+var (
+	names = [Count]string{"string", "integer", "number", "boolean"}
+	nouns = [Count]string{"text", "an integer", "a number", "a boolean"}
+)
+
+// String is the datatype as data spells it.
+func (d DataType) String() string {
+	if d < 0 || d >= Count {
+		return fmt.Sprintf("DataType(%d)", int(d))
+	}
+	return names[d]
+}
+
+// Noun is the datatype as an error names what a value is.
+func Noun(d DataType) string {
+	if d < 0 || d >= Count {
+		return d.String()
+	}
+	return nouns[d]
+}

internal/datatype/datatype_test.go

@@ -0,0 +1,31 @@
+package datatype
+
+import "testing"
+
+func TestEachDatatypeIsSpelledAsDataWritesIt(t *testing.T) {
+	for d, want := range map[DataType][2]string{
+		String:  {"string", "text"},
+		Integer: {"integer", "an integer"},
+		Number:  {"number", "a number"},
+		Boolean: {"boolean", "a boolean"},
+	} {
+		if d.String() != want[0] || Noun(d) != want[1] {
+			t.Errorf("%d: String = %q, Noun = %q, want %q", int(d), d.String(), Noun(d), want)
+		}
+	}
+	if Count != 4 {
+		t.Errorf("Count = %d, want the four datatypes", Count)
+	}
+}
+
+func TestStringNamesADatatypeNoDataWrites(t *testing.T) {
+	if got := DataType(9).String(); got != "DataType(9)" {
+		t.Errorf("String = %q, want DataType(9)", got)
+	}
+}
+
+func TestNounNamesADatatypeNoDataWrites(t *testing.T) {
+	if got := Noun(DataType(9)); got != "DataType(9)" {
+		t.Errorf("Noun = %q, want DataType(9)", got)
+	}
+}

internal/drawstate/drawstate.go

@@ -0,0 +1,37 @@
+// Package drawstate holds a generator's seeded randomness and its {seq()} counters.
+package drawstate
+
+import (
+	"math/rand/v2"
+	"sort"
+)
+
+// State is one generator's draw state, kept across its renders.
+type State struct {
+	rand     *rand.Rand
+	counters map[string]uint64
+}
+
+// New builds the state a seed reproduces.
+func New(seed uint64) *State {
+	return &State{rand: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)), counters: map[string]uint64{}}
+}
+
+func (s *State) IntN(n int) int { return s.rand.IntN(n) }
+
+func (s *State) Float64() float64 { return s.rand.Float64() }
+
+// Weighted draws an index into cum, a running sum of weights of 0 or more with a positive total,
+// with odds by weight, so an index of weight 0 is never drawn.
+func (s *State) Weighted(cum []float64) int {
+	total := cum[len(cum)-1]
+	x := s.Float64() * total
+	// x stays below total: a product of a float64 under 1 and a positive float64 rounds below it.
+	return sort.Search(len(cum), func(i int) bool { return cum[i] > x })
+}
+
+// Seq advances the counter {seq(key)} reads, or {seq()} for an empty key, and returns it.
+func (s *State) Seq(key string) uint64 {
+	s.counters[key]++
+	return s.counters[key]
+}