Make great documentation goal 1.2, and renumber goals 13 to 16 as 12 to 15 wherever they are cited

.github/workflows/packages.yml

@@ -0,0 +1,162 @@
+name: Packages
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        required: true
+    secrets:
+      TAP_GITHUB_TOKEN:
+        required: true
+      PACKAGES_GPG_KEY:
+        required: true
+      PACKAGES_RSA_KEY:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The released tag to package, such as v0.1.0
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: packages
+  cancel-in-progress: false
+
+jobs:
+  packages:
+    name: Binaries, packages and image for ${{ inputs.tag }}
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+      packages: write
+    outputs:
+      latest: ${{ steps.latest.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.tag }}
+          path: src
+          fetch-depth: 0
+          persist-credentials: false
+      # Check out the packaging config at the workflow's commit, so a tag cut before the
+      # config existed can still be packaged.
+      - uses: actions/checkout@v7
+        with:
+          path: tooling
+          persist-credentials: false
+      - name: The Go version the gate tests
+        id: go
+        run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: false
+      - name: Channels holding one version
+        id: latest
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
+          if [ "$latest" = "$TAG" ]; then
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
+            echo "OLDER_TAG=1" >> "$GITHUB_ENV"
+          fi
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: v2.18.3
+          workdir: src
+          args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      - name: The built binary and the published image print the tag
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
+          test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  dockerhub:
+    name: Docker Hub image for ${{ inputs.tag }}
+    needs: packages
+    runs-on: ubuntu-26.04
+    permissions:
+      packages: read
+    steps:
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Copy the GHCR image to Docker Hub, and check it prints the tag
+        env:
+          LATEST: ${{ needs.packages.outputs.latest }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
+          if [ "$LATEST" = true ]; then
+            tags+=(-t docker.io/larvit/fejkdata:latest)
+          fi
+          docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
+          test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  repos:
+    name: apt, dnf and apk repositories for ${{ inputs.tag }}
+    needs: packages
+    if: needs.packages.outputs.latest == 'true'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: The release's Linux packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
+      - uses: docker/setup-buildx-action@v4
+      - name: Install fejkdata from each repository, then export them
+        env:
+          PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+          PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          build() {
+            docker buildx build -f release-tooling/package-repos.Dockerfile \
+              --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
+              --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
+          }
+          build --target checks --output type=local,dest=checks
+          build --target site --output type=local,dest=site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - id: deploy
+        uses: actions/deploy-pages@v5

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * *'
 
 permissions:
   contents: read
@@ -11,9 +14,10 @@ permissions:
 jobs:
   test:
     name: vet + fmt + tests
-    runs-on: ubuntu-24.04
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -26,11 +30,16 @@ jobs:
           else
             base='${{ github.event.before }}'
           fi
-          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
             echo "code=false" >> "$GITHUB_OUTPUT"
           else
             echo "code=true" >> "$GITHUB_OUTPUT"
           fi
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
+            echo "packaging=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "packaging=true" >> "$GITHUB_OUTPUT"
+          fi
           echo "base=$base" >> "$GITHUB_OUTPUT"
       - name: CHANGELOG entry for data changes
         run: |
@@ -48,22 +57,84 @@ jobs:
       # failure is version-specific.
       - name: Lowest supported Go
         if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
-        run: docker build --build-arg GO_VERSION=1.22.12 .
+        run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
+      - name: Package repositories, signed with throwaway keys
+        if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
+        run: |
+          docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
+          PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
+            docker build -f release-tooling/package-repos.Dockerfile --target checks \
+            --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
 
-  release:
+  vulnerabilities:
-    name: Gitea release from CHANGELOG.md
+    name: Known vulnerabilities in this Go version and the required modules
-    needs: test
+    runs-on: ubuntu-26.04
-    if: github.event_name == 'push'
+    timeout-minutes: 10
-    runs-on: ubuntu-24.04
     permissions:
+      actions: write
       contents: write
+      pull-requests: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - run: python3 release-tooling/publish_release.py
+      - name: Fetch govulncheck's report from vuln.go.dev
+        run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
+      - name: File each one in todo.md
+        id: file
+        run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
+      - name: Propose the filed items
+        if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
         env:
-          GITEA_API_URL: ${{ github.api_url }}
+          GH_TOKEN: ${{ github.token }}
-          GITEA_REPOSITORY: ${{ github.repository }}
+        run: |
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
+            exit 0
+          fi
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git switch -C vulnerabilities
+          git commit -qam "File the known vulnerabilities govulncheck reports"
+          git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
+          gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
+            --body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
+          # A pull request the workflow's token opens starts no workflow.
+          gh workflow run test.yml --ref vulnerabilities
+
+  release:
+    name: Release from CHANGELOG.md
+    needs: [test, vulnerabilities]
+    if: github.event_name == 'push'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    outputs:
+      tag: ${{ steps.publish.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - id: publish
+        run: python3 release-tooling/publish_release.py
+        env:
+          FORGE_API_URL: ${{ github.api_url }}
+          FORGE_REPOSITORY: ${{ github.repository }}
+          FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SHA: ${{ github.sha }}
+
+  packages:
+    needs: release
+    if: needs.release.outputs.tag != ''
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/packages.yml
+    with:
+      tag: ${{ needs.release.outputs.tag }}
+    secrets:
+      TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+      PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

AGENTS.md

@@ -1,15 +1,137 @@
 # Rules
 
-- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check.
+- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
 - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
 - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
 - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
-- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it
+- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
-  offers fails with "is it still open?", as does a merge whose required checks are
+  only, `gh pr merge --rebase` where the branch is already on top of main.
-  still pending, so read the checks before believing the style is the problem. Use
+- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
-  `tea api /repos/{owner}/{repo}/pulls/<n>/merge -f Do=fast-forward-only`.
+  its status once when a PR is marked ready, answer a review if one is there, and never
+  wait for one.
+- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
+  nothing else merges — no feature, no category, no data — bar a security fix, a
+  dependency bump, a repair of behaviour the README documents, the infrastructure
+  the round itself runs on, and a chunk the comprehension-floor decision lists as
+  shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
+  findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
+  again, until the score passes. At or above 7.0
+  every pull request scores with the four-seat run and answers it in one run. No merge
+  lowers the last score its own panel recorded: the nine-seat round and the four-seat
+  run are two ratchets, never compared with each other. A lower score blocks the merge
+  even where it may be panel noise: fix what the seats name until the score rises. A PR
+  that only records a run's scores and files what its seats name merges even when a
+  score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
 - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
-- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
+- Prose naming a source states what that source states: read the register's own field
-- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
+  before paraphrasing it, and name the register you read before claiming none
+  publishes a spelling.
 - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
-- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed.
+- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
+
+# Decisions
+
+In [docs/decisions.md](docs/decisions.md):
+
+- GitHub is canonical, and the module path names it
+- The Gitea copy stays, as a pull mirror of GitHub
+- The vocabulary sits below `doc.go`'s package clause, not in the package doc
+- Options and fields share one namespace
+- `{a|b}` stays beside nested choices
+- Flags follow getopt_long
+- A struct tag is a template by its shape
+- The CLI renders exactly the template on its stdin, and adds nothing
+- The CLI reads only the library's public API
+- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+- `FakeTemplate` and `NewTemplate` both stay
+- The shipped data is embedded, not discovered
+- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+- Every `{…}` draws afresh, and only a name keeps a pick
+- Reference sigils follow the filesystem
+- A change to what exists is a major; a minor only adds
+- An error is a contract by what it names, not its bytes
+- A load reports every mistake at once
+- Formats are checked after all data is loaded, so their errors name the category
+- Raising the lowest supported Go is a major
+- The format check runs on the latest Go only
+- The changelog heading is the one spelling of a release; CI cuts the tag
+- A `--data-path` override rebinds every reference to the category it replaces
+- The repeat cap bounds renders, not bytes
+- 64-bit targets only
+- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+- Samples say what they emit, transforms what they do
+- A record is a template seen as columns; a Go struct is the one second schema
+- A struct's records follow Go's field access, and compile on first use
+- A category may reference itself, and a record's fences run at load
+- A record's column set is fixed before the first draw
+- Null is a `null` item, not a rate
+- A typed column holds one value, not composed text
+- A column that only reads one reference or name is the column it reads
+- A typed column's calc is refused unless proven
+- `Column` carries text, not a Go value
+- A unit takes the stem of what it is, and a file the stem of the units it holds
+- One name, one meaning
+- The performance gate asserts allocations, not wall-clock time
+- Rows live in a TSV, the shape in JSON
+- A selector is bracketed, and a dot inside it is literal
+- `parent` names the link column and the table alike
+- After a row, a path names a column or a linked table
+- A selected row is fixed, not drawn
+- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+- A table selects by one key column, a code, never a free-form name
+- A parent row with no child row is a load error
+- A table is a record of string columns
+- The key map is built at load, the rest on first draw
+- Two categories may name one TSV
+- A cell may read a reference, and each row its own
+- A path is walked once without drawing before it is walked for real
+- A country's postal codes and streets are siblings under its locality
+- A locale package imports its country's geo package, and its `Modules()` returns both
+- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+- A locale's `address` binds its country's record once and prints it whole
+- A postort's kommun comes from its name, its tätort or its codes, never from distance
+- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+- `--list` stays a plain list of paths
+- A layout is always quoted
+- A title is a table under `sex`
+- A table owns the spelling of a selector on it
+- No builtin reads the clock, so a date is bounded by days, never by an age
+- `misc` is what every locale shares
+- A register's canonical spelling loses to the one its domain writes
+- `misc.territory` is the spine, and a `misc` table naming a territory links to it
+- A table whose register publishes no frequency draws evenly
+- `misc.timezone` weighs a zone by the people living in it
+- `misc.car` is one flat table, not a make linked to its models
+- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+- `misc.tld` keys carry the leading dot, where other tables key on a bare code
+- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+- `misc.territory` carries a currency code, it does not link to `misc.currency`
+- An extension may name two media types
+- The Swedish ids draw Skatteverket's test series
+- The US given names come from a mirror of the SSA file
+- `List` advertises direct descents only
+- A path draws through its compiled steps
+- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+- A country with no open address register covering it whole ships its places and postal codes without streets
+- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
+
+# Scoring run
+
+The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
+
+- `{language}`: Go
+- `{kind}`: a Go library and CLI that renders fake data from JSON templates
+- `{domain}`: fake test-data generation and template languages
+- `{domain docs}`: the documentation of any fake-data library, such as Faker
+- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another

CHANGELOG.md

@@ -8,72 +8,99 @@ replacement, and each removed path, column or flag.
 
 ### Added
 
-- First release: the CLI, the library and the shipped data set.
+- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
-- Table categories: a category JSON naming a `rows` TSV beside it, with the options
+  dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
-  `key`, `name`, `weight` and `parent`; a path selects a row by key or name,
+  the CLI installs without Go. 0.1.0 ships them too; the README's
-  `misc.territory[SE]`, and descends to a linked table by name; linked tables draw
+  [Install](https://github.com/larvit/fejkdata#install) lists each command.
-  consistently within one render and draw group. `rows` is an option, so no
+
-  template may carry a field of that name. A `name` without a `key` resolves
+## [0.1.0] - 2026-10-13
-  inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
+
-  `parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any
+The first release: a Go library and a CLI that render locale-aware fake data from JSON
-  table above it, a table whose format or cell references its own family, and,
+templates, and the data they ship with. The
-  within one render and draw group, a path drawing a table another path selects a
+[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
-  row of, or two paths pinning different rows of one table.
+Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
-- `New` refuses a root choice of templates sharing one format and one set of string
+library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
-  fields, naming the rows TSV to write instead.
+and any release may change what a seed draws, so pin one exact version
-- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and
+([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
-  `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has
+
-  a capital, a currency and a TLD, with the columns `calling-code`, `capital`,
+### Added
-  `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country`
+
-  holding the alpha-2 code of the sovereign state it belongs to and its own where it
+- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
-  is one; `misc.currency` the
+  `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
-  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
+  repeats a run's output while the version and data stay the same, and without it each
-  added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a
+  run takes a new seed; `-n` renders the template that many times, and `--list`
-  639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each
+  prints the paths the data offers.
-  code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each
+  `--data-path` layers a directory of your own data over the shipped set, and
-  type mime-db records as IANA-registered and gives a filename extension.
+  `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
-  `DATA-LICENSES.md` lists each source.
+  a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
-- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone`
+  error such as data that fails to load, and `2` on misuse.
-  is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the
+- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
-  zone's standard UTC offset and `territory` linking to `misc.territory`, so
+  `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
-  `misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory
+  from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
-  `misc.territory` does not ship, Antarctica's among them, and the constant `UTC`
+  without data, it fails and names `WithDataFS`. The same seed repeats the draws
-  are gone; spell that one as the text `"UTC"`. A `weight` column carries the
+  while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
-  population GeoNames records in each zone, so a draw favours the zones people live
+  and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
-  in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`,
+  `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
-  `device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a
+  `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
-  `model` column.
+  `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
-- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+  and `IsTemplate` tells an inline template from a path. Data that fails to load gives
+  an error matching `ErrLoad`, and a record with no fields gives one matching
+  `ErrNoColumns`.
+- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
+  `cli.Main(modules...)` runs it over the modules given, so
+  `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
+  That CLI names itself by the file it runs from, and its `--version` prints its own
+  module's version before fejkdata's.
+- The template language
+  ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
+  Every value in a data file is a string, a choice, a template, a table, a number or a
+  boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
+  root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
+  or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
+  pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
+  digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
+  `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
+  transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
+  grammar, could mean two things or cannot render a valid value.
+- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
+  a template seen as columns. A column is a string unless it declares a `datatype`,
+  `integer`, `number` or `boolean`, and a `null` item draws it as null.
+- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
+  category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
+  `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
+  linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
+  `{/misc.timezone[Europe/Oslo]..territory.name}`.
+- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
+  a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
+  manifest at a source's root may carry a `namespace`, the dot path the source mounts
+  under, and an `index`, with which each category loads on the first call reaching it.
+- The shipped data, in Go packages that each export `FS` and `Modules()`:
+  `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
+  `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
+  `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
+  [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
+  names each source.
+  - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
+    `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
+    the Census Bureau. A person's first name and sex agree, and so does an `en_US`
+    person's title; `person.prefix` is null where a person has no title.
+    `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
+    `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
+    columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
+    assign.
+  - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
+    HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
+    symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
+    zones weighted by the people living in each, user agents and cars, all as tables;
+    and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
+    ObjectIds and UUIDs.
+  - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
     `locality`, `postal-code` and `street`, weighted by population and address counts
-  and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
+    and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
-  `address` record over one consistent draw of them. `sv_SE.address` and
+    and an `address` record that draws a street and a postal code inside one
-  `en_US.address` read those records, so `en_US.address.street` no longer carries
+    locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
-  `name` and `suffix`, and a locale folder loads only beside `geo`.
+    (postal town) the import can place ships with all its streets. A US place ships
-- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or
+    only if it is incorporated and holds most of some postal code's land. Of those
-  within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`.
+    places, every state capital but Honolulu ships, as does every place of 100,000
-  `sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so
+    people or more and a tenth of all of them drawn by population. Each US place
-  `date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`,
+    carries a quarter of its streets, at least ten.
-  `time.sec` and `time.ampm` are no longer paths — a part of a date is now its own
-  `{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant.
-- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by
-  bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a
-  name both sexes carry is a row under each. `person` reads them, so its columns are
-  `first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are
-  no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead.
-  `en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms`
-  without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the
-  same shape.
-- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series
-  from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose
-  `ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the
-  ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so
-  `--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new
-  and carries the same three. `person.prefix` is null where a person has no title,
-  where it used to be an empty string, so `--format sql` writes `NULL` and a `string`
-  struct field reading it becomes `*string`.
-- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can
-  answer from the rest.
-- An error names a spelling that runs: a layout is named single-quoted and free of
-  its own quotes, a row of a table with no key is named as the path that selects it,
-  `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record
-  that gives it one.

DATA-LICENSES.md

@@ -9,18 +9,23 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
 | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
 | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
 | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
-| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
+| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
 | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
 | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
 | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
 | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
-| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 10) | — | — | — |
+| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
 | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
+| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
 | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
 | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` |
+| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from the [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `<syslog.h>` severity macros | facts | none required | `data-import/loglevel.py` |
 | `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` |
+| `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` |
+| `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` |
 | `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` |
 | `misc/timezone.tsv` | [IANA tzdb](https://www.iana.org/time-zones) 2026d `zone.tab` and the standard offset of each zone; weights from [GeoNames](https://www.geonames.org/) `cities15000` populations | [public domain](https://data.iana.org/time-zones/tzdb/LICENSE); [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | "Populations from GeoNames, www.geonames.org" | `data-import/timezone.py` |
+| `misc/tld.tsv` | [IANA Root Zone Database](https://www.iana.org/domains/root/db), checked against the root zone's [TLD list](https://data.iana.org/TLD/tlds-alpha-by-domain.txt) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/tld.py` |
 | `misc/useragent.tsv` | [top-user-agents](https://github.com/microlinkhq/top-user-agents) desktop and mobile lists | [MIT](https://github.com/microlinkhq/top-user-agents/blob/master/LICENSE.md) | "Copyright © 2020 Kiko Beats" | `data-import/useragent.py` |
 
 Every other category is hand-written under [`data/`](data), MIT like the code.

Dockerfile

@@ -1,19 +1,43 @@
-# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
+ARG GO_VERSION=1.27.2
-# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
+FROM golang:${GO_VERSION} AS portable
-# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
-# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
-ARG GO_VERSION=1.26.4
-FROM golang:${GO_VERSION}
 
 WORKDIR /app
 
-# Module layer cached separately from source.
 COPY go.mod ./
 RUN go mod download
 
 COPY . .
 RUN go vet ./... && \
 	go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
-	{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
-	  { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
 	go test -race ./...
+
+FROM golang:${GO_VERSION} AS vulnerabilities
+WORKDIR /app
+COPY . .
+RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
+
+FROM scratch AS vulnerability-report
+COPY --from=vulnerabilities /report.json /
+
+FROM python:3.14.8-slim AS release-tooling
+COPY release-tooling/ /release-tooling/
+RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
+
+FROM python:3.14.8-slim AS vulnerability-filing
+COPY vulnerability-filing/ /vulnerability-filing/
+# The filer's test files an item into the plan itself.
+COPY todo.md /todo.md
+RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
+
+FROM goreleaser/goreleaser:v2.18.3 AS packaging
+WORKDIR /check
+COPY release-tooling/goreleaser.yaml ./
+RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
+
+FROM portable
+# BuildKit skips a stage nothing copies from.
+COPY --from=release-tooling /tested /tmp/release-tooling-tested
+COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
+COPY --from=packaging /tested /tmp/packaging-tested
+RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
+	{ echo "unformatted files:"; echo "$unformatted"; exit 1; }

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2026 Timewave
+Copyright (c) 2026 Larvit
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

arm.go

@@ -0,0 +1,117 @@
+package fejkdata
+
+import (
+	"fmt"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// arm is one alternative of a {a|b} token or one operand, split into the head,
+// whose node `template.startOf` finds, and the tail of a dotted path into it.
+type arm struct {
+	spelling string // as written, for messages
+	head     string
+	tail     []string
+	levels   []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
+	steps    []pathStep
+	leaves   []node // every node the path may land on, one per variant it passes
+	kind     armKind
+	named    *nameBinding // namedRead: the binding of the name it reads through
+}
+
+// armKind is how expand reads an arm, fixed at compile.
+type armKind uint8
+
+const (
+	freshRead armKind = iota // drawn afresh at every read
+	namedRead                // read through a name, kept in its pick
+)
+
+// splitArm splits one name into head and tail. refs maps a reference to what
+// resolveRefs resolved it to; before that, a reference is whole.
+func splitArm(name string, refs map[string]resolvedRef) arm {
+	if grammar.IsRef(name) {
+		if r, resolved := refs[name]; resolved {
+			return pathArm(name, r.head, r.tail)
+		}
+		return pathArm(name, name, nil)
+	}
+	segs, err := grammar.SplitPath(name)
+	if err != nil || len(segs) == 1 {
+		return pathArm(name, name, nil)
+	}
+	return pathArm(name, segs[0], segs[1:])
+}
+
+func pathArm(name, head string, segs []string) arm {
+	return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
+}
+
+// key is the key of the leaf a lands on, the one key every way of writing this read shares.
+func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
+
+func checkSegments(a arm) error {
+	if len(a.tail) == 0 {
+		return nil
+	}
+	return grammar.CheckSegments(append([]string{a.head}, a.tail...))
+}
+
+// op is one compiled unit of a format string: a literal run, a name read,
+// or a builtin already prepared with its args.
+type op struct {
+	grammar.Token
+	arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
+	call builtinfunc.Call
+	// operands are the fields the builtin reads, in the order its operands func
+	// fixed; expand reads them before the call. nil for a builtin that reads none.
+	operands []arm
+}
+
+func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+	arms := make([]arm, len(names))
+	for i, name := range names {
+		a, err := t.compileArm(name, targets)
+		if err != nil {
+			return nil, err
+		}
+		arms[i] = a
+	}
+	return arms, nil
+}
+
+// compileArm compiles one read into a path: from the head it names, or, for a read through a
+// name, from what that name binds.
+func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
+	a := splitArm(name, t.refs.byName)
+	if !t.isName(a.head) {
+		start := t.startOf(a.head)
+		if start == nil {
+			panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
+		}
+		w := compilePath(start, a.tail)
+		a.steps, a.leaves = w.steps, w.leaves
+		return a, nil
+	}
+	b := t.nameScope.lookup(a.head)
+	if grammar.HasSelector(a.tail) {
+		if b.bindsField() {
+			return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
+		}
+		return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
+	}
+	target := targets[b]
+	full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
+	if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
+		return a, err
+	}
+	w := compilePath(target.start, full)
+	a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
+	a.levels = levelKeys(full, 0)
+	return a, nil
+}

bench_test.go

@@ -6,17 +6,17 @@ import (
 	"testing"
 )
 
-func benchGenerator(b *testing.B, dir string) *Generator {
+func benchGenerator(b *testing.B, data Option) *Generator {
 	b.Helper()
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
+	f, err := New(data, WithSeed(1))
 	if err != nil {
 		b.Fatal(err)
 	}
 	return f
 }
 
-func benchPath(b *testing.B, dir, path string) {
+func benchPath(b *testing.B, data Option, path string) {
-	f := benchGenerator(b, dir)
+	f := benchGenerator(b, data)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
 	}
 }
 
-func BenchmarkPerson(b *testing.B)       { benchPath(b, "data", "sv_SE.person") }
+func BenchmarkPerson(b *testing.B)       { benchPath(b, withShipped(), "sv_SE.person") }
-func BenchmarkAddress(b *testing.B)      { benchPath(b, "data", "sv_SE.address") }
+func BenchmarkAddress(b *testing.B)      { benchPath(b, withShipped(), "sv_SE.address") }
-func BenchmarkWord(b *testing.B)         { benchPath(b, "data", "sv_SE.word") }
+func BenchmarkWord(b *testing.B)         { benchPath(b, withShipped(), "sv_SE.word") }
-func BenchmarkCreditcard(b *testing.B)   { benchPath(b, "data", "misc.creditcard") }
+func BenchmarkCreditcard(b *testing.B)   { benchPath(b, withShipped(), "misc.creditcard") }
-func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") }
+func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
-func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, "data", "misc.uuid") }
+func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, withShipped(), "misc.uuid") }
 
 func tmpData(b *testing.B, name, body string) string {
 	b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
 
 func BenchmarkCalc(b *testing.B) {
 	dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
-	benchPath(b, dir, "inv")
+	benchPath(b, WithDataPath(dir), "inv")
 }
 
 func BenchmarkLongLiteral(b *testing.B) {
 	dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "sql")
+	benchPath(b, WithDataPath(dir), "sql")
 }
 
 func BenchmarkRepeat(b *testing.B) {
 	dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "many")
+	benchPath(b, WithDataPath(dir), "many")
 }
 
-// BenchmarkBound measures a format that binds a head and reads two paths from it —
+// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
-// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
+// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
-// two independent fields.
+// independent fields.
-func BenchmarkBound(b *testing.B) {
+func BenchmarkNamed(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
+	dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-func BenchmarkUnbound(b *testing.B) {
+func BenchmarkUnnamed(b *testing.B) {
 	dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
+// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
-// depth question is about.
+func BenchmarkNamedDeep(b *testing.B) {
-func BenchmarkBoundDeep(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
-	dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
+	benchPath(b, WithDataPath(dir), "addr")
-	benchPath(b, dir, "addr")
 }
 
-// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
+// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
-// expansion holds outgrow a single bucket.
+// outgrow a single bucket.
-func BenchmarkBoundWide(b *testing.B) {
+func BenchmarkNamedWide(b *testing.B) {
-	dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
+	dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
 		{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
 		{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
-	benchPath(b, dir, "row")
+	benchPath(b, WithDataPath(dir), "row")
+}
+
+func BenchmarkFirstFake(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		f, err := New(withShipped(), WithSeed(1))
+		if err != nil {
+			b.Fatal(err)
+		}
+		if _, err := f.Fake("sv_SE.address"); err != nil {
+			b.Fatal(err)
+		}
+	}
 }
 
 func BenchmarkNew(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if _, err := New(WithSeed(1)); err != nil {
+		if _, err := New(withShipped(), WithSeed(1)); err != nil {
 			b.Fatal(err)
 		}
 	}

builtins_test.go

@@ -2,11 +2,14 @@ package fejkdata
 
 import (
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 func TestBuiltinIDGenerators(t *testing.T) {
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
 	}
 }
 
-// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
+// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
 // each call, named counters independent, and the whole thing scoped to one
-// session so a fresh Generator restarts at 1.
+// draw state so a fresh Generator restarts at 1.
-func TestBuiltinSeqPerSession(t *testing.T) {
+func TestBuiltinSeqPerGenerator(t *testing.T) {
 	f := engine(1)
 	for i := 1; i <= 5; i++ {
 		if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
 	if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
 		t.Fatalf("repeat seq = %q, want 1,2,3", got)
 	}
-	// ...and a fresh session restarts from 1.
+	// ...and a fresh generator restarts from 1.
 	if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
-		t.Fatalf("new session seq = %q, want 1", got)
+		t.Fatalf("new generator seq = %q, want 1", got)
 	}
 }
 
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
 		`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
 	}
 	for _, tmpl := range bad {
-		if _, err := compile(parse(t, tmpl)); err == nil {
+		if _, err := resolved(t, tmpl); err == nil {
 			t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
 		}
 	}
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
 	return rem == 1
 }
 
-// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
-// supplies prep, and args parsed at compile only behind a check.
+// separates a reported invariant break from a silently wrong value.
-func TestRegistryShapes(t *testing.T) {
-	for name, b := range builtins {
-		if b.prep == nil {
-			t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
-		}
-		if b.arity != 0 && b.check == nil {
-			t.Errorf("builtin %q parses args in prep with no check", name)
-		}
-	}
-}
-
-// TestArgGuardsPanic pins the guards that report a builtin arg its check should
-// have rejected. No data reaches them — checkFunc runs a builtin's check before
-// compileOps ever calls prep — so they are exercised directly.
-func TestArgGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"atoi on an unvalidated arg": func() { atoi("nope") },
-		"atof on an unvalidated arg": func() { atof("nope") },
-	} {
-		mustPanic(t, name, call)
-	}
-}
-
-// mustPanic fails unless call panics, which is what separates a reported
-// invariant break from a silently wrong value.
 func mustPanic(t *testing.T, name string, call func()) {
 	t.Helper()
 	defer func() {
-		if recover() == nil {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
-			t.Errorf("%s: no panic, want the invariant reported", name)
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
 		}
 	}()
 	call()
 }
 
 func TestClassBuiltinArgs(t *testing.T) {
-	for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
+	for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
-		if _, err := compile(parse(t, bad)); err == nil {
+		if _, err := resolved(t, bad); err == nil {
 			t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
 		}
 	}
 	for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v", ok, err)
 		}
 	}
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
 	}
 }
 
-// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order,
+// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
-// the layout is quoted, names a field, and for time names no date field.
+// layout is quoted.
 func TestBuiltinDateArgs(t *testing.T) {
 	for tmpl, want := range map[string]string{
 		`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`:        "1990-13-01",
+		`"{time('2006-01-02 15:04')}"`:                        "write date(from,to,layout)",
 		`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`:        "is after",
-		`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`:        "write it as text",
 		`"{date(1990-01-01,1990-12-31,2006-01-02)}"`:          "'2006-01-02'",
 		`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`:    "'",
 		`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
-		`"{date(1990-01-01,1990-12-31,'x')}"`:                 "text",
-		`"{date(1990-01-01,1990-12-31,'')}"`:                  "text",
 		`"{date(1990-01-01,1990-12-31)}"`:                     "3 arguments",
 		`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`:     "'January 2, 2006'",
 		`"{time(3:04 PM, Mon)}"`:                              "'3:04 PM, Mon'",
 		`"{time(15:04)}"`:                                     "'15:04'",
-		`"{time('2006-01-02 15:04')}"`:                        "date(",
-		`"{time('x')}"`:                                       "text",
-		`"{date(1990-01-01,1990-12-31,'15:04')}"`:             "time('15:04')",
 	} {
-		_, err := compile(parse(t, tmpl))
+		_, err := resolved(t, tmpl)
 		if err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
 		}
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
 }
 
 // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
-// draws every second of it, and without one it could only emit one value.
+// draws every second of it, and without one it prints that day.
 func TestBuiltinDateSpansOneDay(t *testing.T) {
 	f := engine(1)
 	seen := map[string]bool{}
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
 	if len(seen) < 400 {
 		t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
 	}
-	_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`))
+	if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
-	if err == nil || !strings.Contains(err.Error(), "write it as text") {
+		t.Fatalf("one day in a date-only layout = %q, want that day", got)
-		t.Fatalf("one day in a date-only layout = %v, want it named a constant", err)
+	}
+}
+
+func TestClassBuiltins(t *testing.T) {
+	cases := map[string]*regexp.Regexp{
+		`"{digits(1)}"`:          regexp.MustCompile(`^[0-9]$`),
+		`"{digits(3)}"`:          regexp.MustCompile(`^[0-9]{3}$`),
+		`"{int(1,9)}"`:           regexp.MustCompile(`^[1-9]$`),
+		`"{upper(1)}"`:           regexp.MustCompile(`^[A-Z]$`),
+		`"{lower(1)}"`:           regexp.MustCompile(`^[a-z]$`),
+		`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
+	}
+	f := engine(7)
+	for tmpl, re := range cases {
+		for i := 0; i < 100; i++ {
+			if got := mustRender(t, f, tmpl); !re.MatchString(got) {
+				t.Fatalf("%s produced %q, want %s", tmpl, got, re)
+			}
+		}
+	}
+}
+
+func TestFunctionTokenLuhn(t *testing.T) {
+	// {luhn()} appends a Luhn check digit over the digits emitted so far in the
+	// current expansion (non-digits skipped but kept). It reads the output
+	// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
+	f := engine(1)
+	cases := map[string]string{
+		`"811218987{luhn()}"`:                      "8112189876",  // personnummer body
+		`"7992739871{luhn()}"`:                     "79927398713", // classic Luhn vector
+		`"811218-987{luhn()}"`:                     "811218-9876", // '-' skipped, kept
+		`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876",  // over a rendered token
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Fatalf("%s = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestBuiltinCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{upper(x)}"`,         // count must be an integer
+		`"{int(a,b)}"`,         // non-integer args
+		`"{int(5,1)}"`,         // min > max
+		`"{nanoid(-1)}"`,       // negative count
+		`"{float(1,2,-1)}"`,    // negative decimals
+		`"{float(NaN,NaN,2)}"`, // bounds must be finite
+		`"{float(Inf,Inf,2)}"`, // same-sign infinities
+		`"{float(1,NaN,2)}"`,   // one NaN bound
+		`"{float(-Inf,1,2)}"`,  // one infinite bound
+		`"{iban(US)}"`,         // unsupported country
+		`"{seq(a,b)}"`,         // seq takes at most one name
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestArgErrorsNameTheSpelling(t *testing.T) {
+	if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
+		t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
+	}
+}
+
+func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
+	for long, short := range map[string]string{
+		`"{float(1,2,02)}"`:                 `"{float(1,2,2)}"`,
+		`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
+		`"{digits(+5)}"`:                    `"{digits(5)}"`,
+		`"{digits(05)}"`:                    `"{digits(5)}"`,
+		`"{int(007,9)}"`:                    `"{int(7,9)}"`,
+		`"{int(+1,5)}"`:                     `"{int(1,5)}"`,
+	} {
+		sameRenders(t, long, short)
 	}
 }

calc.go

@@ -2,392 +2,149 @@ package fejkdata
 
 import (
 	"fmt"
-	"math"
+	"slices"
 	"strconv"
 	"strings"
-	"unicode"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// calcNode is a parsed expression node. It evaluates over the operand values expand
+// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
-// read, so the evaluator touches neither the rng nor the node tree.
+// field is a name, which checkCalcNames checks once names resolve.
-type calcNode interface {
+func checkCalcFields(args []string, fields map[string]node) error {
-	eval(operands []string) float64
+	return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
-}
+		if n, ok := fields[name]; ok {
-
+			return []node{n}
-type calcNum float64 // a number literal
-type calcVar string  // a sibling-field name, before indexVars places it
-type calcIdx int     // an operand, by its position in the values expand read
-type calcNeg struct{ x calcNode }
-type calcBin struct { // a + - * / b
-	op   byte
-	l, r calcNode
-}
-
-func (n calcNum) eval([]string) float64 { return float64(n) }
-
-func (n calcIdx) eval(operands []string) float64 {
-	v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
-	if err != nil {
-		return math.NaN() // a non-numeric operand stays visible, never an error
 		}
-	return v
+		return nil
+	})
 }
 
-// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
+// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
-// expression it compiles, so only a calcIdx reaches a render. It panics rather
+func checkCalcNames(label string, t *template) error {
-// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
+	for _, o := range t.compiled.ops {
-// silently wrong number.
+		if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
-func (n calcVar) eval([]string) float64 {
+			continue
-	panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
+		}
+		if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
+			return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
+		}
+	}
+	return nil
 }
 
-func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
+// operandNodes lists every node each operand of o may render, once its template and names resolve.
-
+func operandNodes(o op) func(name string) []node {
-func (n calcBin) eval(operands []string) float64 {
+	return func(name string) []node {
-	l, r := n.l.eval(operands), n.r.eval(operands)
+		a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
-	switch n.op {
+		if len(a.leaves) == 0 {
-	case '+':
+			panic(invariant.Broken("calc operand %q was read before it was compiled", name))
-		return l + r
+		}
-	case '-':
+		return a.leaves
-		return l - r
-	case '*':
-		return l * r
-	default: // '/'
-		return l / r
 	}
 }
 
-// checkCalc validates a calc token at compile time: a parseable expression whose
+// checkOperands refuses an operand that is never a number and a division by a constant zero.
-// operands all name existing fields, and an optional non-negative integer dp. It
+// operand lists every node an operand may render, nil while that is unknown.
-// is a builtin check (fields first), so calc dispatches through the registry like
+func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
-// every other {name(args)} function.
+	for _, name := range c.Operands {
-func checkCalc(fields map[string]node, args []string) error {
+		if rendered, never := allNeverNumeric(operand(name)); never {
-	if len(args) < 1 || len(args) > 2 {
+			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
-		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return fmt.Errorf("calc(%q): %w", args[0], err)
-	}
-	for _, name := range calcVars(expr) {
-		operand, ok := fields[name]
-		if !ok {
-			return fmt.Errorf("calc(%q): no field %q", args[0], name)
-		}
-		if text, never := neverNumeric(operand); never {
-			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
 		}
 	}
-	if divisor, zero := constantZeroDivisor(expr, fields); zero {
+	if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
-		return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
+		return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
-	}
-	if len(args) == 2 {
-		dp, err := plainInt(args[1])
-		if err != nil {
-			return fmt.Errorf("calc decimals %w", err)
-		}
-		if dp < 0 || dp > maxDecimals {
-			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
-		}
 	}
 	return nil
 }
 
 // constantZeroDivisor finds a division whose right side is a constant zero: number
 // literals and fixed operands folded, anything that varies left unknown.
-func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
+func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
 	switch n := n.(type) {
-	case calcNeg:
+	case grammar.CalcNeg:
-		return constantZeroDivisor(n.x, fields)
+		return constantZeroDivisor(n.X, operand)
-	case calcBin:
+	case grammar.CalcBin:
-		if n.op == '/' {
+		if n.Operator == '/' {
-			if v, known := constantValue(n.r, fields); known && v == 0 {
+			if v, known := constantValue(n.R, operand); known && v == 0 {
-				return calcText(n.r), true
+				return grammar.CalcText(n.R), true
 			}
 		}
-		if d, zero := constantZeroDivisor(n.l, fields); zero {
+		if d, zero := constantZeroDivisor(n.L, operand); zero {
 			return d, true
 		}
-		return constantZeroDivisor(n.r, fields)
+		return constantZeroDivisor(n.R, operand)
 	}
 	return "", false
 }
 
 // constantValue evaluates an expression whose every operand is fixed.
-func constantValue(n calcNode, fields map[string]node) (float64, bool) {
+func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
 	switch n := n.(type) {
-	case calcNum:
+	case grammar.CalcNum:
 		return float64(n), true
-	case calcVar:
+	case grammar.CalcVar:
-		t, ok := fields[string(n)].(*template)
+		nodes := operand(n.Name)
-		if !ok || !t.fixed || t.repeat > 1 {
+		if len(nodes) != 1 {
 			return 0, false
 		}
-		v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
+		t, ok := nodes[0].(*template)
+		if !ok || t.repeat != 1 {
+			return 0, false
+		}
+		lit, fixed := t.fixedText()
+		if !fixed {
+			return 0, false
+		}
+		v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
 		return v, err == nil
-	case calcNeg:
+	case grammar.CalcNeg:
-		v, ok := constantValue(n.x, fields)
+		v, ok := constantValue(n.X, operand)
 		return -v, ok
-	case calcBin:
+	case grammar.CalcBin:
-		l, lok := constantValue(n.l, fields)
+		l, lok := constantValue(n.L, operand)
-		r, rok := constantValue(n.r, fields)
+		r, rok := constantValue(n.R, operand)
 		if !lok || !rok {
 			return 0, false
 		}
-		return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
+		return builtinfunc.Arith(n.Operator, l, r), true
 	}
 	return 0, false
 }
 
-// calcText spells an expression node the way an author would read it.
+// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
-func calcText(n calcNode) string {
+// text that does not parse, or a choice of only such items. text is one such render.
-	switch n := n.(type) {
-	case calcNum:
-		return strconv.FormatFloat(float64(n), 'f', -1, 64)
-	case calcVar:
-		return string(n)
-	case calcNeg:
-		return "-" + calcText(n.x)
-	case calcBin:
-		return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
-	}
-	return "?"
-}
-
-// neverNumeric reports a node no render of which is a number: a null, fixed text that
-// does not parse, or a choice of only such items. text is one such render.
 func neverNumeric(n node) (text string, never bool) {
 	switch n := n.(type) {
-	case *null:
+	case *nullItem:
 		return "", true
 	case *template:
-		if !n.fixed || n.repeat > 1 {
+		lit, fixed := n.fixedText()
+		if n.repeat == 0 {
+			return "", true
+		}
+		if !fixed || n.repeat != 1 {
 			return "", false
 		}
-		if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
+		if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
-			return n.lit, true
+			return lit, true
 		}
 	case *choice:
-		for _, it := range n.items {
+		return allNeverNumeric(n.items)
-			t, itemNever := neverNumeric(it)
+	}
-			if !itemNever {
+	return "", false
+}
+
+// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
+func allNeverNumeric(nodes []node) (text string, never bool) {
+	for _, n := range nodes {
+		t, nodeNever := neverNumeric(n)
+		if !nodeNever {
 			return "", false
 		}
 		text = t
 	}
-		return text, true
+	return text, len(nodes) > 0
-	}
-	return "", false
-}
-
-// calcPrep parses the expression and decimals once, at compile time, and places each
-// operand name at the position expand will read it into. checkCalc proved both args
-// valid, so no step here can fail; dp -1 prints the minimal form.
-func calcPrep(args []string) callFn {
-	expr, err := parseCalc(args[0])
-	if err != nil { // a nil AST would be a nil dereference per render, with no message
-		panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
-	}
-	at := make(map[string]int)
-	for i, name := range calcVars(expr) {
-		at[name] = i
-	}
-	placed := indexVars(expr, at)
-	dp := calcDecimals(args)
-	return func(_ *session, _ string, operands []string) string {
-		return formatFloat(placed.eval(operands), dp)
-	}
-}
-
-// calcDecimals is a calc's decimals count, or -1 for the shortest form.
-func calcDecimals(args []string) int {
-	if len(args) == 2 {
-		return atoi(args[1])
-	}
-	return -1
-}
-
-// indexVars replaces each operand name with its position in the values expand reads.
-// Both sides take that order from calcVars, so they cannot drift.
-func indexVars(n calcNode, at map[string]int) calcNode {
-	switch n := n.(type) {
-	case calcVar:
-		i, placed := at[string(n)]
-		if !placed { // calcVars named every operand, so a miss means the two disagree
-			panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
-		}
-		return calcIdx(i)
-	case calcNeg:
-		return calcNeg{indexVars(n.x, at)}
-	case calcBin:
-		return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
-	}
-	return n
-}
-
-// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
-// an expression that does not parse, so one that does not simply names nothing.
-func calcOperands(args []string) []string {
-	if len(args) == 0 {
-		return nil
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return nil
-	}
-	return calcVars(expr)
-}
-
-// calcVars lists the distinct field names an expression reads, in the order it first
-// names each. That order is the contract between expand, which reads the operands
-// into a slice, and indexVars, which places each name at its position in it.
-func calcVars(n calcNode) []string {
-	var out []string
-	seen := map[string]bool{}
-	var walk func(calcNode)
-	walk = func(n calcNode) {
-		switch n := n.(type) {
-		case calcVar:
-			if name := string(n); !seen[name] {
-				seen[name] = true
-				out = append(out, name)
-			}
-		case calcNeg:
-			walk(n.x)
-		case calcBin:
-			walk(n.l)
-			walk(n.r)
-		}
-	}
-	walk(n)
-	return out
-}
-
-// calcParser is a recursive-descent parser over the expression runes, threading
-// expr -> term -> factor for the standard * / before + - precedence.
-type calcParser struct {
-	rs  []rune
-	pos int
-}
-
-// parseCalc parses a whole expression, requiring it to consume all input.
-func parseCalc(expr string) (calcNode, error) {
-	p := &calcParser{rs: []rune(expr)}
-	if p.space(); p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("empty expression")
-	}
-	n, err := p.expr()
-	if err != nil {
-		return nil, err
-	}
-	if p.space(); p.pos != len(p.rs) {
-		return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
-	}
-	return n, nil
-}
-
-func (p *calcParser) space() {
-	for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
-		p.pos++
-	}
-}
-
-func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
-func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
-
-// binary parses a left-associative run of next() operands joined by the given
-// operators, the one shape expr and term share.
-func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
-	n, err := next()
-	if err != nil {
-		return nil, err
-	}
-	for {
-		p.space()
-		if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
-			return n, nil
-		}
-		op := byte(p.rs[p.pos])
-		p.pos++
-		r, err := next()
-		if err != nil {
-			return nil, err
-		}
-		n = calcBin{op, n, r}
-	}
-}
-
-// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
-func (p *calcParser) factor() (calcNode, error) {
-	p.space()
-	if p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("unexpected end of expression")
-	}
-	switch c := p.rs[p.pos]; {
-	case c == '-':
-		p.pos++
-		x, err := p.factor()
-		if err != nil {
-			return nil, err
-		}
-		return calcNeg{x}, nil
-	case c == '(':
-		p.pos++
-		n, err := p.expr()
-		if err != nil {
-			return nil, err
-		}
-		if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
-			return nil, fmt.Errorf("missing ')'")
-		}
-		p.pos++
-		return n, nil
-	case c == '.' || c >= '0' && c <= '9':
-		return p.number()
-	case c == '_' || unicode.IsLetter(c):
-		return p.ident()
-	default:
-		return nil, fmt.Errorf("unexpected %q", string(c))
-	}
-}
-
-func (p *calcParser) number() (calcNode, error) {
-	start, dot := p.pos, false
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c >= '0' && c <= '9' {
-			p.pos++
-		} else if c == '.' && !dot {
-			dot, p.pos = true, p.pos+1
-		} else {
-			break
-		}
-	}
-	v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
-	if err != nil {
-		return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
-	}
-	return calcNum(v), nil
-}
-
-// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
-// is always the minus operator, so a hyphenated field name can't be an operand.
-func (p *calcParser) ident() (calcNode, error) {
-	start := p.pos
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
-			p.pos++
-		} else {
-			break
-		}
-	}
-	return calcVar(string(p.rs[start:p.pos])), nil
-}
-
-func contains(bs []byte, b byte) bool {
-	for _, x := range bs {
-		if x == b {
-			return true
-		}
-	}
-	return false
 }

calc_test.go

@@ -3,8 +3,11 @@ package fejkdata
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
 )
 
 // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
 	if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
 		t.Fatalf("calc(a - b) = %q, want 7", got)
 	}
+	if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
+		t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
+	}
 }
 
 // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
 	}
 }
 
-// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
+// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
-// is not a calc, and for one whose expression does not parse: nothing, either way.
+// call that is not a calc, and for one whose expression does not parse: none, either way.
-// checkCalc is what reports a bad expression, so the callers that run after it
+func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
-// never meet one — but they must not have to depend on that order to be safe.
+	for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
-func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
+		if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
-	for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
+			t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
-		if got := tokenOperands(body); got != nil {
-			t.Errorf("tokenOperands(%q) = %v, want none", body, got)
 		}
 	}
-	if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
+	if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
-		t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
+		t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
-	}
-}
-
-// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
-// the expression before prep sees it, and indexVars places every name calcVars
-// read, so a break in either is reported rather than computed around.
-func TestCalcGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
-		"an operand name never placed":              func() { calcVar("n").eval(nil) },
-		"indexVars on a name it did not read":       func() { indexVars(calcVar("n"), map[string]int{}) },
-	} {
-		mustPanic(t, name, call)
 	}
 }
 
 // --- one draw, one value: a calc operand reads the expansion's draw ---
 
-// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
+// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
-// A field the format renders and a calc reads is drawn once per expansion, so the
+// shows: the format and the calc read one name.
-// operand shown is the operand computed — the correlation the dotted-path rule
+func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
-// already gives a level (see bound_test.go), applied to a plain sibling.
-func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(3))
 	for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	}
 }
 
-// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
+// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
-// expansion, not to the calc, so the bare token rendering the same name reads it too.
+func TestACalcOperandDrawsAfresh(t *testing.T) {
-func TestCalcOperandSharesOneDraw(t *testing.T) {
+	dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
-	dir := writeData(t, map[string]string{
-		"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
-	})
 	f := newGenerator(t, dir, WithSeed(5))
 	for i := 0; i < 200; i++ {
-		got := fake(t, f, "same")
+		if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
-		if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
-			t.Fatalf("same = %q, want one value twice", got)
-		}
-	}
-}
-
-// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
-// held, so an ordinary {w} {w} still draws twice.
-func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
-	dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
-	f := newGenerator(t, dir, WithSeed(5))
-	for i := 0; i < 200; i++ {
-		if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
 			return
 		}
 	}
-	t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
+	t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
 }
 
-// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
+// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
-// its own expansion, so it draws again while staying self-consistent.
+// iteration, staying one value within it.
-func TestCalcHoldIsPerExpansion(t *testing.T) {
+func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
+		"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(11))
 	varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
 	}
 }
 
-// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
+// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
-// {v} and its calc agree with each other, not with the outer pair.
+// pair agrees with itself, not with the outer pair.
-func TestCalcHoldIsPerTemplate(t *testing.T) {
+func TestTwoNamesPickApart(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
+		"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
-			"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
+			"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
 	})
 	f := newGenerator(t, dir, WithSeed(13))
 	differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["a","b"]}`:       `"x"`,
 		`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
 	} {
-		_, err := compile(parse(t, src))
+		_, err := resolved(t, src)
 		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
 			t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
 		}
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
 		`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
 	} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
 		}
 	}
@@ -257,7 +230,7 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 		`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`:   "divides by 0",
 		`{"format":"{calc(a/-0)}","a":"1"}`:              "divides by -0",
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
 		}
 	}
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	}
 	t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
 }
+
+func TestCalcCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{calc(1/0)}"`, // a constant zero divisor
+		`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
+		`"{calc()}"`,        // calc needs an expression
+		`"{calc(1 +)}"`,     // dangling operator
+		`"{calc((1 + 2)}"`,  // unbalanced parenthesis
+		`"{calc(1 2)}"`,     // two operands, no operator
+		`"{calc(price)}"`,   // operand names no field
+		`"{calc(1, 2, 3)}"`, // too many args
+		`"{calc(1, x)}"`,    // decimals arg not an integer
+		`"{calc(1, -1)}"`,   // decimals negative
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestCalcReadsAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3","5","7"]`,
+		"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	for i := 0; i < 50; i++ {
+		var k, double int
+		got := fake(t, f, "card")
+		if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
+			t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
+		}
+	}
+}
+
+func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3"]`,
+		"card": `"{/n as k}{calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	if got := fake(t, f, "card"); got != "4" && got != "6" {
+		t.Fatalf("card = %q, want 4 or 6", got)
+	}
+}
+
+func TestCalcRefusesABadOperand(t *testing.T) {
+	for card, want := range map[string]string{
+		`"{/word as w}{w}{calc(w * 2)}"`:    `operand "w" is never a number: it renders "def"`,
+		`"{/zero as z}{z}{calc(1 / z)}"`:    `divides by z, which is always zero`,
+		`"{calc(nope * 2)}"`:                `no field "nope"`,
+		`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
+	} {
+		dir := writeData(t, map[string]string{
+			"word": `["abc","def"]`,
+			"zero": `"0"`,
+			"card": card,
+		})
+		if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
+		}
+	}
+}
+
+func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
+		"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
+	})))
+	if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
+		t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
+	}
+}
+
+func TestATypedColumnProvesACalcOverAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":     `"{int(1,9)}"`,
+		"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	r, err := f.FakeRecord("order")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
+		t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
+	}
+	if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
+		t.Fatalf("JSON = %s, want double written as a number", r.JSON())
+	}
+	bad := writeData(t, map[string]string{
+		"word":  `["1","x"]`,
+		"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
+	})
+	if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
+		t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
+	}
+}
+
+func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
+	if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
+		t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
+	}
+}

cli/cli.go

@@ -0,0 +1,706 @@
+// Package cli is the fejkdata command: it renders the template on its stdin with fake
+// values from the modules it is given and any directories layered over them. A program
+// passing its own modules to Main is a fejkdata CLI carrying them, for example
+// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
+package cli
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata"
+)
+
+const usage = `Usage: <template> | COMMAND [flags]
+       COMMAND --list | --help | --version
+
+COMMAND reads one template from stdin and renders it:
+
+  echo 'id: {int(1,999)}' | COMMAND
+  echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
+  COMMAND < template.txt
+
+COMMAND <<'EOF'
+born {date(1990-01-01,2010-12-31,'2006-01-02')}
+EOF
+
+A template is a format string, or a JSON object, array, string or null. Its {…} tokens
+reach the data by reference from the root: {/path}, for any path --list prints, shipped
+or layered with --data-path. A reference names a category or a dotted path into one, a
+table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
+links to.
+
+Under --format text, the default, what you send is what renders, and COMMAND adds
+nothing: a format string keeps every byte, so echo's newline ends each render, and
+printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
+object, array or string drops the whitespace around it, and null alone renders nothing;
+42 or true alone is a format string, printed as written. An empty or blank stdin is a
+format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
+a backslash or a quote as written.
+
+With --format json, ndjson, csv or sql the template must be a record — a JSON
+template whose fields are its columns, or a lone reference such as {/path}:
+one reference and nothing else, with one newline after it allowed — and the rows are
+written as one JSON array, one JSON object per line, one CSV row (after a header), or
+one INSERT.
+
+  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
+      --format F         output form: text (default), json, ndjson, csv or sql
+  -h, --help             print this help, then exit
+      --list             list the paths the data offers, each written after {/ in a template, then exit;
+                         --repeat, --separator, --format and --table do nothing here
+      --no-shipped-data  load only the --data-path directories
+  -n, --repeat N         render the template N times, 0..1048576 (default 1); 0 prints nothing,
+                         [] under --format json, or the header under --format csv
+  -s, --seed N           same seed, version and data: identical output (default: a random seed)
+      --separator S      string between repeated renders (default empty); refused with a record
+                         --format, which has its own separators
+      --table T          the INSERT target for --format sql, ignored under any other
+                         (default: a lone reference's last segment, else records)
+      --version          print the version, then exit
+
+A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
+-- ends the flags.
+
+Exit codes: 0 success, 1 runtime error, 2 misuse.
+`
+
+type invocation struct {
+	args         []string
+	dirs         []string
+	format       string
+	help         bool
+	list         bool
+	noShipped    bool
+	repeat       int
+	seed         uint64
+	seeded       bool
+	separator    string
+	separatorSet bool
+	table        string
+	tableSet     bool
+	version      bool
+}
+
+type flagDef struct {
+	long  string
+	short string
+	value bool
+	set   func(*invocation, string) error
+}
+
+var flagDefs = []flagDef{
+	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
+	{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
+	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
+	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
+	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
+	{"repeat", "n", true, func(in *invocation, v string) error {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n > fejkdata.MaxRepeat {
+			return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
+		}
+		in.repeat = n
+		return nil
+	}},
+	{"seed", "s", true, func(in *invocation, v string) error {
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
+		}
+		in.seed, in.seeded = n, true
+		return nil
+	}},
+	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
+	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
+	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
+}
+
+func flagByLong(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].long == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+func flagByShort(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].short == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+// flagArg is one flag as written: its definition and the value attached to it,
+// if any.
+type flagArg struct {
+	def    *flagDef
+	value  string
+	inline bool
+}
+
+// splitFlags reads one argv element as flags: --name or --name=value, or -abc
+// where each letter is a short flag and the first that takes a value takes the
+// rest of the element.
+func splitFlags(arg string) ([]flagArg, error) {
+	if strings.HasPrefix(arg, "--") {
+		name, value, inline := strings.Cut(arg[2:], "=")
+		def := flagByLong(name)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag --%s", name)
+		}
+		return []flagArg{{def, value, inline}}, nil
+	}
+	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
+		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
+	}
+	var flags []flagArg
+	letters := arg[1:]
+	for i, r := range letters {
+		letter := string(r)
+		def := flagByShort(letter)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag -%s", letter)
+		}
+		if !def.value {
+			flags = append(flags, flagArg{def: def})
+			continue
+		}
+		rest := letters[i+len(letter):]
+		if strings.HasPrefix(rest, "=") {
+			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
+				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
+		}
+		return append(flags, flagArg{def, rest, rest != ""}), nil
+	}
+	return flags, nil
+}
+
+func parseArgs(argv []string) (invocation, error) {
+	in := invocation{repeat: 1, format: "text"}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		if arg == "--" {
+			in.args = append(in.args, argv[i+1:]...)
+			return in, nil
+		}
+		if len(arg) < 2 || arg[0] != '-' {
+			in.args = append(in.args, arg)
+			continue
+		}
+		flags, err := splitFlags(arg)
+		if err != nil {
+			return in, err
+		}
+		for _, fl := range flags {
+			if !fl.def.value {
+				if fl.inline {
+					return in, fmt.Errorf("--%s takes no value", fl.def.long)
+				}
+				_ = fl.def.set(&in, "")
+				continue
+			}
+			value := fl.value
+			if !fl.inline {
+				if i++; i >= len(argv) {
+					return in, fmt.Errorf("--%s needs a value", fl.def.long)
+				}
+				value = argv[i]
+			}
+			if err := fl.def.set(&in, value); err != nil {
+				return in, err
+			}
+		}
+	}
+	return in, nil
+}
+
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
+type recordFormat struct {
+	header func(*fejkdata.Record) string
+	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
+	writes string // what it writes between values, which --separator would seem to set
+}
+
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson writes one object per line.
+var recordFormats = map[string]recordFormat{
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
+	"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
+}
+
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
+func (in invocation) writesRecords() bool {
+	return in.format != "text"
+}
+
+// formatNames lists the --format values, text included, for the misuse error.
+func formatNames() string {
+	names := []string{"text"}
+	for name := range recordFormats {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
+}
+
+// checkFlags rejects a flag value or combination that cannot run.
+func (in invocation) checkFlags() error {
+	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
+		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
+	}
+	if in.tableSet && in.table == "" && in.format == "sql" {
+		return errors.New("--table names the INSERT target, so it cannot be empty")
+	}
+	if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
+		return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
+	}
+	return nil
+}
+
+func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
+
+// operandError refuses an argument that is no flag, naming the spelling that reads what
+// it most likely means: dropping it for -, a file for a file's name, else a template
+// piped in.
+func operandError(arg, name string) error {
+	unexpected := "unexpected argument '" + arg + "'"
+	if arg == "-" {
+		return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
+	}
+	if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
+		return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
+	}
+	ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
+	_, argLone := loneReference(arg)
+	inline, err := fejkdata.IsTemplate(arg)
+	_, refLone := loneReference(ref)
+	example := pipeHint(name)
+	switch {
+	case argLone || inline:
+		example = piped(arg, name)
+	case err == nil && refLone:
+		example = piped(ref, name)
+	}
+	return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
+}
+
+// check rejects a flag combination or an operand that cannot run, before any data is
+// read.
+func (in invocation) check(name string) error {
+	if err := in.checkFlags(); err != nil {
+		return err
+	}
+	if len(in.args) > 0 {
+		return operandError(in.args[0], name)
+	}
+	return nil
+}
+
+func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
+	if in.noShipped {
+		modules = nil
+	}
+	opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
+	for _, dir := range in.dirs {
+		opts = append(opts, fejkdata.WithDataPath(dir))
+	}
+	seed := in.seed
+	if !in.seeded {
+		seed = systemSeed
+	}
+	return append(opts, fejkdata.WithSeed(seed))
+}
+
+// write streams the input's renders to w, repeat of them joined by the separator; a write
+// failure surfaces from Flush, bufio keeping the first one.
+func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
+	if in.writesRecords() {
+		return in.writeRecords(f, src, w)
+	}
+	t, err := f.NewTemplate(src.template)
+	if err != nil {
+		return templateError(err)
+	}
+	out := bufio.NewWriter(w)
+	for i := 0; i < in.repeat; i++ {
+		if i > 0 {
+			out.WriteString(in.separator)
+		}
+		out.WriteString(t.Fake())
+	}
+	return out.Flush()
+}
+
+// recordStream builds the record drawer for the input, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
+	record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
+	if src.record == "" {
+		t, err := f.NewRecordTemplate(src.template)
+		if err != nil {
+			return nil, "", templateError(spacedReference(src.template, err))
+		}
+		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
+	}
+	table := in.table
+	if table == "" {
+		table = defaultTable(src.record)
+	}
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
+	record, table, err := in.recordStream(f, src)
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	if in.repeat == 0 {
+		return writeNoRecord(record, format, w)
+	}
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
+		r, err := record()
+		if err != nil {
+			return err
+		}
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
+		}
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
+}
+
+// writeNoRecord writes zero records: the header where the format writes one, an empty document
+// where it frames one, else nothing. It draws one record, which names the header's columns, so a
+// lone reference to nothing fails as it would with records.
+func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
+	r, err := record()
+	switch {
+	case err != nil:
+		return err
+	case format.header != nil:
+		_, err = io.WriteString(w, format.header(r)+"\n")
+	case format.open != "":
+		_, err = io.WriteString(w, format.open+format.close+"\n")
+	}
+	return err
+}
+
+// defaultTable names the INSERT target when --table is absent: the path's last
+// segment, or "records" when stdin is not a lone reference.
+func defaultTable(path string) string {
+	if path == "" {
+		return "records"
+	}
+	segments := strings.Split(withoutSelectors(path), ".")
+	return segments[len(segments)-1]
+}
+
+// withoutSelectors is path with its [selectors] cut out.
+func withoutSelectors(path string) string {
+	var names strings.Builder
+	for depth, i := 0, 0; i < len(path); i++ {
+		switch {
+		case path[i] == '[':
+			depth++
+		case path[i] == ']':
+			depth--
+		case depth == 0:
+			names.WriteByte(path[i])
+		}
+	}
+	return names.String()
+}
+
+// inputError marks a failure that is the input's own fault — nothing piped in, a template
+// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
+// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
+// error (exit 1).
+type inputError struct{ error }
+
+func (e inputError) Unwrap() error { return e.error }
+
+// jsonSpace is the whitespace JSON allows around a value.
+const jsonSpace = " \t\r\n"
+
+// input is what stdin holds: the template, and the path a lone reference names as a
+// record.
+type input struct {
+	template string
+	record   string
+}
+
+// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
+// reference once JSON's surrounding whitespace, or one final newline, is cut.
+func parseInput(raw string) input {
+	trimmed := strings.Trim(raw, jsonSpace)
+	text := raw
+	if json.Valid([]byte(trimmed)) {
+		text = trimmed
+	}
+	text, found := strings.CutSuffix(text, "\n")
+	if found {
+		text = strings.TrimSuffix(text, "\r")
+	}
+	record, _ := loneReference(text)
+	return input{template: raw, record: record}
+}
+
+// loneReference returns the path that text reads when it is one reference from the root
+// and nothing else, bare or as a JSON string.
+func loneReference(text string) (string, bool) {
+	var unquoted string
+	if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
+		text = unquoted
+	}
+	path, opens := strings.CutPrefix(text, "{/")
+	path, closes := strings.CutSuffix(path, "}")
+	if !opens || !closes || !isPath(path) {
+		return "", false
+	}
+	return path, true
+}
+
+// isPath reports whether a reference's body after its / is a path alone: no second /, no
+// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
+// It copies the grammar, since the CLI reads only the library's public API.
+func isPath(body string) bool {
+	if body == "" || strings.ContainsAny(body[:1], "/.") {
+		return false
+	}
+	names := withoutSelectors(body)
+	if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
+		return false
+	}
+	inline, err := fejkdata.IsTemplate(body)
+	return !inline && err == nil
+}
+
+// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
+// step up, and only a step up may end the path.
+func hasEmptySegment(path string) bool {
+	segments := strings.Split(path, ".")
+	stepUp := false
+	for i, seg := range segments[1:] {
+		last := i == len(segments)-2
+		switch {
+		case seg != "":
+			stepUp = false
+		case stepUp != last:
+			return true
+		default:
+			stepUp = !stepUp
+		}
+	}
+	return false
+}
+
+// spacedReference adds to a record refusal that whitespace around a lone reference made
+// the template text.
+func spacedReference(template string, err error) error {
+	if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
+		if path, lone := loneReference(trimmed); lone {
+			return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
+		}
+	}
+	return err
+}
+
+// piped is the command piping template into the command called name; some shells' echo
+// reads a backslash as an escape, so a template holding one goes through printf.
+func piped(template, name string) string {
+	if strings.ContainsRune(template, '\\') {
+		return "printf '%s' " + shellQuoted(template) + " | " + name
+	}
+	return "echo " + shellQuoted(template) + " | " + name
+}
+
+// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
+func shellQuoted(s string) string {
+	if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
+		return s
+	}
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// Main runs the command on the process's arguments and standard streams over modules, each
+// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
+// 2 misuse. --no-shipped-data leaves modules out.
+func Main(modules ...fs.FS) int {
+	var stdin io.Reader = os.Stdin
+	if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+		stdin = nil
+	}
+	return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
+}
+
+// commandName is the name the command runs under: the file arg0 names, less a Windows
+// .exe, or fejkdata where arg0 names none.
+func commandName(arg0 string) string {
+	name := filepath.Base(arg0)
+	if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
+		name = strings.TrimSuffix(name, ext)
+	}
+	if name == "." || name == "" {
+		return "fejkdata"
+	}
+	return name
+}
+
+// run is Main over the given streams, run as the command name, seeded by systemSeed where
+// no --seed is given. A nil stdin is a terminal, which holds no template.
+func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
+	in, err := parseArgs(args)
+	if err != nil {
+		return misuse(stderr, name, err)
+	}
+	if in.help {
+		io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
+		return 0
+	}
+	if in.version {
+		info, _ := debug.ReadBuildInfo()
+		fmt.Fprintln(stdout, versionLine(name, info))
+		return 0
+	}
+	if err := in.check(name); err != nil {
+		return misuse(stderr, name, err)
+	}
+	var src input
+	if !in.list {
+		if src, err = readStdin(stdin, name); err != nil {
+			return fail(stderr, name, err)
+		}
+	}
+	f, err := fejkdata.New(in.options(modules, systemSeed)...)
+	if err != nil {
+		return fail(stderr, name, err)
+	}
+	if in.list {
+		for _, p := range f.List() {
+			fmt.Fprintln(stdout, p)
+		}
+		return 0
+	}
+	if err := in.write(f, src, stdout); err != nil {
+		return fail(stderr, name, err)
+	}
+	return 0
+}
+
+func readStdin(stdin io.Reader, name string) (input, error) {
+	if stdin == nil {
+		return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
+	}
+	return parseInput(string(raw)), nil
+}
+
+// templateError is misuse unless the data the template reads failed to load.
+func templateError(err error) error {
+	if errors.Is(err, fejkdata.ErrLoad) {
+		return err
+	}
+	return inputError{err}
+}
+
+func fail(stderr io.Writer, name string, err error) int {
+	var te inputError
+	if errors.As(err, &te) {
+		return misuse(stderr, name, te.error)
+	}
+	report(stderr, name, err)
+	if errors.Is(err, fejkdata.ErrNoColumns) {
+		fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
+	}
+	return 1
+}
+
+func misuse(stderr io.Writer, name string, err error) int {
+	report(stderr, name, err)
+	fmt.Fprintf(stderr, "try '%s --help'\n", name)
+	return 2
+}
+
+// report writes err under the command name, in place of the library's own on each line.
+func report(stderr io.Writer, name string, err error) {
+	msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
+	fmt.Fprintf(stderr, "%s: %s\n", name, msg)
+}
+
+// fejkdataModule is the path of the module holding the library; the root package's path is
+// the module's.
+var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
+
+// versionLine names the fejkdata version the binary is built with. Where another module's
+// main package carries the CLI, the main module's version comes first, under the command name.
+func versionLine(name string, info *debug.BuildInfo) string {
+	if info == nil {
+		return "fejkdata devel"
+	}
+	if info.Main.Path == fejkdataModule {
+		return "fejkdata " + moduleVersion(info.Main)
+	}
+	for _, m := range info.Deps {
+		if m.Path == fejkdataModule {
+			return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
+		}
+	}
+	return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
+}
+
+// moduleVersion is the version m is built at: the replacement's where a replace directive
+// names one, and "devel" where it carries none, as a local directory does.
+func moduleVersion(m debug.Module) string {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Version == "" || m.Version == "(devel)" {
+		return "devel"
+	}
+	return m.Version
+}

cli/cli_test.go

@@ -0,0 +1,961 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// shipped holds the shipped modules as read from the checkout, each folder under ../data
+// holding a manifest, so these tests link no data package.
+var shipped = func() []fs.FS {
+	var modules []fs.FS
+	err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
+		if err == nil && d.Name() == ".fejkdata.json" {
+			modules = append(modules, os.DirFS(filepath.Dir(path)))
+		}
+		return err
+	})
+	if err != nil || len(modules) == 0 {
+		panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
+	}
+	return modules
+}()
+
+// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
+// layers its categories unprefixed with -d.
+var svSE, enUS, misc string
+
+func TestMain(m *testing.M) {
+	tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
+	if err != nil {
+		panic(err)
+	}
+	svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
+	code := m.Run()
+	os.RemoveAll(tmp)
+	os.Exit(code)
+}
+
+// bareCopy copies the category and rows files of the shipped module in data/<module> into a
+// folder under tmp, and returns that folder.
+func bareCopy(tmp, module string) string {
+	src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
+	if err := os.Mkdir(dst, 0o755); err != nil {
+		panic(err)
+	}
+	for _, pattern := range []string{"*.json", "*.tsv"} {
+		files, err := filepath.Glob(filepath.Join(src, pattern))
+		if err != nil {
+			panic(err)
+		}
+		for _, f := range files {
+			if strings.HasPrefix(filepath.Base(f), ".") {
+				continue
+			}
+			b, err := os.ReadFile(f)
+			if err == nil {
+				err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
+			}
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	return dst
+}
+
+func runOut(stdin string, args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+// runTerminal runs as from a shell with nothing piped in.
+func runTerminal(args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
+	module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
+		t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
+	}
+	out.Reset()
+	if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
+		t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
+	}
+}
+
+func TestRunOutputsValue(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if out == "" || strings.Contains(out, "\n") {
+		t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
+	}
+}
+
+func TestRunDotPath(t *testing.T) {
+	code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person run = %d", code)
+	}
+	code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(last) == "" {
+		t.Fatal("empty person.last output")
+	}
+	if last == full {
+		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
+}
+
+func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
+	var drawn, seeded, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
+		t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
+		t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if drawn.String() != seeded.String() {
+		t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
+	}
+}
+
+func TestRunSeedSpellings(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"--seed=42", "--data-path", svSE},
+		{"-s", "42", "-d", svSE},
+		{"--data-path", svSE, "--seed", "42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+}
+
+func TestRunShortFlagValues(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"-s42", "-d", svSE},
+		{"-s", "42", "-d" + svSE},
+		{"-d", svSE, "-s42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+	_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
+	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
+		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-s=42", "-d", svSE}, "-s42"},
+		{[]string{"-s=42", "-d", svSE}, "--seed=42"},
+		{[]string{"-d=" + svSE}, "--data-path="},
+		{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
+		{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
+	} {
+		code, out, errb := runOut("{/word}", c.args...)
+		if code != 2 || out != "" {
+			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+	code, out, _ := runOut("", "-hd", svSE)
+	if code != 0 || !strings.Contains(out, "Usage") {
+		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
+	}
+}
+
+func TestRunDoubleDashEndsFlags(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
+	if code != 2 || !strings.Contains(errb, "stdin") {
+		t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSingleDashLongIsRejected(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
+		{[]string{"-seed=42", "-d", svSE}, "use --seed"},
+		{[]string{"-data-path", svSE}, "use --data-path"},
+		{[]string{"-list", "-d", svSE}, "use --list"},
+		{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
+		{[]string{"-x", "-d", svSE}, "unknown flag -x"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunFlagValues(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
+		{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
+		{[]string{"--seed", "x", "-d", svSE}, "--seed"},
+		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
+		{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
+		{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunRepeat(t *testing.T) {
+	code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
+	}
+	_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
+	if short != out {
+		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
+	}
+}
+
+func TestRunSeparator(t *testing.T) {
+	code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if n := strings.Count(out, "\n"); n != 0 {
+		t.Errorf("want no newline, got %d: %q", n, out)
+	}
+	if !strings.Contains(out, ",") {
+		t.Errorf("values should be comma-joined: %q", out)
+	}
+}
+
+func TestRunRepeatAdvancesRNG(t *testing.T) {
+	code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	uniq := map[string]bool{}
+	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+		uniq[l] = true
+	}
+	if len(uniq) < 2 {
+		t.Errorf("repeat should vary output, all identical: %q", out)
+	}
+}
+
+func TestRunMisuse(t *testing.T) {
+	for _, c := range []struct {
+		stdin string
+		args  []string
+	}{
+		{"{/person}", []string{"-d", svSE, "word"}},
+		{"", []string{"-d", svSE, "--list", "person"}},
+		{"{/sv_SE.person}", []string{"--no-shipped-data"}},
+	} {
+		code, out, errb := runOut(c.stdin, c.args...)
+		if code != 2 {
+			t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
+		}
+		if !strings.Contains(errb, "try 'fejkdata --help'") {
+			t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
+		}
+		if out != "" {
+			t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
+		}
+	}
+}
+
+func TestRunTakesNoOperand(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "users.tmpl")
+	if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
+		{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
+		{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
+		{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
+		{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
+		{[]string{"-"}, "drop it"},
+		{[]string{file}, "fejkdata < " + file},
+	} {
+		code, out, errb := runOut("", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+type failingReader struct{}
+
+func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
+
+func TestRunUnreadableStdinIsMisuse(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
+		t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
+	}
+}
+
+func TestRunTerminalStdinIsMisuse(t *testing.T) {
+	code, out, errb := runTerminal("--seed", "1")
+	if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
+		t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
+	}
+	for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
+		if code, out, errb := runTerminal(args...); code != 0 || out == "" {
+			t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
+		}
+	}
+}
+
+func TestRunRendersExactlyWhatIsSent(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"hihi":                           "hihi",
+		"hihi\n":                         "hihi\n",
+		"hihi\r\n":                       "hihi\r\n",
+		"hihi\n\n":                       "hihi\n\n",
+		"{{x}}\n":                        "{x}\n",
+		"\"a\"\n":                        "a",
+		" \t\"a\"\r\n\n":                 "a",
+		"{/misc.territory[SE].alpha2}":   "SE",
+		"{/misc.territory[SE].alpha2}\n": "SE\n",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
+		}
+	}
+	if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
+		t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
+	}
+	if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
+		t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
+	}
+}
+
+func TestRunRendersBlankStdin(t *testing.T) {
+	for _, stdin := range []string{"", "\n", " \r\n\t"} {
+		if code, out, errb := runOut(stdin); code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunTextPrintsAsWritten(t *testing.T) {
+	for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
+		code, out, errb := runOut(stdin)
+		if code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunMultipleDataPaths(t *testing.T) {
+	code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(out) == "" {
+		t.Fatal("empty output for multi-dir run")
+	}
+}
+
+func TestRunUnknownCategoryFails(t *testing.T) {
+	code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if !strings.Contains(errb, "nope") {
+		t.Errorf("stderr %q should name the unknown category", errb)
+	}
+}
+
+func TestRunList(t *testing.T) {
+	code, out, errb := runOut("", "--data-path", svSE, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	for _, want := range []string{"person", "person.last", "address"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("list output missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRunHelpOnStdout(t *testing.T) {
+	for _, h := range []string{"-h", "--help"} {
+		code, out, errb := runOut("", h)
+		if code != 0 {
+			t.Errorf("%s = %d, want 0", h, code)
+		}
+		if !strings.Contains(out, "Usage") || errb != "" {
+			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
+		}
+	}
+}
+
+func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
+	fejkdata := func(version string, replace *debug.Module) *debug.Module {
+		return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
+	}
+	mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
+	for _, c := range []struct {
+		info *debug.BuildInfo
+		want string
+	}{
+		{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
+		{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
+		{nil, "fejkdata devel"},
+	} {
+		if got := versionLine("mycli", c.info); got != c.want {
+			t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
+		}
+	}
+}
+
+func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
+	for arg0, want := range map[string]string{
+		"/usr/local/bin/mycli": "mycli",
+		"mycli.EXE":            "mycli",
+		"mycli.exe":            "mycli",
+		"my.cli":               "my.cli",
+		"":                     "fejkdata",
+	} {
+		if got := commandName(arg0); got != want {
+			t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
+		}
+	}
+}
+
+func TestRunNamesItselfByTheCommandName(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
+		t.Errorf("mycli --nope = %d, %q", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
+		t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
+	}
+	if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
+		t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	code, out, errb := runOut("", "--version")
+	if code != 0 {
+		t.Fatalf("--version = %d, stderr=%q", code, errb)
+	}
+	version, ok := strings.CutPrefix(out, "fejkdata ")
+	if !ok || strings.TrimSpace(version) == "" {
+		t.Errorf("--version = %q, want the command name and a version on stdout", out)
+	}
+}
+
+func TestRunMissingDirFails(t *testing.T) {
+	code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if errb == "" {
+		t.Error("want an error message on stderr")
+	}
+}
+
+func TestRunShippedDataByDefault(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, list, _ := runOut("", "--list")
+	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
+		t.Errorf("--list without --data-path = %d, %q", code, list)
+	}
+	code, _, errb = runOut("{/person}")
+	if code != 2 || !strings.Contains(errb, "person") {
+		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSelectsATableRow(t *testing.T) {
+	dir := t.TempDir()
+	for name, content := range map[string]string{
+		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
+		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
+		t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
+	}
+	if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
+		t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
+	}
+	if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
+		t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
+	}
+	if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
+		t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
+	}
+	if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
+		t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
+	}
+}
+
+func TestUsageExamplesRenderWithNoData(t *testing.T) {
+	examples := regexp.MustCompile(`(?m)^  echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n  "+pipeHint("COMMAND")+"\n", -1)
+	if len(examples) < 3 {
+		t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
+	}
+	for _, e := range examples {
+		if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
+			t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
+		}
+	}
+	for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
+		if token != "{/path}" && token != "{/table[KEY]}" {
+			t.Errorf("usage names %s, a path a custom CLI may not carry", token)
+		}
+	}
+	if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
+		t.Errorf("usage does not list the exit codes")
+	}
+}
+
+func TestRunOperandMisuseBeforeLoad(t *testing.T) {
+	code, _, errb := runOut("", "--no-shipped-data", "x")
+	if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
+		t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
+	}
+}
+
+func TestRunInlineTemplate(t *testing.T) {
+	code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
+	if code != 0 || (out != "name: bosse" && out != "name: lina") {
+		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
+	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
+		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
+	if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
+		t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
+	}
+}
+
+func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
+	for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
+		if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
+			t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
+		}
+	}
+	if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
+		t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
+	}
+}
+
+func TestRunTemplateMisuse(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"{bad":                  "unterminated",
+		"name: {/no.such.path}": "no entry",
+		"a } b":                 "}}",
+		"{//sv_SE.person}":      "write {/sv_SE.person}",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
+			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
+		}
+		if strings.Contains(errb, "fejkdata: fejkdata:") {
+			t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
+		}
+	}
+}
+
+func TestRunNoShippedData(t *testing.T) {
+	code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
+		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
+	}
+	code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Errorf("run = %d, out=%q", code, out)
+	}
+	if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
+		t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
+	}
+	if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
+		t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
+	}
+}
+
+func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{nil, ""},
+		{[]string{"--format", "json"}, "[]\n"},
+		{[]string{"--format", "ndjson"}, ""},
+		{[]string{"--format", "csv"}, "first,last\n"},
+		{[]string{"--format", "sql"}, ""},
+	} {
+		if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
+			t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
+		}
+	}
+	if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
+		t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
+	}
+}
+
+func TestRunRepeatIsBounded(t *testing.T) {
+	for _, args := range [][]string{
+		{"--repeat", "9223372036854775807"},
+		{"--repeat", "1048577"},
+		{"-n", "99999999999"},
+	} {
+		code, out, errb := runOut("{/sv_SE.word}", args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
+			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
+		}
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
+	if code != 0 || len(out) != 1048576 {
+		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
+	}
+}
+
+func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		stdin         string
+		base, ignored []string
+	}{
+		{"", []string{"--list"}, []string{"-n", "3"}},
+		{"", []string{"--list"}, []string{"--separator", ","}},
+		{"", []string{"--list"}, []string{"--format", "json"}},
+		{"", []string{"--list"}, []string{"--table", "t"}},
+		{"{/users}", nil, []string{"--table", "t"}},
+		{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
+	} {
+		args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
+		_, want, _ := runOut(c.stdin, args...)
+		code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
+		if code != 0 || want == "" || got != want {
+			t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
+		}
+	}
+}
+
+func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
+	if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
+		t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
+	}
+}
+
+func TestRunRendersAnyJSONValue(t *testing.T) {
+	for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
+		if code, out, errb := runOut(stdin); code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
+		}
+	}
+}
+
+func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "-ä")
+	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
+		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
+	}
+}
+
+func TestRunEmptyDataPathIsNamed(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
+	if code != 1 || !strings.Contains(errb, "empty") {
+		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
+	}
+}
+
+func recordDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
+	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestRunRecordJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
+		}
+	}
+}
+
+func TestRunRecordCSVRoundTrips(t *testing.T) {
+	code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 4 {
+		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
+	}
+	if lines[0] != "first,last" {
+		t.Fatalf("csv header = %q, want first,last", lines[0])
+	}
+	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
+	if err != nil {
+		t.Fatalf("csv output is not valid CSV: %v", err)
+	}
+	if len(r) != 4 || len(r[0]) != 2 {
+		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
+	}
+}
+
+func TestRunRecordSQL(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
+		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
+	}
+}
+
+func TestRunRecordDefaultTable(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
+	}
+}
+
+func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
+	code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
+	}
+	code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
+	if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
+		t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
+	}
+}
+
+func TestRunRecordInlineTemplate(t *testing.T) {
+	code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
+	if code != 0 {
+		t.Fatalf("inline record = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
+	}
+}
+
+func TestRunRecordMisuse(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
+		{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
+		{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
+		{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
+	} {
+		code, out, errb := runOut("{/users}", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
+	if code != 1 {
+		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
+	}
+}
+
+func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
+	dir := t.TempDir()
+	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
+	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
+	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
+	}
+	code, _, errb = runOut(content, "--format", "json")
+	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
+	}
+	if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
+		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
+	}
+}
+
+func TestRunNumbersFollowTheShell(t *testing.T) {
+	_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
+	code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
+	if code != 0 || got != want {
+		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	}
+}
+
+// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
+// for columns: the record to write, and the quoting a shell needs to take it. The
+// template is well-formed and only the data cannot serve it, so the code is 1.
+func TestRunRecordOfAFieldlessPath(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
+	}
+	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
+		if !strings.Contains(errb, want) {
+			t.Errorf("stderr = %q, want it to name %s", errb, want)
+		}
+	}
+	if out != "" {
+		t.Errorf("stdout = %q, want nothing", out)
+	}
+}
+
+func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
+	dir := t.TempDir()
+	for name, data := range map[string]string{
+		".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
+		"bad.json":       `"{/nope}"`,
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
+		if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
+			t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
+		}
+	}
+}

cli/lone_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// grammarLoneReference is the path a format reads when the grammar parses it as one path
+// read of one reference from the root, and nothing else.
+func grammarLoneReference(text string) (string, bool) {
+	toks, err := grammar.ParseFormat(text)
+	if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
+		return "", false
+	}
+	sigil, path, err := grammar.RefShape(toks[0].Arms[0])
+	if err != nil || sigil != "/" {
+		return "", false
+	}
+	return path, true
+}
+
+func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
+	for _, text := range []string{
+		" {/x}",
+		"x",
+		"{.x}",
+		"{/.x}",
+		"{//x}",
+		"{/a...b}",
+		"{/a..b}",
+		"{/a..}",
+		"{/a.[x]}",
+		"{/a.}",
+		"{/a|/b}",
+		"{/a}{/b}",
+		"{/misc.currency[US Dollar (Next day)]}",
+		"{/sv_SE.person}",
+		"{/t[a as b].c}",
+		"{/t[f(x)]}",
+		"{/x as p}",
+		"{/x(1)}",
+		"{/x}}",
+		"{/}",
+	} {
+		path, lone := loneReference(text)
+		want, wantLone := grammarLoneReference(text)
+		if lone != wantLone || path != want {
+			t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
+		}
+	}
+}

cmd/fejkdata/main.go

@@ -1,512 +1,12 @@
-// Command fejkdata prints fake values from the shipped data and any directories
+// Command fejkdata renders the template on its stdin with fake values from every shipped
-// layered over it.
+// module; package cli documents it.
-//
-//	fejkdata sv_SE.person                        # a full person
-//	fejkdata sv_SE.person.last                   # just the surname
-//	fejkdata --data-path ./mydata sv_SE.person   # layer custom data; the last dir wins
-//	fejkdata --seed 42 sv_SE.address
 package main
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"runtime/debug"
-	"sort"
-	"strconv"
-	"strings"
 
-	"gitea.larvit.se/larvit/fejkdata"
+	"github.com/larvit/fejkdata/cli"
+	"github.com/larvit/fejkdata/data"
 )
 
-const usage = `Usage: fejkdata [flags] <path|template>
+func main() { os.Exit(cli.Main(data.Modules()...)) }
-
-  <path>                 a category, or a dotted path into one (person, person.last);
-                         a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'
-  <template>             a format string or JSON value to render inline, e.g.
-                         'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
-
-A layout inside a template is single-quoted, so quote the whole argument with " to
-keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
-
-An argument containing a { token, or a JSON object, array or string, is a
-template; any other argument is a path (a path never contains a brace or a quote,
-and a bracket only as a [selector] after a table's name). Templates reach the
-data by reference from the root — {/sv_SE.person.last} — whether the data is
-shipped or layered with --data-path. An argument carrying a closing brace or a
-quote but no valid JSON names neither.
-
-With --format json, ndjson, csv or sql the argument must name a record — a
-template whose fields are its columns — and the rows are written as one JSON
-array, one JSON object per line, one CSV row (after a header), or one INSERT.
-
-  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, ndjson, csv or sql
-  -h, --help             print this help, then exit
-      --list             list the paths the data offers, then exit
-      --no-shipped-data  load only the --data-path directories
-  -n, --repeat N         render the value N times, 1..1048576 (default 1)
-  -s, --seed N           seed for reproducible output
-      --separator S      string between repeated values (default newline)
-      --table T          the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
-      --version          print the version, then exit
-
-Flags may come before or after <path|template>; -- ends the flags. A short flag's
-value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
-`
-
-type invocation struct {
-	dirs         []string
-	format       string
-	formatSet    bool
-	help         bool
-	list         bool
-	noShipped    bool
-	paths        []string
-	repeat       int
-	repeatSet    bool
-	seed         uint64
-	seeded       bool
-	separator    string
-	separatorSet bool
-	table        string
-	tableSet     bool
-	version      bool
-}
-
-type flagDef struct {
-	long  string
-	short string
-	value bool
-	set   func(*invocation, string) error
-}
-
-var flagDefs = []flagDef{
-	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
-	{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
-	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
-	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
-	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
-	{"repeat", "n", true, func(in *invocation, v string) error {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > fejkdata.MaxRepeat {
-			return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
-		}
-		in.repeat, in.repeatSet = n, true
-		return nil
-	}},
-	{"seed", "s", true, func(in *invocation, v string) error {
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
-		}
-		in.seed, in.seeded = n, true
-		return nil
-	}},
-	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
-	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
-	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
-}
-
-func flagByLong(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].long == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-func flagByShort(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].short == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-// flagArg is one flag as written: its definition and the value attached to it,
-// if any.
-type flagArg struct {
-	def    *flagDef
-	value  string
-	inline bool
-}
-
-// splitFlags reads one argv element as flags: --name or --name=value, or -abc
-// where each letter is a short flag and the first that takes a value takes the
-// rest of the element.
-func splitFlags(arg string) ([]flagArg, error) {
-	if strings.HasPrefix(arg, "--") {
-		name, value, inline := strings.Cut(arg[2:], "=")
-		def := flagByLong(name)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag --%s", name)
-		}
-		return []flagArg{{def, value, inline}}, nil
-	}
-	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
-		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
-	}
-	var flags []flagArg
-	letters := arg[1:]
-	for i, r := range letters {
-		letter := string(r)
-		def := flagByShort(letter)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag -%s", letter)
-		}
-		if !def.value {
-			flags = append(flags, flagArg{def: def})
-			continue
-		}
-		rest := letters[i+len(letter):]
-		if strings.HasPrefix(rest, "=") {
-			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
-				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
-		}
-		return append(flags, flagArg{def, rest, rest != ""}), nil
-	}
-	return flags, nil
-}
-
-func parseArgs(argv []string) (invocation, error) {
-	in := invocation{repeat: 1, separator: "\n", format: "text"}
-	for i := 0; i < len(argv); i++ {
-		arg := argv[i]
-		if arg == "--" {
-			in.paths = append(in.paths, argv[i+1:]...)
-			return in, nil
-		}
-		if len(arg) < 2 || arg[0] != '-' {
-			in.paths = append(in.paths, arg)
-			continue
-		}
-		flags, err := splitFlags(arg)
-		if err != nil {
-			return in, err
-		}
-		for _, fl := range flags {
-			if !fl.def.value {
-				if fl.inline {
-					return in, fmt.Errorf("--%s takes no value", fl.def.long)
-				}
-				_ = fl.def.set(&in, "")
-				continue
-			}
-			value := fl.value
-			if !fl.inline {
-				if i++; i >= len(argv) {
-					return in, fmt.Errorf("--%s needs a value", fl.def.long)
-				}
-				value = argv[i]
-			}
-			if err := fl.def.set(&in, value); err != nil {
-				return in, err
-			}
-		}
-	}
-	return in, nil
-}
-
-// recordFormat is one way to write a record out: the line each record renders,
-// the header that precedes the first one, and the open/close frame plus the
-// between-record separator a document form needs.
-type recordFormat struct {
-	header func(*fejkdata.Record) string
-	line   func(r *fejkdata.Record, table string) string
-	open   string
-	close  string
-	sep    string
-}
-
-// recordFormats is every --format that writes records. json frames the records
-// as one array document; ndjson is the same column, one object per line.
-var recordFormats = map[string]recordFormat{
-	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
-	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
-	"ndjson": {line: jsonLine, sep: "\n"},
-	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
-}
-
-func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
-
-// writesRecords reports whether the format writes records rather than plain text.
-func (in invocation) writesRecords() bool {
-	return in.format != "text"
-}
-
-// formatNames lists the --format values, text included, for the misuse error.
-func formatNames() string {
-	names := []string{"text"}
-	for name := range recordFormats {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
-}
-
-// checkFlags rejects a flag value or combination that cannot run.
-func (in invocation) checkFlags() error {
-	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
-		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
-	}
-	if in.tableSet && in.table == "" {
-		return errors.New("--table names the INSERT target, so it cannot be empty")
-	}
-	if in.tableSet && in.format != "sql" {
-		return errors.New("--table names the INSERT target, so it needs --format sql")
-	}
-	if in.writesRecords() && in.separatorSet {
-		return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
-	}
-	return nil
-}
-
-// check rejects a flag combination or an argument that cannot run, and reports
-// what the argument names, so its shape is settled before any data is read.
-func (in invocation) check() (argKind, error) {
-	if err := in.checkFlags(); err != nil {
-		return argPath, err
-	}
-	if in.list && len(in.paths) > 0 {
-		return argPath, errors.New("--list takes no path")
-	}
-	if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
-		return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
-	}
-	if in.list {
-		return argPath, nil
-	}
-	if len(in.paths) != 1 {
-		return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
-	}
-	return classify(in.paths[0])
-}
-
-func (in invocation) options() []fejkdata.Option {
-	var opts []fejkdata.Option
-	if in.noShipped {
-		opts = append(opts, fejkdata.WithoutShippedData())
-	}
-	for _, dir := range in.dirs {
-		opts = append(opts, fejkdata.WithDataPath(dir))
-	}
-	if in.seeded {
-		opts = append(opts, fejkdata.WithSeed(in.seed))
-	}
-	return opts
-}
-
-// write streams the argument's renders to w, repeat of them joined by the
-// separator and ended by a newline. A value that renders once renders every time,
-// so a render failure comes before anything is written; a write failure surfaces
-// from Flush, bufio keeping the first one.
-func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	if in.writesRecords() {
-		return in.writeRecords(f, kind, w)
-	}
-	draw, err := in.textDraw(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	out := bufio.NewWriter(w)
-	for i := 0; i < in.repeat; i++ {
-		v, err := draw()
-		if err != nil {
-			return err
-		}
-		if i > 0 {
-			out.WriteString(in.separator)
-		}
-		out.WriteString(v)
-	}
-	out.WriteString("\n")
-	return out.Flush()
-}
-
-// textDraw builds what one text render yields: the value, from a path or an
-// inline template.
-func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if kind != argTemplate {
-		return func() (string, error) { return f.Fake(arg) }, nil
-	}
-	t, err := f.NewTemplate(arg)
-	if err != nil {
-		return nil, templateError{err}
-	}
-	return func() (string, error) { return t.Fake(), nil }, nil
-}
-
-// recordStream builds the record drawer for the argument, plus the INSERT table
-// a sql format names.
-func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
-	record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
-	if kind == argTemplate {
-		t, err := f.NewRecordTemplate(arg)
-		if err != nil {
-			return nil, "", templateError{err}
-		}
-		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
-	}
-	table := in.table
-	if table == "" {
-		table = defaultTable(arg, kind)
-	}
-	return record, table, nil
-}
-
-// writeRecords streams a record per line in the chosen format, framing a document
-// form with its open/close brackets and a header preceding the first record.
-func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	record, table, err := in.recordStream(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	format := recordFormats[in.format]
-	out := bufio.NewWriter(w)
-	if format.open != "" {
-		out.WriteString(format.open)
-		out.WriteByte('\n')
-	}
-	for i := 0; i < in.repeat; i++ {
-		r, err := record()
-		if err != nil {
-			return err
-		}
-		if i == 0 && format.header != nil {
-			out.WriteString(format.header(r))
-			out.WriteByte('\n')
-		}
-		if i > 0 {
-			out.WriteString(format.sep)
-		}
-		out.WriteString(format.line(r, table))
-	}
-	if format.close != "" {
-		out.WriteByte('\n')
-		out.WriteString(format.close)
-	}
-	out.WriteByte('\n')
-	return out.Flush()
-}
-
-// defaultTable names the INSERT target when --table is absent: the path's last
-// segment, or "records" for an inline template that sits in no folder.
-func defaultTable(arg string, kind argKind) string {
-	if kind == argTemplate {
-		return "records"
-	}
-	var names strings.Builder // the path with its [selectors] cut out
-	for depth, i := 0, 0; i < len(arg); i++ {
-		switch {
-		case arg[i] == '[':
-			depth++
-		case arg[i] == ']':
-			depth--
-		case depth == 0:
-			names.WriteByte(arg[i])
-		}
-	}
-	segments := strings.Split(names.String(), ".")
-	return segments[len(segments)-1]
-}
-
-// templateError marks a render failure that is the argument's own fault — an
-// inline template that does not compile. run reports it as misuse (exit 2, with a
-// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
-type templateError struct{ error }
-
-func (e templateError) Unwrap() error { return e.error }
-
-type argKind int
-
-const (
-	argPath argKind = iota
-	argTemplate
-)
-
-// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
-func classify(arg string) (argKind, error) {
-	inline, err := fejkdata.IsTemplate(arg)
-	if inline {
-		return argTemplate, nil
-	}
-	return argPath, err
-}
-
-func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
-
-// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
-func run(args []string, stdout, stderr io.Writer) int {
-	in, err := parseArgs(args)
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	if in.help {
-		fmt.Fprint(stdout, usage)
-		return 0
-	}
-	if in.version {
-		fmt.Fprintln(stdout, "fejkdata "+buildVersion())
-		return 0
-	}
-	kind, err := in.check()
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	f, err := fejkdata.New(in.options()...)
-	if errors.Is(err, fejkdata.ErrNoData) {
-		return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
-	}
-	if err != nil {
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	if in.list {
-		for _, p := range f.List() {
-			fmt.Fprintln(stdout, p)
-		}
-		return 0
-	}
-	if err := in.write(f, kind, stdout); err != nil {
-		var te templateError
-		if errors.As(err, &te) {
-			return misuse(stderr, te.error)
-		}
-		fmt.Fprintln(stderr, err)
-		if errors.Is(err, fejkdata.ErrNoColumns) {
-			fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
-		}
-		return 1
-	}
-	return 0
-}
-
-// templateSplit names the cure for a template a shell split on its spaces, which is
-// what several arguments carrying a token mean.
-func templateSplit(paths []string) string {
-	for _, p := range paths {
-		if strings.Contains(p, "{") {
-			return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
-		}
-	}
-	return ""
-}
-
-func misuse(stderr io.Writer, err error) int {
-	// A library error already names the program, so the prefix is not doubled.
-	fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
-	return 2
-}
-
-// buildVersion is the module version go install stamps into the binary, or "devel".
-func buildVersion() string {
-	if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
-		return info.Main.Version
-	}
-	return "devel"
-}

cmd/fejkdata/main_test.go

@@ -2,640 +2,46 @@ package main
 
 import (
 	"bytes"
-	"encoding/csv"
-	"encoding/json"
 	"os"
-	"path/filepath"
+	"os/exec"
-	"regexp"
 	"strings"
 	"testing"
 )
 
-const (
+func TestMain(m *testing.M) {
-	svSE = "../../data/sv_SE"
+	if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
-	enUS = "../../data/en_US"
+		main()
-	misc = "../../data/misc"
+	}
-)
+	os.Exit(m.Run())
-
-func runOut(args ...string) (int, string, string) {
-	var out, errb bytes.Buffer
-	code := run(args, &out, &errb)
-	return code, out.String(), errb.String()
 }
 
-func TestRunOutputsValue(t *testing.T) {
+func TestMainRendersTheShippedData(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "person")
+	cmd := exec.Command(os.Args[0], "--seed", "1")
-	if code != 0 {
+	cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-		t.Fatalf("run = %d, stderr=%q", code, errb)
+	cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
-	}
+	var stderr bytes.Buffer
-	if strings.TrimSpace(out) == "" {
+	cmd.Stderr = &stderr
-		t.Fatalf("empty output, stderr=%q", errb)
+	out, err := cmd.Output()
-	}
+	if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
-	if !strings.HasSuffix(out, "\n") {
+		t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
-		t.Errorf("output should end with newline, got %q", out)
 	}
 }
 
-func TestRunDotPath(t *testing.T) {
+func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
-	code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
+	var outs [2]string
-	if code != 0 {
+	for i := range outs {
-		t.Fatalf("person run = %d", code)
+		cmd := exec.Command(os.Args[0])
+		cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
+		cmd.Stdin = strings.NewReader("{uuid()}")
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+		out, err := cmd.Output()
+		if err != nil || len(out) == 0 {
+			t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
 		}
-	code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
+		outs[i] = string(out)
-	if code != 0 {
-		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
 	}
-	if strings.TrimSpace(last) == "" {
+	if outs[0] == outs[1] {
-		t.Fatal("empty person.last output")
+		t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
-	}
-	if last == full {
-		t.Errorf("person.last %q should differ from person %q", last, full)
-	}
-}
-
-func TestRunSeedSpellings(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"--seed=42", "--data-path", svSE, "address"},
-		{"-s", "42", "-d", svSE, "address"},
-		{"--data-path", svSE, "address", "--seed", "42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-}
-
-func TestRunShortFlagValues(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"-s42", "-d", svSE, "address"},
-		{"-s", "42", "-d" + svSE, "address"},
-		{"-d", svSE, "address", "-s42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-	_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
-	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
-		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
-	}
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
-		{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
-		{[]string{"-d=" + svSE, "address"}, "--data-path="},
-		{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
-		{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" {
-			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-	code, out, _ := runOut("-hd", svSE)
-	if code != 0 || !strings.Contains(out, "Usage") {
-		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
-	}
-}
-
-func TestRunDoubleDashEndsFlags(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--", "person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, _, errb = runOut("--data-path", svSE, "--", "--list")
-	if code != 1 || !strings.Contains(errb, "--list") {
-		t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestRunSingleDashLongIsRejected(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-data-path", svSE, "person"}, "use --data-path"},
-		{[]string{"-list", "-d", svSE}, "use --list"},
-		{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
-		{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunFlagValues(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
-		{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
-		{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
-		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
-		{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
-		{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunRepeat(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 3 {
-		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
-	}
-	_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
-	if short != out {
-		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
-	}
-}
-
-func TestRunSeparator(t *testing.T) {
-	code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if n := strings.Count(out, "\n"); n != 1 {
-		t.Errorf("want one trailing newline, got %d: %q", n, out)
-	}
-	if !strings.Contains(out, ",") {
-		t.Errorf("values should be comma-joined: %q", out)
-	}
-}
-
-func TestRunRepeatAdvancesRNG(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	uniq := map[string]bool{}
-	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
-		uniq[l] = true
-	}
-	if len(uniq) < 2 {
-		t.Errorf("repeat should vary output, all identical: %q", out)
-	}
-}
-
-func TestRunMisuse(t *testing.T) {
-	for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
-		code, out, errb := runOut(args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", args, code)
-		}
-		if !strings.Contains(errb, "try 'fejkdata --help'") {
-			t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
-		}
-		if out != "" {
-			t.Errorf("run(%v) stdout = %q, want nothing", args, out)
-		}
-	}
-	// A template a shell split names both quotings, since which one cures it turns
-	// on whether the argument carries a double-quoted JSON.
-	_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
-	for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
-		}
-	}
-}
-
-func TestRunMultipleDataPaths(t *testing.T) {
-	code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatal("empty output for multi-dir run")
-	}
-}
-
-func TestRunUnknownCategoryFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", svSE, "nope")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if !strings.Contains(errb, "nope") {
-		t.Errorf("stderr %q should name the unknown category", errb)
-	}
-}
-
-func TestRunList(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	for _, want := range []string{"person", "person.last", "address"} {
-		if !strings.Contains(out, want) {
-			t.Errorf("list output missing %q:\n%s", want, out)
-		}
-	}
-}
-
-func TestRunHelpOnStdout(t *testing.T) {
-	for _, h := range []string{"-h", "--help"} {
-		code, out, errb := runOut(h)
-		if code != 0 {
-			t.Errorf("%s = %d, want 0", h, code)
-		}
-		if !strings.Contains(out, "Usage") || errb != "" {
-			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
-		}
-	}
-}
-
-func TestRunVersion(t *testing.T) {
-	code, out, errb := runOut("--version")
-	if code != 0 {
-		t.Fatalf("--version = %d, stderr=%q", code, errb)
-	}
-	version, ok := strings.CutPrefix(out, "fejkdata ")
-	if !ok || strings.TrimSpace(version) == "" {
-		t.Errorf("--version = %q, want the command name and a version on stdout", out)
-	}
-}
-
-func TestRunMissingDirFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", "../../data/nope", "person")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if errb == "" {
-		t.Error("want an error message on stderr")
-	}
-}
-
-func TestRunShippedDataByDefault(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "sv_SE.person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, list, _ := runOut("--list")
-	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
-		t.Errorf("--list without --data-path = %d, %q", code, list)
-	}
-	code, _, errb = runOut("person")
-	if code != 1 || !strings.Contains(errb, "person") {
-		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestClassify(t *testing.T) {
-	for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
-		if got, err := classify(arg); err != nil || got != want {
-			t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
-		}
-	}
-	if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
-		t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
-	}
-	if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
-		t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
-	}
-}
-
-func TestRunSelectsATableRow(t *testing.T) {
-	dir := t.TempDir()
-	for name, content := range map[string]string{
-		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
-		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
-	} {
-		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
-		t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
-		t.Fatalf("region[Skåne län].code = %d, %q", code, out)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
-		t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
-		t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
-		t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
-		t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out)
-	}
-}
-
-func TestUsageReferencesResolve(t *testing.T) {
-	for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
-		path := token[2 : len(token)-1]
-		code, out, errb := runOut("--seed", "1", path)
-		if code != 0 || strings.TrimSpace(out) == "" {
-			t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
-		}
-	}
-}
-
-func TestRunShapeMisuseBeforeLoad(t *testing.T) {
-	code, _, errb := runOut("--no-shipped-data", "[abc]")
-	if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
-		t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
-	}
-}
-
-func TestRunInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
-	if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
-		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
-	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
-		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
-	}
-}
-
-func TestRunTemplateMisuse(t *testing.T) {
-	for arg, want := range map[string]string{
-		"{bad":                  "unterminated",
-		"[red,green]":           `starts with "["`,
-		`{"format":"x"}`:        "is a string",
-		"name: {/no.such.path}": "no entry",
-		"{/sv_SE.person}":       "write sv_SE.person",
-		"x[1]y":                 `"]"`,
-		` ["a","b"] `:           "may not be padded",
-	} {
-		code, out, errb := runOut("--seed", "1", arg)
-		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
-			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
-		}
-		if strings.Contains(errb, "fejkdata: fejkdata:") {
-			t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
-		}
-	}
-}
-
-func TestRunNoShippedData(t *testing.T) {
-	code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
-		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
-	}
-	code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Errorf("run = %d, out=%q", code, out)
-	}
-	code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
-	if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
-		t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
-	}
-}
-
-func TestRunRepeatIsBounded(t *testing.T) {
-	for _, args := range [][]string{
-		{"--repeat", "9223372036854775807", "sv_SE.word"},
-		{"--repeat", "1048577", "sv_SE.word"},
-		{"-n", "99999999999", "sv_SE.word"},
-	} {
-		code, out, errb := runOut(args...)
-		if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
-			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
-		}
-	}
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
-	if code != 0 || len(out) != 1048576+1 {
-		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
-	}
-}
-
-func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
-	for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
-		code, _, errb := runOut(args...)
-		if code != 2 || !strings.Contains(errb, "--list takes no") {
-			t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
-		}
-	}
-}
-
-func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
-	code, _, errb := runOut("-ä", "sv_SE.word")
-	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
-		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
-	}
-}
-
-func TestRunEmptyDataPathIsNamed(t *testing.T) {
-	code, _, errb := runOut("--data-path=", "sv_SE.word")
-	if code != 1 || !strings.Contains(errb, "empty") {
-		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
-	}
-}
-
-func recordDir(t *testing.T) string {
-	t.Helper()
-	dir := t.TempDir()
-	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
-	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
-func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil {
-		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
-	}
-	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
-		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
-	}
-}
-
-func TestRunRecordNDJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
-	}
-	for _, line := range lines {
-		var m map[string]string
-		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
-			t.Fatalf("ndjson line %q is not one object: %v", line, err)
-		}
-	}
-}
-
-func TestRunRecordCSVRoundTrips(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 4 {
-		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
-	}
-	if lines[0] != "first,last" {
-		t.Fatalf("csv header = %q, want first,last", lines[0])
-	}
-	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
-	if err != nil {
-		t.Fatalf("csv output is not valid CSV: %v", err)
-	}
-	if len(r) != 4 || len(r[0]) != 2 {
-		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
-	}
-}
-
-func TestRunRecordSQL(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
-		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
-	}
-}
-
-func TestRunRecordDefaultTable(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
-	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
-		t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
-	}
-}
-
-func TestRunRecordInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
-	if code != 0 {
-		t.Fatalf("inline record = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
-		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
-	}
-}
-
-func TestRunRecordMisuse(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
-		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
-		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
-		}
-	}
-}
-
-func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
-	if code != 1 {
-		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
-	}
-}
-
-func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
-	dir := t.TempDir()
-	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
-	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
-	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
-	}
-	code, _, errb = runOut("--format", "json", content)
-	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
-		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
-	}
-}
-
-func TestRunNumbersFollowTheShell(t *testing.T) {
-	_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
-	code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
-	if code != 0 || got != want {
-		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
-	}
-}
-
-// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
-// for columns: the record to write, and the quoting a shell needs to take it. The
-// argument is well-formed and only the data cannot serve it, so the code is 1.
-func TestRunRecordOfAFieldlessPath(t *testing.T) {
-	code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
-	if code != 1 {
-		t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
-	}
-	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("stderr = %q, want it to name %s", errb, want)
-		}
-	}
-	if out != "" {
-		t.Errorf("stdout = %q, want nothing", out)
 	}
 }

compose.yaml

@@ -1,21 +1,5 @@
-# Docker-based dev workflow — no local tooling beyond Docker is required.
-#
-#   docker compose run --rm test
-#   docker compose run --rm vet
-#   docker compose run --rm build
-#
-# The full gate — vet, formatting and tests — is `docker build .`, which is what
-# CI runs.
-#
-# Build caches persist in the `gocache` volume, so re-runs are fast and the
-# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
-# ownership when run with `--user "$(id -u):$(id -g)"`.
-#
-# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
-# supported Go with: GO_VERSION=1.22 docker compose run --rm test
-
 x-go: &go
-  image: golang:${GO_VERSION:-1.26.4}
+  image: golang:${GO_VERSION:-1.27.2}
   working_dir: /app
   volumes:
     - .:/app
@@ -33,12 +17,16 @@ services:
 
   cover:
     <<: *go
-    command: go test -cover ./...
+    command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
 
   bench:
     <<: *go
     command: go test -run XXX -bench . -benchmem ./...
 
+  generate:
+    <<: *go
+    command: go generate ./...
+
   vet:
     <<: *go
     command: go vet ./...
@@ -47,6 +35,10 @@ services:
     <<: *go
     command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
 
+  vulncheck:
+    <<: *go
+    command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
+
   build:
     <<: *go
     command: go build ./...
@@ -59,13 +51,12 @@ services:
     <<: *go
     command: go mod tidy
 
-  # Interactive shell for ad-hoc work: docker compose run --rm dev
   dev:
     <<: *go
     command: sh
 
   data-import:
-    image: python:3.14.7-slim
+    image: python:3.14.8-slim
     working_dir: /app
     volumes:
       - .:/app

data-import/geo-se.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
 
-    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
+    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import csv
 import io
 import json
-import math
 import os
 import re
 import sys
@@ -16,6 +14,7 @@ import urllib.request
 import zipfile
 from pathlib import Path
 
+import geo
 import source
 import tsv
 import xlsx
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
 POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
 NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
 NVDB_PAGE = 50000
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
-CACHE = Path(__file__).resolve().parent / "cache"
 TIMEZONE = "Europe/Stockholm"
+# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
 ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
+# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
 UNMATCHED_POPULATION = 200
+# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
+# or a description in one.
+NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
+PARTICLES = {"af", "de", "von"}
 
 
 def scb_codes(cache):
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
         yield name, float(lat), float(lon)
 
 
-class Nearest:
-    """Nearest point by an equirectangular distance, over a degree grid."""
-
-    def __init__(self, points, cell=0.05):
-        self.cell = cell
-        self.grid = collections.defaultdict(list)
-        for lat, lon, value in points:
-            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
-
-    def find(self, lat, lon):
-        ci, cj = int(lat // self.cell), int(lon // self.cell)
-        best, best_d = None, math.inf
-        ring = 0
-        while ring < 400:
-            for i in range(ci - ring, ci + ring + 1):
-                for j in range(cj - ring, cj + ring + 1):
-                    if max(abs(i - ci), abs(j - cj)) != ring:
-                        continue
-                    for plat, plon, value in self.grid.get((i, j), ()):
-                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
-                        if d < best_d:
-                            best, best_d = value, d
-            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
-                return best
-            ring += 1
-        return best
-
-
 def street_delivery(name, codes):
-    """The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
+    """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
+    least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
+    for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
+    docs/research/research-geo-se.md, Postal codes → Structure."""
     if name in ONE_POSITION:
         return [c for c in codes if c[1] != "0"]
     largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
 
 
 def well_cased(name):
+    """Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
+    capitals or lower case."""
     return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
 
 
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
         municipality, method = municipality_of(name, rows, tatorter, municipalities)
         how[method] += 1
         kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
-        with_point = [r for r in rows if r["lat"] is not None]
+        point = geo.centroid(rows)
-        if municipality is None or not kept or not with_point or not well_cased(name):
+        if municipality is None or not kept or point is None or not well_cased(name):
             continue
-        lat = sum(r["lat"] for r in with_point) / len(with_point)
+        lat, lon = point
-        lon = sum(r["lon"] for r in with_point) / len(with_point)
         out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
-    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr)
+    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
     return out
 
 
-def streets(segments, codes, localities, per_locality):
+def street_name(name):
+    # A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
+    opening = name[0].isupper() or name.split()[0] in PARTICLES
+    return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
+
+
+def streets(segments, codes, localities, share, minimum):
     """The names with most segments per locality, each segment at its nearest code centroid."""
-    nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
+    nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
     count = collections.Counter()
     for name, lat, lon in segments:
-        if name[0].isalpha():
+        if street_name(name):
             count[(nearest.find(lat, lon), name)] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "segments")
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
     key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
     if not key:
         sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
-    cache, out = Path(a.cache), Path(a.out)
+    cache, out = geo.directories(a)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
-    regions, municipalities = scb_codes(cache)
+    region_names, municipality_names = scb_codes(cache)
     population = scb_population(cache)
     codes = geonames(cache)
-    places = localities(codes, scb_tatorter(cache), municipalities, population)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
-    named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
+    named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
-    places = {name: l for name, l in places.items() if name in named}
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
-    if empty:
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
-        sys.exit(f"municipalities without a locality: {empty}")
+    geo.log_dropped_localities(dropped["locality"])
+    if dropped["municipality"] or dropped["region"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo-us.py

@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
-"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
+"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
 
-    data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
+    data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import concurrent.futures
 import csv
 import io
+import math
+import random
 import re
 import struct
 import sys
+import urllib.parse
 import zipfile
-from pathlib import Path
 
+import geo
 import source
 import tsv
 
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
 PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
 ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
 TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
-CACHE = Path(__file__).resolve().parent / "cache"
 ESTIMATE = "POPESTIMATE2025"
 CDP = "57"
+# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
+CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
+SELECT ?iso ?capitalLabel ?gnis WHERE {
+  ?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
+  OPTIONAL { ?capital wdt:P590 ?gnis. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}"""})
+SHIPPED_POPULATION = 100000
 HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
 SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
-# Places whose Census name is a merged government's; the postal city is what an address carries.
+# Places whose Census name is not the postal city an address carries.
-NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
+NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
 TIMEZONES = {
     "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
     "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -107,8 +115,8 @@ def place_name(geoid, name):
     return stripped
 
 
-def localities(cache, min_population, counties):
+def localities(cache, counties):
-    """Each shipped place with its county, population and centroid."""
+    """Each incorporated place with its county, population and centroid."""
     place_population, county_part = {}, collections.defaultdict(list)
     for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
         if r["SUMLEV"] == "162":
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
     out = {}
     for r in gazetteer(cache, "place"):
         geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
-        if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid):
+        if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
             continue
         county = max(county_part[geoid])[1]
         if county not in counties:
             print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
             continue
-        out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
+        out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
     return out
 
 
-def postal_codes(cache, localities):
+def capitals(cache, places, counties):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
+    """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
+    by_gnis = {p["gnis"]: code for code, p in places.items()}
+    named = collections.defaultdict(list)
+    for code, p in places.items():
+        named[(counties[p["municipality"]]["region"], p["name"])].append(code)
+    by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
+    gnis_of = collections.defaultdict(set)
+    for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
+        gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
+    out = set()
+    for (state, name), ids in sorted(gnis_of.items()):
+        code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
+        if code:
+            out.add(code)
+        else:
+            print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
+    return out
+
+
+def drawn(places, share):
+    """The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
+    sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
+    the sources moves only the places near the cut."""
+    peopled = [code for code, p in places.items() if p["population"] > 0]
+    key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
+    return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
+
+
+def zcta_localities(cache, localities):
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
     return {zcta: place for zcta, place in largest.items() if place in localities}
 
 
-def streets(cache, counties, locality_of_zcta, per_locality):
+def streets(cache, counties, locality_of_zcta, share, minimum):
     """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
     with concurrent.futures.ThreadPoolExecutor(3) as pool:
         list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
             if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
                 for z in zips.get(r["TLID"], ()):
                     count[(locality_of_zcta[z], r["FULLNAME"])] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "addresses"), addresses
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
-    return named, addresses
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
+    cache, out = geo.directories(a)
-    p.add_argument("--min-population", type=int, default=25000)
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
     state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
     regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
@@ -182,20 +208,29 @@ def main():
         counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
 
-    places = localities(cache, a.min_population, counties)
+    places = localities(cache, counties)
-    locality_of_zcta = postal_codes(cache, places)
+    owner_of_zcta = zcta_localities(cache, places)
-    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
+    capital_codes = capitals(cache, places, counties)
-    for geoid in [l for l in places if l not in named]:
+    owners = {p: places[p] for p in set(owner_of_zcta.values())}
-        print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr)
+    for code in sorted(capital_codes - set(owners)):
-        del places[geoid]
+        print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
-    kept_counties = {l["municipality"] for l in places.values()}
+    places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
+    locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
+    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped_localities(dropped["locality"])
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
-    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo.py

@@ -0,0 +1,98 @@
+"""The steps a country's geo import may share with another's."""
+import argparse
+import collections
+import math
+import sys
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
+CACHE = Path(__file__).resolve().parent / "cache"
+
+
+def parser(doc, country, street_share, **own):
+    """A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
+    "_" standing for "-"."""
+    options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
+    p = argparse.ArgumentParser(description=doc.splitlines()[0])
+    for name in sorted(options):
+        p.add_argument("--" + name.replace("_", "-"), **options[name])
+    return p
+
+
+def directories(args):
+    """The cache and out directories of the parsed options, both created."""
+    cache, out = Path(args.cache), Path(args.out)
+    cache.mkdir(parents=True, exist_ok=True)
+    out.mkdir(parents=True, exist_ok=True)
+    return cache, out
+
+
+def centroid(rows):
+    """The mean lat and lon of the rows with a point, or None where no row has one."""
+    points = [r for r in rows if r["lat"] is not None]
+    if not points:
+        return None
+    return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
+
+
+class Nearest:
+    """Nearest point by an equirectangular distance, over a degree grid."""
+
+    def __init__(self, points, cell=0.05):
+        self.cell = cell
+        self.grid = collections.defaultdict(list)
+        for lat, lon, value in points:
+            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
+
+    def find(self, lat, lon):
+        ci, cj = int(lat // self.cell), int(lon // self.cell)
+        best, best_d = None, math.inf
+        ring = 0
+        while ring < 400:
+            for i in range(ci - ring, ci + ring + 1):
+                for j in range(cj - ring, cj + ring + 1):
+                    if max(abs(i - ci), abs(j - cj)) != ring:
+                        continue
+                    for plat, plon, value in self.grid.get((i, j), ()):
+                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
+                        if d < best_d:
+                            best, best_d = value, d
+            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
+                return best
+            ring += 1
+        return best
+
+
+def top_streets(count, share, minimum, column):
+    """count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
+    them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
+    of = collections.defaultdict(list)
+    for (locality, name), n in count.items():
+        of[locality].append((n, name))
+    return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
+
+
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
+
+
+def log_dropped_localities(rows):
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)

data-import/httpmethod.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/httpmethod.tsv from the IANA HTTP Method Registry.
+
+    data-import/httpmethod.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A method ships when the register cites RFC 9110 section 9.3 or RFC 5789 for it, which
+is the eight HTTP defines and PATCH; the register's other entries do not. The
+register's `yes` and `no` ship as `true` and `false`, which a consumer's boolean reads.
+"""
+import argparse
+import csv
+import io
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/http-methods/methods.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpmethod.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["idempotent", "method", "safe"]
+CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
+BOOLEAN = {"yes": "true", "no": "false"}
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+EXPECTED = 9
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        method = (r["Method Name"] or "").strip()
+        if not CORE.search(r["Reference"] or ""):
+            continue
+        if set(method) & set(SELECTOR):
+            sys.exit(f"{method}: a method holding one of {SELECTOR} is one a selector cannot spell")
+        answers = {}
+        for column in ("Safe", "Idempotent"):
+            answer = (r[column] or "").strip()
+            if answer not in BOOLEAN:
+                sys.exit(f"{method}: {column} {answer!r} is neither yes nor no")
+            answers[column] = BOOLEAN[answer]
+        yield {"idempotent": answers["Idempotent"], "method": method, "safe": answers["Safe"]}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "http-methods.csv").decode("utf-8")), key=lambda r: r["method"])
+    if len(table) != EXPECTED:
+        sys.exit(f"{len(table)} methods cite RFC 9110 section 9.3 or RFC 5789, not {EXPECTED}; the register's Reference column has moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/loglevel.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/loglevel.tsv from RFC 5424 and POSIX <syslog.h>.
+
+    data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE]
+
+Every severity in Table 2 ships, paired to POSIX's keywords by the order each source
+lists them, which holds only because a keyword prefixes the severity RFC 5424 numbers
+alike.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt"
+POSIX = "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["code", "keyword", "severity"]
+TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S)
+ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M)
+SECTION = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S)
+MACRO = re.compile(r"\bLOG_([A-Z]+)\b")
+EXPECTED = 8
+
+
+def severities(text):
+    table = TABLE.search(text)
+    if not table:
+        sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved")
+    return ROW.findall(table.group(1))
+
+
+def keywords(page):
+    section = SECTION.search(html.unescape(re.sub(r"<[^>]+>", "", page)))
+    if not section:
+        sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved")
+    return [m.lower() for m in MACRO.findall(section.group(1))]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--posix", default=POSIX)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(severities(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")))
+    codes = [code for code, _ in table]
+    if codes != [str(i) for i in range(EXPECTED)]:
+        sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}")
+    words = keywords(source.fetch(a.posix, a.cache, "posix-syslog.h.html").decode("utf-8"))
+    if len(words) != EXPECTED:
+        sys.exit(f"POSIX lists {len(words)} severity macros, not {EXPECTED}: {words}")
+    rows = []
+    for (code, severity), keyword in zip(table, words):
+        if not severity.lower().startswith(keyword):
+            sys.exit(f"severity {code} is {severity} in the RFC and {keyword} in POSIX, which is no prefix of it")
+        rows.append({"code": code, "keyword": keyword, "severity": severity})
+    tsv.write(a.out, COLUMNS, rows)
+
+
+if __name__ == "__main__":
+    main()

data-import/port.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/port.tsv from the IANA Service Name and Transport Protocol Port Number Registry.
+
+    data-import/port.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a service name and a numeric TCP port. A port the registry lists more than
+once keeps the first service it describes, or the first of them where it describes none,
+so a number selects one row.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "port.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["number", "service"]
+FLOOR = 4000
+
+
+def rows(text):
+    best, described = {}, set()
+    for r in csv.DictReader(io.StringIO(text)):
+        service, number = (r["Service Name"] or "").strip(), (r["Port Number"] or "").strip()
+        transport, description = (r["Transport Protocol"] or "").strip(), (r["Description"] or "").strip()
+        if transport != "tcp" or not service or not number.isdigit():
+            continue
+        if not 1 <= int(number) <= 65535:
+            sys.exit(f"{service}: port {number} is outside 1-65535")
+        if number not in best or (description and number not in described):
+            best[number] = {"number": number, "service": service}
+        if description:
+            described.add(number)
+    return sorted(best.values(), key=lambda r: int(r["number"]))
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(source.fetch(a.source, a.cache, "service-names-port-numbers.csv").decode("utf-8"))
+    if len(table) < FLOOR:
+        sys.exit(f"only {len(table)} ports named a TCP service; the registry's columns have moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/protocol.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/protocol.tsv from the IANA Protocol Numbers registry.
+
+    data-import/protocol.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a number and a keyword, which drops the seven numbers assigned to a class of
+protocols rather than to one; a keyword the registry reserves or marks deprecated goes
+too. Where the registry names no protocol beside the keyword, the keyword is the name.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["keyword", "name", "number"]
+SKIP = ("Reserved", "deprecated")
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        number, keyword = (r["Decimal"] or "").strip(), (r["Keyword"] or "").strip()
+        if not number.isdigit() or not keyword or any(s in keyword for s in SKIP):
+            continue
+        if not 0 <= int(number) <= 255:
+            sys.exit(f"{keyword}: protocol number {number} is outside 0-255")
+        yield {"keyword": keyword, "name": " ".join((r["Protocol"] or "").split()) or keyword, "number": number}
+
+
+def refuse_an_unselectable_row(table):
+    """What the loader refuses at New, and a name that names two rows rather than one."""
+    keywords, names = {}, {}
+    for r in table:
+        for column in ("keyword", "name"):
+            if set(r[column]) & set(SELECTOR):
+                sys.exit(f"{r['keyword']}: {column} {r[column]!r} holds one of {SELECTOR}, which a selector cannot spell")
+        if r["keyword"] in keywords:
+            sys.exit(f"{r['keyword']}: repeats the keyword of protocol {keywords[r['keyword']]}")
+        keywords[r["keyword"]] = r["number"]
+    for r in table:
+        if r["name"] != r["keyword"] and r["name"] in keywords:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} is another protocol's keyword")
+        if r["name"] in names:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} repeats {names[r['name']]}'s, so it selects neither")
+        names[r["name"]] = r["keyword"]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "protocol-numbers-1.csv").decode("utf-8")), key=lambda r: int(r["number"]))
+    refuse_an_unselectable_row(table)
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/source.py

@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
     if path.exists():
         return path.read_bytes()
     for attempt in range(1, 6):
-        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
+        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
         try:
             with urllib.request.urlopen(req, timeout=600) as r:
                 body = r.read()

data-import/tld.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/tld.tsv from the IANA Root Zone Database.
+
+    data-import/tld.py [--source URL_OR_FILE] [--zone URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A TLD ships when the database names a manager for it; one it marks "Not assigned" is a
+delegation the root zone no longer holds, and the zone file is fetched to prove the two
+agree. The key is the A-label the root zone holds; `unicode` is the form the database
+displays, less the bidi marks it wraps a right-to-left label in, and proved to punycode
+back to the key.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/domains/root/db"
+ZONE = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "tld.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["tld", "type", "unicode"]
+TYPES = ("country-code", "generic", "generic-restricted", "infrastructure", "sponsored")
+BIDI = {0x200E: None, 0x200F: None}
+ROW = re.compile(r'<a href="/domains/root/db/([^"]+)\.html">([^<]+)</a>.*?</td>\s*<td>([^<]*)</td>\s*<td>(.*?)</td>', re.S)
+
+
+def rows(page):
+    for a_label, shown, kind, manager in ROW.findall(page):
+        if "Not assigned" in manager:
+            continue
+        kind = kind.strip()
+        if kind not in TYPES:
+            sys.exit(f"{a_label}: the database types it {kind!r}, which is none of {TYPES}")
+        yield {"tld": "." + a_label, "type": kind, "unicode": html.unescape(shown).strip().translate(BIDI)}
+
+
+def prove_each_unicode_form_is_its_key(table):
+    """The displayed form is the key's own spelling, which punycode encodes back to."""
+    for r in table:
+        label = r["unicode"][1:]
+        encoded = label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
+        if "." + encoded != r["tld"]:
+            sys.exit(f"{r['tld']}: the database displays {r['unicode']!r}, which is no spelling of it")
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--zone", default=ZONE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "root-db.html").decode("utf-8")), key=lambda r: r["tld"])
+    prove_each_unicode_form_is_its_key(table)
+    listing = source.fetch(a.zone, a.cache, "tlds-alpha-by-domain.txt").decode("utf-8")
+    delegated = {"." + line.strip().lower() for line in listing.splitlines() if line.strip() and not line.startswith("#")}
+    shipped = {r["tld"] for r in table}
+    if len(table) != len(delegated) or shipped != delegated:
+        sys.exit(f"the database assigns {len(table)} rows where the root zone holds {len(delegated)} TLDs: {sorted(shipped ^ delegated)[:10]}")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/tsv.py

@@ -5,14 +5,14 @@ from pathlib import Path
 
 
 def write(path, columns, rows):
-    """Write the rows as a TSV; the table needs two rows, and every cell must be non-empty and free of tabs, newlines and braces."""
+    """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
     lines = ["\t".join(columns)]
     for row in rows:
-        cells = [str(row[c]) for c in columns]
+        cells = [str(row[c]).strip() for c in columns]
         if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
             raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
         lines.append("\t".join(cells))
-    if len(lines) < 3:
+    if len(lines) < 2:
-        raise ValueError(f"{path}: has {len(lines) - 1} rows below its header; a table is at least two rows")
+        raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
     Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
     print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)

data.go

@@ -1,195 +1,175 @@
 package fejkdata
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
+// loadSources merges sources in order, the last winning a clash. A source whose manifest
-// label prefixes file names in errors; onDisk marks path as a directory that must
+// carries an index leaves its categories unloaded; every other source loads here, with
-// exist.
+// the unloaded categories it needs.
-type dataSource struct {
+// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
-	fsys   fs.FS
+func loadSources(sources []datafiles.Source) (folder, error) {
-	label  string
+	root := folder{children: map[string]node{}}
-	onDisk bool
-	path   string
-	root   string
-}
-
-func (s dataSource) name(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-// loadData loads every source into one namespace tree and returns its root
-// children. A directory becomes a folder; each *.json file in it compiles to a node
-// keyed by its base name (address.json -> "address"); each subdirectory becomes a
-// nested folder, so folders turn into dot-path segments. Sources merge left to right:
-// matching folders merge by their children, and any other clash is won by the last
-// source loaded. Once merged, linkRefs binds every reference against the
-// final tree.
-func loadData(sources []dataSource) (map[string]node, error) {
-	root := map[string]node{}
 	for _, src := range sources {
-		if src.onDisk {
+		m, err := src.Manifest()
-			if src.path == "" {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.path)
 		if err != nil {
-				return nil, fmt.Errorf("data path %s: %w", src.path, err)
+			return folder{}, err
 		}
-			if !info.IsDir() {
+		if err := addSource(&root, src, m); err != nil {
-				return nil, fmt.Errorf("%s is not a directory", src.path)
+			return folder{}, err
 		}
 	}
-		dir := src.root
+	return root, loadReached(&root, categorySites(&root), nil)
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
-		if err != nil {
-			return nil, err
-		}
-		mergeChildren(root, g.children)
-	}
-	if len(root) == 0 {
-		return nil, fmt.Errorf("no .json data found")
-	}
-	setTablePaths(root)
-	if err := linkTables(root); err != nil {
-		return nil, err
-	}
-	if err := linkRefs(root); err != nil {
-		return nil, err
-	}
-	if err := checkNoCycles(root); err != nil {
-		return nil, err
-	}
-	if err := checkScope(treeScope(root)); err != nil {
-		return nil, err
-	}
-	return root, nil
 }
 
-// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
+func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
-// by name, so the tree is built deterministically. Empty subdirectories (no JSON
+	src = src.Mounted(m.Namespace)
-// anywhere under them) are skipped rather than added as empty namespaces.
+	if m.Index != nil {
-func loadDir(src dataSource, dir string) (*folder, error) {
+		placeIndex(root, src, m.Index)
-	entries, err := fs.ReadDir(src.fsys, dir)
+		return nil
-	if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.name(dir), err)
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
-	for _, e := range entries {
+		return err
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
 	}
-	}
+	mergeFolder(root, g)
-	for _, e := range entries {
+	return nil
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
-		}
-		full := path.Join(dir, e.Name())
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
-		}
-	}
-	return g, nil
 }
 
-// categoryFiles is what a category may name beside itself: the rows files of its
+func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
-// directory, each marked once a category names it.
+	return func(c datafiles.Category) error {
-type categoryFiles struct {
+		n, err := compileCategory(c)
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-// rows reads the rows file a category names beside it.
-func (c *categoryFiles) rows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
 		if err != nil {
 			return err
 		}
-	if len(child.children) == 0 {
+		place(c.Folders).put(c.Name, n)
 		return nil
 	}
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.name(full), err)
-	}
-	g.children[name] = child
-	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
+// madeFolder is the folder that dir names under root, made where it is missing or a category.
-// is skipped.
+func madeFolder(root *folder, dir []string) *folder {
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
+	g := root
-	if !strings.HasSuffix(file, ".json") {
+	for _, seg := range dir {
-		return nil
+		sub, isFolder := g.children[seg].(*folder)
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.put(seg, sub)
 		}
-	name := strings.TrimSuffix(file, ".json")
+		g = sub
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.name(full), err)
 	}
-	b, err := fs.ReadFile(src.fsys, full)
+	return g
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	g.children[name] = n
-	return nil
 }
 
-// mergeChildren overlays src onto dst. Two folders under the same key merge
+func mergeFolder(dst, src *folder) {
-// recursively (so locales/categories from several paths combine); every other
+	for k, v := range src.children {
-// key is replaced, making the last-loaded directory win on a conflict.
+		if dg, ok := dst.children[k].(*folder); ok {
-func mergeChildren(dst, src map[string]node) {
-	for k, v := range src {
-		if dg, ok := dst[k].(*folder); ok {
 			if sg, ok := v.(*folder); ok {
-				mergeChildren(dg.children, sg.children)
+				mergeFolder(dg, sg)
 				continue
 			}
 		}
-		dst[k] = v
+		dst.put(k, v)
 	}
 }
+
+// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
+// single inline node.
+type nodeSet func(fn func(label string, n node) error) error
+
+// categorySite is a loaded category, where it sits and its own dot path.
+type categorySite struct {
+	categoryAt
+	n    node
+	path string
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, siteIn(dir, g, name))
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func siteIn(dir []string, in *folder, name string) categorySite {
+	return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
+}
+
+func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
+
+func siteNodes(sites []categorySite) nodeSet {
+	return func(fn func(label string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func inlineNodes(n node, label string) nodeSet {
+	return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
+}
+
+// pipeline is the one load pipeline, run over one nodeSet; its fields are where
+// entry points differ.
+type pipeline struct {
+	nodes   nodeSet
+	resolve func() error
+	// Set by a caller that proves the columns against their Go types itself.
+	typedByGo bool
+}
+
+// categoryPipeline is the pipeline over the categories of one load, their references
+// resolving against root.
+func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
+	return pipeline{
+		nodes: siteNodes(sites),
+		resolve: func() error {
+			if err := linkTables(sites); err != nil {
+				return err
+			}
+			return resolveCategoryTemplates(sites, root)
+		},
+	}
+}
+
+func (p pipeline) run() error {
+	if err := p.resolve(); err != nil {
+		return err
+	}
+	if err := checkNoCycles(p.nodes); err != nil {
+		return err
+	}
+	settleRecords(p.nodes)
+	if !p.typedByGo {
+		if err := checkColumns(p.nodes); err != nil {
+			return err
+		}
+	}
+	return checkNodeFences(p.nodes)
+}
+
+// checkNodeFences runs the per-node fences over s, each over every node before the next,
+// so which of several broken nodes is reported does not depend on the walk. Its walks
+// recurse unguarded, so pipeline.run refuses cycles first.
+func checkNodeFences(s nodeSet) error {
+	mem := renderCounts{}
+	if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
+		return err
+	}
+	return s((&valueProof{}).checkDatatype)
+}

data/en_US/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"en_US.color": {"paths":[""]},
+		"en_US.company": {"paths":["","base","suffix"]},
+		"en_US.date": {"paths":[""]},
+		"en_US.email": {"paths":["","domain","local","local.n"]},
+		"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"en_US.ip": {"paths":[""]},
+		"en_US.itin": {"paths":["","area","group","serial"]},
+		"en_US.last-name": {"paths":["","count","name"]},
+		"en_US.person": {"paths":["","first","last","prefix","sex"]},
+		"en_US.phone": {"paths":["","area","exchange","line"]},
+		"en_US.phone-exchange": {"paths":[""]},
+		"en_US.price": {"paths":["","amt","cents"]},
+		"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
+		"en_US.ssn": {"paths":["","area","group","serial"]},
+		"en_US.time": {"paths":[""]},
+		"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
+		"en_US.url": {"paths":["","host","path"]},
+		"en_US.username": {"paths":["","n"]},
+		"en_US.version": {"paths":["","n","pre","suffix"]},
+		"en_US.word": {"paths":[""]}
+	},
+	"namespace": "en_US"
+}

data/en_US/address.json

@@ -1,8 +1,8 @@
 {
-  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
+  "format": "{/geo.US.address as a}{a}",
-  "street": "{/geo.US.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.US.address.street-number}",
+  "street-number": "{a.street-number}",
-  "locality": "{/geo.US.address.locality}",
+  "locality": "{a.locality}",
-  "region": "{/geo.US.address.region}",
+  "region": "{a.region}",
-  "postal-code": "{/geo.US.address.postal-code}"
+  "postal-code": "{a.postal-code}"
 }

data/en_US/fs.go

@@ -0,0 +1,19 @@
+// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
+// geo.US, which package geous holds.
+package enus
+
+import (
+	"embed"
+	"io/fs"
+
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.US.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }

data/en_US/fs_test.go

@@ -0,0 +1,22 @@
+package enus_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	enus "github.com/larvit/fejkdata/data/en_US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/en_US/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
-  "prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }],
+  "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data/en_US/phone-exchange.json

@@ -0,0 +1,17 @@
+[
+  { "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
+  { "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
+  { "format": "{int(2,9)}10", "weight": 8 },
+  { "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
+  { "format": "55{int(0,4)}", "weight": 5 },
+  { "format": "55{int(6,9)}", "weight": 4 },
+  { "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
+  { "format": "95{int(1,7)}", "weight": 7 },
+  { "format": "96{digits(1)}", "weight": 10 },
+  { "format": "97{int(0,5)}", "weight": 6 },
+  { "format": "97{int(7,9)}", "weight": 3 }
+]

data/en_US/phone.json

@@ -1,15 +1,15 @@
 [
   {
-    "format": "({area}) {exch}-{line}",
+    "format": "({area}) {exchange}-{line}",
     "weight": 2,
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   },
   {
-    "format": "{area}-{exch}-{line}",
+    "format": "{area}-{exchange}-{line}",
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   }
 ]

data/geo/SE/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
+		"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
+		"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
+		"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
+	},
+	"namespace": "geo.SE"
+}

data/geo/SE/address.json

@@ -1,6 +1,6 @@
 {
   "format": "{street} {street-number}\n{postal-code} {locality}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(1,9)}",
     "{int(10,99)}",
@@ -9,6 +9,6 @@
     { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
     { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
   ],
-  "postal-code": "{.postal-code.code}",
+  "postal-code": "{l.postal-code.code}",
-  "locality": "{.locality.name}"
+  "locality": "{l.name}"
 }

data/geo/SE/fs.go

@@ -0,0 +1,15 @@
+// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
+package geose
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/SE/fs_test.go

@@ -0,0 +1,22 @@
+package geose_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/geo/SE/locality.tsv

@@ -866,7 +866,6 @@ Nynäshamn	0192	16447	58.9034	17.9479
 Nyvång	1277	200	56.1333	12.9000
 Nyåker	2403	200	63.7500	19.3167
 Nälden	2309	986	63.3500	14.2500
-Nämdö	0120	200	59.1833	18.6833
 Nässjö	0682	18639	57.6531	14.6968
 Näsum	1272	1274	56.1667	14.4833
 Näsviken	2184	994	61.7667	16.8667

data/geo/SE/postal-code.tsv

@@ -949,7 +949,6 @@ code	locality
 130 25	Ingmarsö
 130 33	Gällnöby
 130 34	Skälvik
-130 36	Nämdö
 130 37	Stavsnäs
 130 38	Runmarö
 130 39	Sandhamn

data/geo/US/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
+		"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
+		"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
+		"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
+	},
+	"namespace": "geo.US"
+}

data/geo/US/address.json

@@ -1,13 +1,13 @@
 {
   "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 2 },
     { "format": "{int(1000,9999)}", "weight": 0.5 },
     { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
   ],
-  "locality": "{.locality.name}",
+  "locality": "{l.name}",
-  "region": "{.region.abbr}",
+  "region": "{l..municipality..region.abbr}",
-  "postal-code": "{.postal-code.code}"
+  "postal-code": "{l.postal-code.code}"
 }

data/geo/US/fs.go

@@ -0,0 +1,15 @@
+// Package geous is fejkdata's US places module, whose categories are geo.US.*.
+package geous
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/US/fs_test.go

@@ -0,0 +1,22 @@
+package geous_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/.fejkdata.json

@@ -0,0 +1,25 @@
+{
+	"index": {
+		"misc.car": {"paths":["","make","model"]},
+		"misc.coordinate": {"paths":["","lat","lon"]},
+		"misc.creditcard": {"paths":["","d"]},
+		"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
+		"misc.datetime": {"paths":[""]},
+		"misc.emoji": {"paths":[""]},
+		"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
+		"misc.httpstatus": {"paths":["","code","reason"]},
+		"misc.language": {"paths":["","code","code3","name"]},
+		"misc.loglevel": {"paths":["","code","keyword","severity"]},
+		"misc.mac": {"paths":[""]},
+		"misc.mimetype": {"paths":["","ext","type"]},
+		"misc.objectid": {"paths":[""]},
+		"misc.port": {"paths":["","number","service"]},
+		"misc.protocol": {"paths":["","keyword","name","number"]},
+		"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
+		"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
+		"misc.tld": {"paths":["","tld","type","unicode"]},
+		"misc.useragent": {"paths":["","browser","device","os","ua"]},
+		"misc.uuid": {"paths":["","variant"]}
+	},
+	"namespace": "misc"
+}

data/misc/fs.go

@@ -0,0 +1,15 @@
+// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
+package misc
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/misc/fs_test.go

@@ -0,0 +1,22 @@
+package misc_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	"github.com/larvit/fejkdata/data/misc"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/httpmethod.json

@@ -0,0 +1 @@
+{ "format": "{method}", "rows": "httpmethod.tsv", "key": "method" }

data/misc/httpmethod.tsv

@@ -0,0 +1,10 @@
+idempotent	method	safe
+false	CONNECT	false
+true	DELETE	false
+true	GET	true
+true	HEAD	true
+true	OPTIONS	true
+false	PATCH	false
+false	POST	false
+true	PUT	false
+true	TRACE	true

data/misc/loglevel.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "loglevel.tsv", "key": "code", "name": "keyword" }

data/misc/loglevel.tsv

@@ -0,0 +1,9 @@
+code	keyword	severity
+0	emerg	Emergency
+1	alert	Alert
+2	crit	Critical
+3	err	Error
+4	warning	Warning
+5	notice	Notice
+6	info	Informational
+7	debug	Debug

data/misc/port.json

@@ -0,0 +1 @@
+{ "format": "{number}", "rows": "port.tsv", "key": "number" }

data/misc/protocol.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "protocol.tsv", "key": "keyword", "name": "name" }

data/misc/protocol.tsv

@@ -0,0 +1,140 @@
+keyword	name	number
+HOPOPT	IPv6 Hop-by-Hop Option	0
+ICMP	Internet Control Message	1
+IGMP	Internet Group Management	2
+GGP	Gateway-to-Gateway	3
+IPv4	IPv4 encapsulation	4
+ST	Stream	5
+TCP	Transmission Control	6
+CBT	CBT	7
+EGP	Exterior Gateway Protocol	8
+IGP	any private interior gateway (used by Cisco for their IGRP)	9
+BBN-RCC-MON	BBN RCC Monitoring	10
+NVP-II	Network Voice Protocol	11
+PUP	PUP	12
+EMCON	EMCON	14
+XNET	Cross Net Debugger	15
+CHAOS	Chaos	16
+UDP	User Datagram	17
+MUX	Multiplexing	18
+DCN-MEAS	DCN Measurement Subsystems	19
+HMP	Host Monitoring	20
+PRM	Packet Radio Measurement	21
+XNS-IDP	XEROX NS IDP	22
+TRUNK-1	Trunk-1	23
+TRUNK-2	Trunk-2	24
+LEAF-1	Leaf-1	25
+LEAF-2	Leaf-2	26
+RDP	Reliable Data Protocol	27
+IRTP	Internet Reliable Transaction	28
+ISO-TP4	ISO Transport Protocol Class 4	29
+NETBLT	Bulk Data Transfer Protocol	30
+MFE-NSP	MFE Network Services Protocol	31
+MERIT-INP	MERIT Internodal Protocol	32
+DCCP	Datagram Congestion Control Protocol	33
+3PC	Third Party Connect Protocol	34
+IDPR	Inter-Domain Policy Routing Protocol	35
+XTP	XTP	36
+DDP	Datagram Delivery Protocol	37
+IDPR-CMTP	IDPR Control Message Transport Proto	38
+TP++	TP++ Transport Protocol	39
+IL	IL Transport Protocol	40
+IPv6	IPv6 encapsulation	41
+SDRP	Source Demand Routing Protocol	42
+IPv6-Route	Routing Header for IPv6	43
+IPv6-Frag	Fragment Header for IPv6	44
+IDRP	Inter-Domain Routing Protocol	45
+RSVP	Reservation Protocol	46
+GRE	Generic Routing Encapsulation	47
+DSR	Dynamic Source Routing Protocol	48
+BNA	BNA	49
+ESP	Encap Security Payload	50
+AH	Authentication Header	51
+I-NLSP	Integrated Net Layer Security TUBA	52
+NARP	NBMA Address Resolution Protocol	54
+Min-IPv4	Minimal IPv4 Encapsulation	55
+TLSP	Transport Layer Security Protocol using Kryptonet key management	56
+SKIP	SKIP	57
+IPv6-ICMP	ICMP for IPv6	58
+IPv6-NoNxt	No Next Header for IPv6	59
+IPv6-Opts	Destination Options for IPv6	60
+CFTP	CFTP	62
+SAT-EXPAK	SATNET and Backroom EXPAK	64
+KRYPTOLAN	Kryptolan	65
+RVD	MIT Remote Virtual Disk Protocol	66
+IPPC	Internet Pluribus Packet Core	67
+SAT-MON	SATNET Monitoring	69
+VISA	VISA Protocol	70
+IPCV	Internet Packet Core Utility	71
+CPNX	Computer Protocol Network Executive	72
+CPHB	Computer Protocol Heart Beat	73
+WSN	Wang Span Network	74
+PVP	Packet Video Protocol	75
+BR-SAT-MON	Backroom SATNET Monitoring	76
+SUN-ND	SUN ND PROTOCOL-Temporary	77
+WB-MON	WIDEBAND Monitoring	78
+WB-EXPAK	WIDEBAND EXPAK	79
+ISO-IP	ISO Internet Protocol	80
+VMTP	VMTP	81
+SECURE-VMTP	SECURE-VMTP	82
+VINES	VINES	83
+IPTM	Internet Protocol Traffic Manager	84
+NSFNET-IGP	NSFNET-IGP	85
+DGP	Dissimilar Gateway Protocol	86
+TCF	TCF	87
+EIGRP	EIGRP	88
+OSPFIGP	OSPFIGP	89
+Sprite-RPC	Sprite RPC Protocol	90
+LARP	Locus Address Resolution Protocol	91
+MTP	Multicast Transport Protocol	92
+AX.25	AX.25 Frames	93
+IPIP	IP-within-IP Encapsulation Protocol	94
+SCC-SP	Semaphore Communications Sec. Pro.	96
+ETHERIP	Ethernet-within-IP Encapsulation	97
+ENCAP	Encapsulation Header	98
+GMTP	GMTP	100
+IFMP	Ipsilon Flow Management Protocol	101
+PNNI	PNNI over IP	102
+PIM	Protocol Independent Multicast	103
+ARIS	ARIS	104
+SCPS	SCPS	105
+QNX	QNX	106
+A/N	Active Networks	107
+IPComp	IP Payload Compression Protocol	108
+SNP	Sitara Networks Protocol	109
+Compaq-Peer	Compaq Peer Protocol	110
+IPX-in-IP	IPX in IP	111
+VRRP	Virtual Router Redundancy Protocol	112
+PGM	PGM Reliable Transport Protocol	113
+L2TP	Layer Two Tunneling Protocol	115
+DDX	D-II Data Exchange (DDX)	116
+IATP	Interactive Agent Transfer Protocol	117
+STP	Schedule Transfer Protocol	118
+SRP	SpectraLink Radio Protocol	119
+UTI	UTI	120
+SMP	Simple Message Protocol	121
+PTP	Performance Transparency Protocol	123
+ISIS over IPv4	ISIS over IPv4	124
+FIRE	FIRE	125
+CRTP	Combat Radio Transport Protocol	126
+CRUDP	Combat Radio User Datagram	127
+SSCOPMCE	SSCOPMCE	128
+IPLT	IPLT	129
+SPS	Secure Packet Shield	130
+PIPE	Private IP Encapsulation within IP	131
+SCTP	Stream Control Transmission Protocol	132
+FC	Fibre Channel	133
+RSVP-E2E-IGNORE	RSVP-E2E-IGNORE	134
+Mobility Header	Mobility Header	135
+UDPLite	UDPLite	136
+MPLS-in-IP	MPLS-in-IP	137
+manet	MANET Protocols	138
+HIP	Host Identity Protocol	139
+Shim6	Shim6 Protocol	140
+WESP	Wrapped Encapsulating Security Payload	141
+ROHC	Robust Header Compression	142
+Ethernet	Ethernet	143
+AGGFRAG	AGGFRAG encapsulation payload for ESP	144
+NSH	Network Service Header	145
+Homa	Homa	146
+BIT-EMU	Bit-stream Emulation	147

data/misc/tld.json

@@ -0,0 +1 @@
+{ "format": "{tld}", "rows": "tld.tsv", "key": "tld", "name": "unicode" }

data/modules.go

@@ -0,0 +1,19 @@
+// Package data lists every module fejkdata ships.
+package data
+
+import (
+	"io/fs"
+
+	enus "github.com/larvit/fejkdata/data/en_US"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+	"github.com/larvit/fejkdata/data/misc"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
+
+// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS {
+	return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
+}

data/sv_SE/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
+		"sv_SE.color": {"paths":[""]},
+		"sv_SE.company": {"paths":["","base","suffix"]},
+		"sv_SE.date": {"paths":[""]},
+		"sv_SE.email": {"paths":["","domain","local","local.n"]},
+		"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"sv_SE.ip": {"paths":[""]},
+		"sv_SE.last-name": {"paths":["","count","name"]},
+		"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
+		"sv_SE.personnummer": {"paths":[""]},
+		"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
+		"sv_SE.price": {"paths":["","amt","ore"]},
+		"sv_SE.samordningsnummer": {"paths":[""]},
+		"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
+		"sv_SE.time": {"paths":[""]},
+		"sv_SE.title": {"paths":["","name","share"]},
+		"sv_SE.url": {"paths":["","host","path"]},
+		"sv_SE.username": {"paths":["","n"]},
+		"sv_SE.version": {"paths":["","n","pre","suffix"]},
+		"sv_SE.word": {"paths":[""]}
+	},
+	"namespace": "sv_SE"
+}

data/sv_SE/address.json

@@ -1,7 +1,7 @@
 {
-  "format": "{street} {street-number}\n{postal-code} {locality}",
+  "format": "{/geo.SE.address as a}{a}",
-  "street": "{/geo.SE.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.SE.address.street-number}",
+  "street-number": "{a.street-number}",
-  "postal-code": "{/geo.SE.address.postal-code}",
+  "postal-code": "{a.postal-code}",
-  "locality": "{/geo.SE.address.locality}"
+  "locality": "{a.locality}"
 }

data/sv_SE/fs.go

@@ -0,0 +1,19 @@
+// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
+// geo.SE, which package geose holds.
+package svse
+
+import (
+	"embed"
+	"io/fs"
+
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.SE.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }

data/sv_SE/fs_test.go

@@ -0,0 +1,22 @@
+package svse_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/sv_SE/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
   "prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data_test.go

@@ -1,7 +1,10 @@
 package fejkdata
 
 import (
+	"io/fs"
+	"os"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
 			regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
 	}
 
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, c := range cases {
 		for i := 0; i < 200; i++ {
 			if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
 	}
 }
 
-// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper
+// TestShippedMiscCategories covers the locale-neutral misc module: a proper
 // v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
 // whose trailing {luhn()} check passes.
 func TestShippedMiscCategories(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
 	mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
 	objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
@@ -93,7 +96,7 @@ func TestShippedMiscCategories(t *testing.T) {
 // codes follow their standard shape, dotted sub-paths resolve, emoji is a real
 // glyph, and a coordinate parses to a point within geographic bounds.
 func TestShippedMiscReferenceData(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	re := map[string]*regexp.Regexp{
 		"currency":              regexp.MustCompile(`^[A-Z]{3}$`),
 		"currency.name":         regexp.MustCompile(`\p{L}`),
@@ -111,6 +114,20 @@ func TestShippedMiscReferenceData(t *testing.T) {
 		"timezone.weight":       regexp.MustCompile(`^[1-9]\d*$`),
 		"mimetype":              regexp.MustCompile(`^[a-z]+/[a-z0-9.+-]+$`),
 		"mimetype.ext":          regexp.MustCompile(`^\.[a-z0-9_-]+$`),
+		"httpmethod":            regexp.MustCompile(`^(CONNECT|DELETE|GET|HEAD|OPTIONS|PATCH|POST|PUT|TRACE)$`),
+		"httpmethod.safe":       regexp.MustCompile(`^(true|false)$`),
+		"httpmethod.idempotent": regexp.MustCompile(`^(true|false)$`),
+		"port":                  regexp.MustCompile(`^[1-9]\d{0,4}$`),
+		"port.service":          regexp.MustCompile(`^\S+$`),
+		"protocol":              regexp.MustCompile(`^\S+( \S+)*$`),
+		"protocol.name":         regexp.MustCompile(`\S`),
+		"protocol.number":       regexp.MustCompile(`^\d{1,3}$`),
+		"loglevel":              regexp.MustCompile(`^(alert|crit|debug|emerg|err|info|notice|warning)$`),
+		"loglevel.code":         regexp.MustCompile(`^[0-7]$`),
+		"loglevel.severity":     regexp.MustCompile(`^(Alert|Critical|Debug|Emergency|Error|Informational|Notice|Warning)$`),
+		"tld":                   regexp.MustCompile(`^\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$`),
+		"tld.type":              regexp.MustCompile(`^(country-code|generic|generic-restricted|infrastructure|sponsored)$`),
+		"tld.unicode":           regexp.MustCompile(`^\.\S+$`),
 		"httpstatus":            regexp.MustCompile(`^[1-5]\d{2} \S.*$`),
 		"httpstatus.code":       regexp.MustCompile(`^[1-5]\d{2}$`),
 		"useragent":             regexp.MustCompile(`^Mozilla/5\.0 .+`),
@@ -142,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
 // TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
 // regexes miss: no generated name repeats a character three times over.
 func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
 		for i := 0; i < 20000; i++ {
 			name := fake(t, f, path)
@@ -161,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
 // male), the trailing digit is a Luhn checksum over the other nine, and a
 // samordningsnummer is the same with 60 added to the day.
 func TestSwedishPersonnummer(t *testing.T) {
-	sv := newGenerator(t, "data", WithSeed(1))
+	sv := shipped(t, WithSeed(1))
 	re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
 	sawLongMonthEnd, birth := false, map[string]bool{}
 	for i := 0; i < 2000; i++ {
@@ -194,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
 	if !birth["238"] || !birth["239"] {
 		t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
 	}
-	for i := 0; i < 200; i++ {
-		got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
-		sex, id, _ := strings.Cut(got, " ")
-		if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
-			t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
-		}
-	}
 }
 
 // TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
 // 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
 // in 50-65, 70-88, 90-92 or 94-99.
 func TestShippedUSTaxIds(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
 	itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
 	for i := 0; i < 2000; i++ {
@@ -231,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
 // TestShippedPersonNames pins the name tables: a sex pins its first names, a name
 // both sexes carry appears under both, and the record's sex agrees with its name.
 func TestShippedPersonNames(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(9))
+	f := shipped(t, WithSeed(9))
 	for path, want := range map[string]string{
 		"sv_SE.sex[f]":                        "kvinna",
 		"sv_SE.sex[man].code":                 "m",
@@ -259,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
 		seen := map[string]bool{}
 		for i := 0; i < 2000; i++ {
 			seen[fake(t, f, locale+".sex[f].first-name")] = true
-			first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex")
+			got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
-			if first == "" || sex == "" {
+			first, sex, _ := strings.Cut(got, "|")
-				t.Fatalf("%s.person lacks a first name or a sex", locale)
+			fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
-			}
 		}
 		if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
 			t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
 		}
-		got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`)
+	}
-		code, first, _ := strings.Cut(got, "|")
+}
-		if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
+
-			t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code)
+// TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
+// from one pick, and that two names bound to one person category are two people.
+func TestShippedPersonIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(9))
+	for _, locale := range []string{"sv_SE", "en_US"} {
+		apart := false
+		for i := 0; i < 300; i++ {
+			got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
+			parts := strings.Split(got, "|")
+			for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
+				fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
+			}
+			if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
+				fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
+			}
+			apart = apart || parts[0] != parts[3]
+		}
+		if !apart {
+			t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
 		}
 	}
 }
 
 func TestShippedSwedishPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
 	for i := 0; i < 50; i++ {
 		if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
@@ -286,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
 }
 
 func TestShippedSwedishAddress(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(3))
+	f := shipped(t, WithSeed(3))
 	digit := regexp.MustCompile(`\d`)
 	for i := 0; i < 30; i++ {
 		a := fake(t, f, "sv_SE.address")
@@ -303,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
 	}
 }
 
+// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
+// locality and region are one place in the geo tables, read through a name in a template or
+// across a struct's tags, and that a reference beside the name is another address.
+func TestShippedAddressIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(3))
+	for _, c := range []struct{ locale, country, template, largest string }{
+		{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
+		{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
+	} {
+		places := geoPlaces(t, c.country)
+		apart, localities := false, map[string]int{}
+		for i := 0; i < 2000; i++ {
+			got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
+			parts := strings.Split(got, "|")
+			if !places[strings.Join(parts[:4], "|")] {
+				t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
+			}
+			apart = apart || parts[0] != parts[4]
+			localities[parts[2]]++
+		}
+		if !apart {
+			t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
+		}
+		if c.country == "SE" {
+			var row struct {
+				Street     string `fake:"{/sv_SE.address as a}{a.street}"`
+				PostalCode string `fake:"{a.postal-code}"`
+				Locality   string `fake:"{a.locality}"`
+			}
+			for i := 0; i < 200; i++ {
+				if err := f.FakeStruct(&row); err != nil {
+					t.Fatal(err)
+				}
+				if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
+					t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
+				}
+			}
+		}
+		for l, n := range localities {
+			if n > localities[c.largest] {
+				t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
+			}
+		}
+	}
+}
+
+// 0644000 is Los Angeles.
+func TestShippedPlacesSpreadAddresses(t *testing.T) {
+	for country, city := range map[string]struct {
+		key   string
+		least int
+	}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
+		streets := 0
+		for _, r := range geoRows(t, country, "street") {
+			if r[1] == city.key {
+				streets++
+			}
+		}
+		if streets < city.least {
+			t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
+		}
+	}
+	header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
+	population := slices.Index(strings.Split(header, "\t"), "population")
+	if population < 0 {
+		t.Fatal("geo.US locality.tsv has no population column")
+	}
+	for _, r := range geoRows(t, "US", "locality") {
+		n, err := strconv.Atoi(r[population])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n < 2500 {
+			return
+		}
+	}
+	t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
+}
+
+// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
+// draw by population leaves out.
+func TestShippedCapitalsAndBigPlaces(t *testing.T) {
+	state := map[string]string{}
+	for _, r := range geoRows(t, "US", "municipality") {
+		state[r[0]] = r[2]
+	}
+	shipped := map[string]bool{}
+	for _, r := range geoRows(t, "US", "locality") {
+		shipped[r[1]+", "+state[r[2]]] = true
+	}
+	for _, place := range []string{
+		"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
+		"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
+		"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
+		"Ventura, CA",
+	} {
+		if !shipped[place] {
+			t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
+		}
+	}
+}
+
+func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
+	refused := map[string]bool{
+		"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
+		"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
+		"Öresundsvägen, Bondevägen": true,
+	}
+	for _, r := range geoRows(t, "SE", "street") {
+		if refused[r[0]] {
+			t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
+		}
+	}
+}
+
+func geoTSV(t *testing.T, country, table string) string {
+	b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func geoRows(t *testing.T, country, table string) [][]string {
+	var out [][]string
+	for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
+		out = append(out, strings.Split(line, "\t"))
+	}
+	return out
+}
+
+// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
+// one locality holds, the region empty for SE, whose address carries none.
+func geoPlaces(t *testing.T, country string) map[string]bool {
+	rows := func(table string) [][]string { return geoRows(t, country, table) }
+	name, region := map[string]string{}, map[string]string{}
+	if country == "US" {
+		county := map[string]string{}
+		for _, r := range rows("municipality") {
+			county[r[0]] = r[2]
+		}
+		for _, r := range rows("locality") {
+			name[r[0]], region[r[0]] = r[1], county[r[2]]
+		}
+	} else {
+		for _, r := range rows("locality") {
+			name[r[0]] = r[0]
+		}
+	}
+	codes := map[string][]string{}
+	for _, r := range rows("postal-code") {
+		codes[r[1]] = append(codes[r[1]], r[0])
+	}
+	places := map[string]bool{}
+	for _, r := range rows("street") {
+		for _, code := range codes[r[1]] {
+			places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
+		}
+	}
+	return places
+}
+
 func TestShippedPersonHasParts(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(7))
+	f := shipped(t, WithSeed(7))
 	for _, path := range []string{"sv_SE.person", "en_US.person"} {
 		for i := 0; i < 30; i++ {
 			if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
@@ -314,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
 	}
 }
 
+// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
 func TestShippedUSPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
-	re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`)
+	re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
-	for i := 0; i < 50; i++ {
+	unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
-		if n := fake(t, f, "en_US.phone"); !re.MatchString(n) {
+	for i := 0; i < 2000; i++ {
+		n := fake(t, f, "en_US.phone")
+		m := re.FindStringSubmatch(n)
+		if m == nil {
 			t.Fatalf("phone %q does not match %s", n, re)
 		}
+		if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
+			t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
+		}
 	}
 }
 
-// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and
+func TestShippedTableCellsAreTrimmed(t *testing.T) {
-// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person.
+	err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil || !strings.HasSuffix(path, ".tsv") {
+			return err
+		}
+		b, err := fs.ReadFile(os.DirFS("data"), path)
+		if err != nil {
+			return err
+		}
+		for i, line := range strings.Split(string(b), "\n") {
+			for _, cell := range strings.Split(line, "\t") {
+				if cell != strings.TrimSpace(cell) {
+					t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
+// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
 func TestShippedNamespacedTree(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
 		if got := fake(t, f, path); got == "" {
 			t.Fatalf("Fake(%q) returned empty", path)
@@ -371,7 +590,7 @@ func luhnValid(s string) bool {
 var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
 
 func TestShippedStreetIsARegisteredName(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
@@ -383,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
 func TestShippedLastNameComposition(t *testing.T) {
 	// last is a choice of patronymic {first}sson templates, compound
 	// {first}{last} templates and literal surnames.
-	f := newGenerator(t, "data", WithSeed(6))
+	f := shipped(t, WithSeed(6))
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
 			t.Fatalf("last name %q is not a Swedish surname", s)
@@ -393,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
 
 func TestShippedStreetNumberFormats(t *testing.T) {
 	// Reachable via a hyphenated path; covers all five weighted number variants.
-	f := newGenerator(t, "data", WithSeed(8))
+	f := shipped(t, WithSeed(8))
 	re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
 	for i := 0; i < 300; i++ {
 		if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
@@ -405,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
 // TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
 // that the everyday titles are reachable.
 func TestShippedUSTitleAgreesWithSex(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(4))
+	f := shipped(t, WithSeed(4))
 	female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
 	seen := map[string]bool{}
 	for i := 0; i < 3000; i++ {
-		got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`)
+		got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
 		title, sex, _ := strings.Cut(got, "|")
 		title = strings.TrimSpace(title)
 		seen[title] = true

datatype.go

@@ -3,97 +3,83 @@ package fejkdata
 import (
 	"errors"
 	"fmt"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
-// DataType is what a record column holds, which decides how a record writes its value.
+// DataType is what a record column holds, which decides how a record writes its value. It
-type DataType int
+// prints as data spells it: string, integer, number or boolean.
+type DataType = datatype.DataType
 
 // The datatypes a column declares with "datatype"; a column without one is a string.
 const (
-	DataTypeString DataType = iota
+	DataTypeString  DataType = datatype.String
-	DataTypeInteger
+	DataTypeInteger DataType = datatype.Integer
-	DataTypeNumber
+	DataTypeNumber  DataType = datatype.Number
-	DataTypeBoolean
+	DataTypeBoolean DataType = datatype.Boolean
 )
 
-var (
+// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
-	dataTypeNames = [...]string{"string", "integer", "number", "boolean"}
+func datatypeOf(m map[string]any, pos position) (*DataType, error) {
-	dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
-)
-
-// String is the datatype as data spells it.
-func (d DataType) String() string {
-	if d < 0 || int(d) >= len(dataTypeNames) {
-		return fmt.Sprintf("DataType(%d)", int(d))
-	}
-	return dataTypeNames[d]
-}
-
-// position is where a JSON value sits, which decides whether it may carry a datatype or
-// be null.
-type position int
-
-const (
-	inFormat position = iota // rendered by a format, so neither
-	atTop                    // a category or an inline template, whose fields may be columns
-	inColumn                 // a column, or a choice item standing in for one
-)
-
-// datatypeOf reads a template's "datatype" (default DataTypeString).
-func datatypeOf(m map[string]any, pos position) (DataType, error) {
 	v, ok := m["datatype"]
 	if !ok {
-		return DataTypeString, nil
+		return nil, nil
 	}
 	name, ok := v.(string)
 	if !ok {
-		return 0, fmt.Errorf("datatype must be a string, got %T", v)
+		return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
 	}
-	if name == DataTypeString.String() {
+	for d := DataTypeString; d < datatype.Count; d++ {
-		return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
-	}
-	for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
 		if name != d.String() {
 			continue
 		}
-		if pos != inColumn {
+		if !pos.column() {
-			return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here")
+			return nil, nil
 		}
-		return d, nil
+		return &d, nil
 	}
-	return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name)
+	return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
 }
 
 // checkColumns rejects a record column whose items hold different datatypes, checking a column
 // after the columns its items read, so a column read is named before its readers.
-func checkColumns(s nodeScope) error {
+func checkColumns(s nodeSet) error {
 	checked := map[node]bool{}
-	var check func(path, name string, column node) error
+	var check func(label, name string, column node) error
-	check = func(path, name string, column node) error {
+	check = func(label, name string, column node) error {
 		if checked[column] {
 			return nil
 		}
 		checked[column] = true
 		items, _ := columnItems(column)
 		for _, it := range items {
-			if r := it.readsColumn; r != nil {
+			if r := it.readsColumn; r != nil && r.category != "" {
-				if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil {
+				if err := check(r.category, r.field, r.column); err != nil {
 					return err
 				}
 			}
 		}
-		if _, err := columnDatatype(column); err != nil {
+		if len(items) == 0 {
-			return fmt.Errorf("%s: field %q: %w", path, name, err)
+			return nil
+		}
+		first := columnDatatype(column)
+		for _, it := range items[1:] {
+			if d := itemDatatype(it); d != first {
+				return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
+			}
 		}
 		return nil
 	}
-	return s(func(path string, n node) error {
+	return s(func(label string, n node) error {
 		t, ok := n.(*template)
-		if !ok || !t.record {
+		if !ok || !t.isRecord {
 			return nil
 		}
-		for _, name := range recordColumns(t) {
+		for _, name := range sortedNames(t.fields) {
-			if err := check(path, name, t.fields[name]); err != nil {
+			if err := check(label, name, t.fields[name]); err != nil {
 				return err
 			}
 		}
@@ -101,41 +87,25 @@ func checkColumns(s nodeScope) error {
 	})
 }
 
-// columnDatatype is the datatype a column's items hold. They must agree, since a
+// columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
-// column holds one; a column only ever null is a string.
+// a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
-func columnDatatype(n node) (DataType, error) {
+func columnDatatype(n node) DataType {
 	items, _ := columnItems(n)
 	if len(items) == 0 {
-		return DataTypeString, nil
+		return DataTypeString
 	}
-	first := itemDatatype(items[0])
+	return itemDatatype(items[0])
-	for _, t := range items[1:] {
-		if d := itemDatatype(t); d != first {
-			return first, disagreement(items[0], first, t, d)
-		}
-	}
-	return first, nil
 }
 
-// itemDatatype is the datatype a column item declares, else that of the column it is.
+// itemDatatype is the datatype a column item declares, else that of the column it reads.
 func itemDatatype(t *template) DataType {
-	if t.datatype != DataTypeString {
+	switch {
-		return t.datatype
+	case t.datatype != nil:
+		return *t.datatype
+	case t.readsColumn != nil:
+		return columnDatatype(t.readsColumn.column)
 	}
-	return readDatatype(t)
-}
-
-// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
-// when t reads none.
-func readDatatype(t *template) DataType {
-	if t.readsColumn == nil {
 	return DataTypeString
-	}
-	items, _ := columnItems(t.readsColumn.column)
-	if len(items) == 0 {
-		return DataTypeString
-	}
-	return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
 }
 
 // columnItems is a column's template items, its choices unwrapped, and whether one is null.
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
 			}
 		case *template:
 			items = append(items, n)
-		case *null:
+		case *nullItem:
 			nullable = true
+		case *tableColumn:
+		default:
+			panic(invariant.Broken("columnItems has no case for node %T", n))
 		}
 	}
 	collect(n)
 	return items, nullable
 }
 
-// disagreement names the fix for two items of one column holding different datatypes.
+// itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
-func disagreement(a *template, da DataType, b *template, db DataType) error {
+type itemKind int
-	bare, typed, want := b, a, da
+
-	if da == DataTypeString {
+const (
-		bare, typed, want = a, b, db
+	kindText itemKind = iota
-	}
+	kindReads
+	kindDeclares
+)
+
+func kindOf(t *template) itemKind {
 	switch {
-	case da != DataTypeString && db != DataTypeString:
+	case t.datatype != nil:
-		return bothTyped(a, da, b, db)
+		return kindDeclares
-	case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "":
+	case itemDatatype(t) != DataTypeString:
-		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed))
+		return kindReads
-	case bare.fromString: // an object may carry a weight, which this spelling would drop
-		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
 	}
-	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)
+	return kindText
 }
 
-// bothTyped names the fix for two items holding different datatypes: the one taking its datatype
+// disagreement names the fix for items a and b of one column holding different datatypes, fixing
-// from the column it reads is typed as the other where its values prove it, else read as text.
+// the one that declares least, a when both declare alike.
-func bothTyped(a *template, da DataType, b *template, db DataType) error {
+func disagreement(a *template, da DataType, b *template, db DataType) error {
-	read, other := a, db
+	c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
-	if a.datatype != DataTypeString {
+	if kindOf(b) < kindOf(a) {
-		read, other = b, da
+		c.fix, c.other, c.want = b, a, da
 	}
-	switch {
+	message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
-	case read.datatype != DataTypeString:
+	if !ok {
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
+		panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
-	case (&valueProof{}).columnItem(read).not[other] == "":
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
 	}
-	return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read))
+	return message(c)
+}
+
+// clash is two items of one column holding different datatypes: other holds want; held says
+// what each holds.
+type clash struct {
+	fix, other *template
+	want       DataType
+	held       string
+}
+
+// disagreements is a clash's message by the kinds of its items, fix's first.
+var disagreements = map[[2]itemKind]func(clash) error{
+	{kindDeclares, kindDeclares}: clash.bothDeclare,
+	{kindReads, kindReads}:       clash.retypeRead,
+	{kindReads, kindDeclares}:    clash.retypeRead,
+	{kindText, kindReads}:        clash.textBesideRead,
+	{kindText, kindDeclares}:     clash.declareText,
+}
+
+func (c clash) bothDeclare() error { return errors.New(c.held) }
+
+// fits reports whether fix may hold want.
+func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
+
+func (c clash) retypeRead() error {
+	if c.fits() {
+		return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
+	}
+	return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
+}
+
+func (c clash) textBesideRead() error {
+	if !c.fits() {
+		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
+	}
+	return c.declareText()
+}
+
+func (c clash) declareText() error {
+	if c.fix.fromString { // an object may carry a weight, which this spelling would drop
+		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
+	}
+	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
 }
 
 // typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
 	return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
 }
 
-// asText names the spelling that reads the column a column-read item reads as text, keeping the
+// columnRead is a record's column read by a format that only reads one reference or name,
-// other keys an object item carries.
+// which is the column: it takes the column's datatype and null. category and field name it;
-func asText(t *template) string {
+// category is "" for a column of the reading template's own record, which checkColumns reaches
-	if t.fromString {
+// on its own.
-		return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format)
+type columnRead struct {
-	}
+	a               arm
-	return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format)
+	category, field string
+	column          node
+}
+
+// columnReadOf is the record's column t's format reads, where the format only reads one reference
+// or name, and nil where it does not or t declares a string, which reads the column as text.
+func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
+	ops := t.compiled.ops
+	if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
+		return nil
+	}
+	a := ops[0].arms[0]
+	var start node
+	var category string
+	var tail []string
+	switch b := a.named; {
+	case a.kind == namedRead && b.bindsField():
+		// {f as n}{n}: column f of the record binding n, in t's own category.
+		start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
+	case a.kind == namedRead:
+		// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
+		start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
+		tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
+	case grammar.IsRef(a.head):
+		// {/c.x}: column x of category c.
+		start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
+	default:
+		return nil
+	}
+	if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
+		return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
+	}
+	return nil
 }

datatype_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 )
 
-func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
+func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
 	for _, src := range []string{
 		`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
 		`{"format":"","n":{"format":"42","datatype":"integer"}}`,
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 		`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
 		`{"format":"","pick":[[null,"a"],"b"]}`,
 	} {
-		if _, err := compile(parse(t, src)); err != nil {
+		if _, err := resolved(t, src); err != nil {
 			t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
 		}
 	}
 	for src, want := range map[string]string{
-		`{"format":"","n":{"format":"1","datatype":"int"}}`:                             `datatype takes "integer", "number" or "boolean", got "int"`,
+		`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
-		`{"format":"","n":{"format":"1","datatype":1}}`:                                 "datatype must be a string",
+		`{"format":"","n":{"format":"1","datatype":1}}`:     "datatype must be a string, not a number",
-		`{"format":"{int(1,9)}","datatype":"integer"}`:                                  "datatype only types a record column",
-		`[{"format":"1","datatype":"integer"},"x"]`:                                     "datatype only types a record column",
-		`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
-		`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`:           "datatype only types a record column",
-		`null`: `so write ""`,
-		`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
 		}
 	}
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 	tree := map[string]string{
 		"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
-		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
 	}
 	for _, c := range []struct{ name, column, want string }{
 		{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
 		{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
-		{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
 		{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
 		{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
-		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
-		{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
 		{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
-		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
 		{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
-		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`},
+		{"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
+		{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
+		{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
+		{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
 		{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
 		{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
 		{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
 		{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
 		{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
 		{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
-		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"},
+		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
 		{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
 		{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
 		{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
-		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"},
+		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
 		{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
 		{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
-		{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
-		{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
 	} {
 		row := `{"format":"","col":` + c.column + `}`
 		files := map[string]string{"row": row}
 		for name, body := range tree {
 			files[name] = body
 		}
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
+		_, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
 		if err == nil || !strings.Contains(err.Error(), c.want) {
 			t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
 		}
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 			t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
 		}
 	}
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
+	for column, want := range map[string]string{
+		`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
+		`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`:   "its items hold integer and number; a column holds one datatype",
+	} {
+		row := `{"format":"","col":` + column + `}`
+		if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
+		}
+		if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
+		}
+	}
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
 		"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
 		"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
 	})))
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
 	} {
 		row := `{"format":"","col":` + column + `}`
 		src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
-		f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
+		f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
 		if err != nil {
 			t.Errorf("%s: New = %v, want it loaded", column, err)
 			continue
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
 		t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
 	}
 }
+
+func TestDisagreementRefusesTwoTextItems(t *testing.T) {
+	mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
+}
+
+func TestASignedZeroIsZero(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
+	r, err := f.FakeRecord("row")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
+		t.Errorf("columns = %+v, want each written as its format", c)
+	}
+}
+
+func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{
+		"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
+	}), WithSeed(1))
+	empty := 0
+	for i := 0; i < 50; i++ {
+		r, err := f.FakeRecord("row")
+		if err != nil {
+			t.Fatal(err)
+		}
+		same, text := r.Columns()[0], r.Columns()[1]
+		if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
+			t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
+		}
+		if text.Value == "" {
+			empty++
+		}
+	}
+	if empty == 0 {
+		t.Error(`50 records never read a null as "", want the string column to render one so`)
+	}
+}

doc.go

@@ -0,0 +1,46 @@
+// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
+//
+// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
+// or a folder. A generator loads the modules you name, such as data.Modules(), every
+// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
+// become a dot-path namespace, and you generate values by path:
+//
+//	f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
+//	f.Fake("sv_SE.address")          // "Trekungagatan 43\n442 42 Kungälv"
+//	f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
+//
+// Several sources merge in order, the last winning a name clash, so custom data
+// layers over the built-ins. The JSON template format is documented in the [README].
+//
+// [README]: https://github.com/larvit/fejkdata#readme
+package fejkdata
+
+// Vocabulary
+//
+//   - draw — one item taken from a choice, or one row from a table: `drawItem`,
+//     `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
+//   - expansion — one render of one format: `expand`.
+//   - render — one walk of the tree from an entry point, over one `frameStack`:
+//     `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
+//     start one each, and `FakeStruct` one per record.
+//   - env — what a render reads names and rows through: `renderEnv`.
+//   - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
+//   - named pick — the draw a token such as {/person as p} binds to a name, drawn
+//     on its first read and kept while its scope renders: `nameBinding`,
+//     `namedPick`, `pickFrame`.
+//   - level, pick key, addressed key — `pickKey`.
+//   - memo — what a named pick keeps its draws in, the variant drawn at each level
+//     and the value each path read: `drawMemo`.
+//   - pin — fixing which row of a table one path, or one named pick, uses:
+//     `pinSet`, `namedPick.pins`, `table.followStep`.
+//   - link — a table's tie to its parent table, each row to its parent row:
+//     `table.linkParent`.
+//   - family — a table and every table reaching it through a chain of parent
+//     links.
+//   - resolve — tying a reference or a name to what it names in the assembled tree:
+//     `resolveTemplates`, `resolveRefs`.
+//   - head — the string a read starts from: a field or a name as written, or a
+//     reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
+//     the read's start: `template.startOf`, `nameTarget.start`.
+//   - fence — a load-time check, run over each category, template or struct as it
+//     loads or compiles, so rendering a compiled tree cannot fail.

doc_test.go

@@ -0,0 +1,286 @@
+package fejkdata
+
+import (
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"os"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Every backtick in the Vocabulary is a symbol.
+var backticked = regexp.MustCompile("`([^`\n]+)`")
+
+// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
+func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
+	_, files := sourceFiles(t)
+	declared := declaredSymbols(files)
+	if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
+		t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
+	}
+}
+
+// namesDeclared is every symbol text backticks, failing for each one the package
+// declares nowhere.
+func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
+	t.Helper()
+	var named []string
+	for _, m := range backticked.FindAllStringSubmatch(text, -1) {
+		if !declared[m[1]] {
+			t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
+		}
+		named = append(named, m[1])
+	}
+	return named
+}
+
+// namesAPass fails unless one symbol text backticks is a function: a field names
+// what a pass reads or writes, never the pass.
+func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
+	t.Helper()
+	named := namesDeclared(t, where, text, declared)
+	if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
+		t.Errorf("%s names no function, so it names no pass", where)
+	}
+}
+
+func packageDir(t *testing.T) *build.Package {
+	t.Helper()
+	pkg, err := build.ImportDir(".", 0)
+	if err != nil {
+		t.Fatalf("read the package directory: %v", err)
+	}
+	return pkg
+}
+
+func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	return parseFiles(t, packageDir(t).GoFiles)
+}
+
+func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	fset := token.NewFileSet()
+	var files []*ast.File
+	for _, name := range names {
+		f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
+		if err != nil {
+			t.Fatalf("parse %s: %v", name, err)
+		}
+		files = append(files, f)
+	}
+	return fset, files
+}
+
+func vocabulary(t *testing.T, files []*ast.File) string {
+	t.Helper()
+	var sections strings.Builder
+	for _, f := range files {
+		for _, c := range f.Comments {
+			section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
+			if found && c.Pos() > f.Package {
+				sections.WriteString(section)
+			}
+		}
+	}
+	if sections.Len() == 0 {
+		t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
+	}
+	return sections.String()
+}
+
+// declaredSymbols is every name the package declares at the top level, a method and
+// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
+func declaredSymbols(files []*ast.File) map[string]bool {
+	names := declaredFuncs(files)
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			if d, isGen := decl.(*ast.GenDecl); isGen {
+				addSpecs(names, d)
+			}
+		}
+	}
+	return names
+}
+
+// declaredFuncs is every function the package declares, a method keyed under its
+// type: template.fixedText.
+func declaredFuncs(files []*ast.File) map[string]bool {
+	names := map[string]bool{}
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isFunc := decl.(*ast.FuncDecl)
+			if !isFunc {
+				continue
+			}
+			names[d.Name.Name] = true
+			if recv := receiverType(d.Recv); recv != "" {
+				names[recv+"."+d.Name.Name] = true
+			}
+		}
+	}
+	return names
+}
+
+func addSpecs(names map[string]bool, d *ast.GenDecl) {
+	for _, spec := range d.Specs {
+		switch s := spec.(type) {
+		case *ast.TypeSpec:
+			names[s.Name.Name] = true
+			addFields(names, s)
+		case *ast.ValueSpec:
+			for _, id := range s.Names {
+				names[id.Name] = true
+			}
+		}
+	}
+}
+
+func addFields(names map[string]bool, s *ast.TypeSpec) {
+	st, isStruct := s.Type.(*ast.StructType)
+	if !isStruct {
+		return
+	}
+	for _, field := range st.Fields.List {
+		for _, id := range field.Names {
+			names[s.Name.Name+"."+id.Name] = true
+		}
+	}
+}
+
+func receiverType(recv *ast.FieldList) string {
+	if recv == nil || len(recv.List) == 0 {
+		return ""
+	}
+	e := recv.List[0].Type
+	if star, isPointer := e.(*ast.StarExpr); isPointer {
+		e = star.X
+	}
+	id, isIdent := e.(*ast.Ident)
+	if !isIdent {
+		return ""
+	}
+	return id.Name
+}
+
+const fillHeader = "Filled by "
+
+// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
+// fills, and to the template doc.
+func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
+	fset, files := sourceFiles(t)
+	declared, funcs := declaredSymbols(files), declaredFuncs(files)
+	doc, st, headers := templateStruct(t, fset, files)
+	namesAPass(t, "the template doc", doc, declared, funcs)
+	for _, field := range st.Fields.List {
+		if len(field.Names) == 0 {
+			t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
+		}
+		for _, id := range field.Names {
+			header := headerAbove(headers, id.Pos())
+			if header == "" {
+				t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
+				continue
+			}
+			namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
+		}
+	}
+}
+
+// templateStruct is the template struct, its doc, and the fill headers standing
+// between its fields, in source order.
+func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
+	t.Helper()
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isGen := decl.(*ast.GenDecl)
+			if !isGen {
+				continue
+			}
+			for _, spec := range d.Specs {
+				s, isType := spec.(*ast.TypeSpec)
+				if !isType || s.Name.Name != "template" {
+					continue
+				}
+				st, isStruct := s.Type.(*ast.StructType)
+				if !isStruct {
+					t.Fatal("template is not a struct")
+				}
+				return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
+			}
+		}
+	}
+	t.Fatal("the package declares no template struct")
+	return "", nil, nil
+}
+
+// fillHeaders skips a header trailing a field: it names that field, never the
+// fields below it.
+func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
+	trailing := map[int]bool{}
+	for _, field := range st.Fields.List {
+		trailing[fset.Position(field.End()).Line] = true
+	}
+	var found []*ast.CommentGroup
+	for _, c := range f.Comments {
+		if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
+			continue
+		}
+		if strings.HasPrefix(c.Text(), fillHeader) {
+			found = append(found, c)
+		}
+	}
+	return found
+}
+
+// headerAbove is the last fill header standing above pos.
+func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
+	text := ""
+	for _, c := range headers {
+		if c.Pos() < pos {
+			text = c.Text()
+		}
+	}
+	return text
+}
+
+var (
+	printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
+	printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
+)
+
+// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
+func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
+	for _, name := range []string{"doc.go", "README.md"} {
+		src, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines := printedCall.FindAllString(string(src), -1)
+		if lines == nil {
+			t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
+		}
+		f, err := New(withShipped(), WithSeed(42))
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, line := range lines {
+			c := printedFake.FindStringSubmatch(line)
+			if c == nil {
+				t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
+			}
+			want, err := strconv.Unquote(c[2])
+			if err != nil {
+				t.Fatalf("%s prints %s: %v", name, c[2], err)
+			}
+			if got := fake(t, f, c[1]); got != want {
+				t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
+			}
+		}
+	}
+}

docs/comprehension-history.md

@@ -0,0 +1,722 @@
+# Comprehension history
+
+## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
+- `datatype.go:217` `columnRead`: beside the code that reads it
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
+- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
+- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
+
+## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
+- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
+- `node.go:70` `template.readsColumn`: names the four readers that change together
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
+- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
+
+## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
+- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
+- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
+- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
+- `path.go:119` `compileStep`: six parameters, the one place that reads harder
+
+## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
+- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
+- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
+- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
+- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
+
+## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
+- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
+- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
+- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
+- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
+
+## 2026-10-07, scoring run at ad59967
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
+
+## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
+| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
+| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
+| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
+
+Junior A, hardest first:
+
+- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
+- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
+
+Mid A, hardest first:
+
+- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
+- `node.go:70` `readsColumn`: four functions in three files must change together
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
+- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
+
+Inherited architect, would restructure first:
+
+- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
+- `node.go:52` `template`: filled in four phases by four functions
+
+## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
+- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
+- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
+
+Inherited architect, decided by:
+
+- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
+- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
+- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
+
+## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
+- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
+
+Inherited architect, decided by:
+
+- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
+- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
+- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
+- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
+
+## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
+- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
+- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
+- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
+- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
+- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
+
+## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
+- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
+- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
+- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
+- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
+- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
+- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
+
+## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
+- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
+- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
+- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
+
+Inherited architect, decided by:
+
+- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
+- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
+- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
+
+## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | better | same | better | better |
+| Junior A | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
+- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
+- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
+- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
+- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
+
+Junior A, decided by:
+
+- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
+- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
+- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
+
+## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
+- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
+- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
+- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
+- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
+- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
+- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
+- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
+- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
+- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
+
+## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | worse | worse | better | same | worse |
+| Maintainability senior | worse | same | worse | same | worse |
+| Junior A | same | worse | same | same | same |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
+- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
+- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
+- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
+- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
+- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
+- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
+- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
+
+Maintainability senior, decided by:
+
+- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
+- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
+- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
+- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
+
+Junior A, decided by:
+
+- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
+- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
+
+## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | worse | same | worse |
+| Inherited architect | worse | same | worse | same | worse |
+
+Mid A, decided by:
+
+- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
+- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
+- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
+- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
+
+Inherited architect, decided by:
+
+- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
+- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
+- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
+- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
+- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
+
+## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | worse | worse | same | worse |
+| Inherited architect | worse | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
+- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
+- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
+- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
+- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
+
+## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
+
+Ruling: worse
+
+Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | better | same | same | same |
+| Inherited architect | worse | same | same | same | same |
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6 | 6 | 6 |
+| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
+| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
+- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
+- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
+- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
+
+Inherited architect, decided by:
+
+- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
+- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
+- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
+
+## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | better | better | same | better | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
+- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
+
+Inherited architect, decided by:
+
+- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
+- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
+- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
+- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
+
+## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | same | worse | worse | same | worse |
+| Maintainability senior | same | worse | same | same | worse |
+| Junior A | same | worse | worse | better | worse |
+
+Mid A, decided by:
+
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
+- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
+- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
+- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
+- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
+- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
+- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
+- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
+- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
+
+Junior A, decided by:
+
+- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
+- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
+
+## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
+- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
+- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
+- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
+- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
+- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
+- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
+- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
+
+## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
+- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
+- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
+- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
+- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
+- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
+
+## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | better | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
+- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
+- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
+- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
+- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
+- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
+- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
+
+Inherited architect, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
+- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
+- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
+
+## 2026-10-12T16:30:06Z, PR #216 at 52e0304
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
+| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
+| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
+
+Junior A, hardest first:
+
+- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
+- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
+- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
+- `node.go:65` `template`: filled in four phases by different files, with a change-together set
+- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
+
+Mid A, hardest first:
+
+- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
+- `render.go:146` `readField`: four read paths, and the name undersells it
+- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
+- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
+- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
+- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
+- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
+- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
+- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
+
+Inherited architect, would restructure first:
+
+- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
+- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
+- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
+- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
+- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
+
+## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | better | worse | same | same | better |
+| Inherited architect | better | worse | same | same | same |
+| Maintainability senior | better | worse | same | same | same |
+| Junior A | better | worse | same | better | better |
+
+Mid A, decided by:
+
+- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
+- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
+- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
+
+Inherited architect, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
+- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
+- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
+- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
+
+Maintainability senior, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
+- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
+- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
+
+Junior A, decided by:
+
+- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
+- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
+- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
+- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
+
+## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-14T11:34:34Z, PR #225 at f734c9e, against f90b4d6
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | same |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | worse | same | worse | worse |
+| Junior A | same | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality: `Dockerfile:26` `vulnerability-filing`: the filer's test reads the real todo.md inside docker build ., so a plan edit can turn the Go gate red
+- Locality: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the filer splices todo.md as text, assuming its row shape, the Next ID marker and the Details heading
+- Locality: `.github/workflows/test.yml:106` `release`: a release now also waits on vuln.go.dev's answer, visible only in a needs list
+
+Inherited architect, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the plan is copied into the merge-gate build, so a planning edit that changes the table's shape fails docker build .
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: edits todo.md as text; a missing Next ID crashes with exit 1, which reads like a vulnerability already planned
+- Locality: `.github/workflows/test.yml:69` `vulnerabilities`: whether a release happens depends on vuln.go.dev that day, and every PR shows a red job while one is known
+
+Maintainability senior, decided by:
+
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: finds the release by an unnamed regex group, never checks the Next ID match, and splices Details by a replace that does nothing when the heading is missing
+- Locality, Self-sufficiency: `.github/workflows/test.yml:83` `File each one in todo.md`: one exit code holds the release and triggers the proposal, which must tell filed, planned and crashed apart by a git diff
+- Locality: `Dockerfile:26` `vulnerability-filing`: copies todo.md into the gate build, and the path filter lists todo.md
+- Self-sufficiency: `README.md:1213` "Development": says the build runs the release tooling's tests and leaves out the filer's
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:2` module docstring: builds on rank_todo.py, which the repository does not hold
+
+Junior A, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the Go merge gate copies todo.md and files an item into it, so editing the plan can turn the build red
+- Locality: `.github/workflows/test.yml:106` `release`: whether a push to main releases also depends on vuln.go.dev's answer that day
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the regex and the Details splice encode todo.md's layout, which the script never shows
+- Self-sufficiency: `README.md:1213` "Development": the gate paragraph leaves out the filer's tests while AGENTS.md now defers to it

docs/reading-cost.md

@@ -0,0 +1,637 @@
+# Reading cost
+
+## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
+
+68314 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `compose.yaml` 3
+- `data-import/territory.py` 2
+- `data_test.go` 2
+- `shipped_data_test.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `data-import/tsv.py` 1
+- `data/en_US/phone.json` 1
+
+## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
+
+118930 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 8
+- `README.md` 6
+- `table_test.go` 4
+- `render.go` 3
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `fejkdata.go` 1
+
+## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
+
+93128 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `data_test.go` 4
+- `todo.md` 3
+- `format_stability_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data/en_US/email.json` 1
+- `data/en_US/first-name.tsv` 1
+- `data/en_US/sex.tsv` 1
+- `data/en_US/title.tsv` 1
+
+## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
+
+79628 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `todo.md` 5
+- `format.go` 4
+- `valueproof.go` 4
+- `calc_test.go` 3
+- `datatype_test.go` 2
+- `name.go` 2
+- `name_test.go` 2
+- `record_test.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
+
+366332 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `table_test.go` 17
+- `table.go` 9
+- `struct.go` 8
+- `README.md` 7
+- `docs/decisions.md` 7
+- `record.go` 5
+- `calc.go` 4
+- `data_test.go` 4
+- `reference.go` 4
+- `render.go` 4
+
+## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
+
+705270 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 16
+- `todo.md` 16
+- `README.md` 10
+- `render.go` 5
+- `table.go` 5
+- `format.go` 4
+- `AGENTS.md` 3
+- `calc.go` 3
+- `data.go` 3
+- `fejkdata.go` 3
+
+## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
+
+59846 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `CHANGELOG.md` 2
+- `README.md` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `fejkdata_test.go` 2
+- `go.mod` 2
+- `.dockerignore` 1
+- `builtins_test.go` 1
+- `cmd/fejkdata/main.go` 1
+
+## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
+
+98933 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `format_test.go` 5
+- `todo.md` 5
+- `calc.go` 4
+- `calc_test.go` 4
+- `format.go` 4
+- `path.go` 3
+- `reference.go` 3
+- `README.md` 2
+- `builtins_test.go` 2
+- `inline.go` 2
+
+## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
+
+129910 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `builtins_test.go` 7
+- `todo.md` 5
+- `format.go` 4
+- `calc_test.go` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `reference.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `calc.go` 1
+
+## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
+
+91864 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `table_test.go` 4
+- `data.go` 3
+- `todo.md` 3
+- `graph.go` 2
+- `node.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+
+## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
+
+98023 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `path_test.go` 4
+- `README.md` 3
+- `docs/decisions.md` 3
+- `imports_test.go` 2
+- `path.go` 2
+- `reference_test.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
+
+115125 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `name_test.go` 5
+- `todo.md` 5
+- `shape_test.go` 3
+- `README.md` 2
+- `graph.go` 2
+- `inline.go` 2
+- `node.go` 2
+- `record_test.go` 2
+- `struct.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
+
+53085 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `AGENTS.md` 3
+- `README.md` 3
+- `todo.md` 3
+- `.gitignore` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
+
+67696 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 3
+- `pick_test.go` 2
+- `render.go` 2
+- `table_test.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `data/geo/SE/address.json` 1
+- `data/sv_SE/address.json` 1
+- `docs/decisions.md` 1
+- `pick.go` 1
+
+## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
+
+75959 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 5
+- `README.md` 2
+- `inline.go` 2
+- `namefence.go` 2
+- `path.go` 2
+- `pick_test.go` 2
+- `record.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
+
+129293 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `README.md` 6
+- `format.go` 6
+- `docs/decisions.md` 5
+- `record.go` 5
+- `inline.go` 4
+- `render.go` 4
+- `shipped.go` 4
+- `CHANGELOG.md` 3
+- `data.go` 3
+
+## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
+
+79101 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `format.go` 5
+- `todo.md` 5
+- `path.go` 3
+- `docs/decisions.md` 2
+- `env.go` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
+
+2986167 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 24
+- `AGENTS.md` 18
+- `CHANGELOG.md` 18
+- `data-import/geo-se.py` 17
+- `record.go` 17
+- `cmd/fejkdata/main.go` 16
+- `struct.go` 16
+- `testdata/shipped_shape.txt` 16
+- `.github/workflows/test.yml` 15
+- `arm.go` 15
+
+## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
+
+96842 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `CHANGELOG.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 2
+- `doc.go` 2
+- `docs/decisions.md` 2
+- `path.go` 2
+- `resolve.go` 2
+- `AGENTS.md` 1
+- `arm.go` 1
+
+## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
+
+152553 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 8
+- `data-import/geo-us.py` 5
+- `AGENTS.md` 4
+- `fejkdata.go` 4
+- `graph.go` 4
+- `docs/decisions.md` 3
+- `name.go` 3
+- `pick.go` 3
+- `resolve.go` 3
+
+## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
+
+2852445 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 23
+- `cmd/fejkdata/main.go` 23
+- `pick.go` 18
+- `AGENTS.md` 17
+- `CHANGELOG.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+- `arm.go` 15
+- `fejkdata.go` 15
+- `path.go` 15
+
+## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
+
+93957 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `name.go` 6
+- `README.md` 5
+- `graph.go` 4
+- `env.go` 3
+- `path.go` 3
+- `resolve.go` 3
+- `arm.go` 2
+- `compose.yaml` 2
+- `docs/decisions.md` 2
+
+## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
+
+68500 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `render_test.go` 6
+- `todo.md` 3
+- `README.md` 2
+- `docs/decisions.md` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `doc.go` 1
+- `doc_test.go` 1
+- `node.go` 1
+- `render.go` 1
+
+## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
+
+69756 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 3
+- `docs/decisions.md` 3
+- `CHANGELOG.md` 2
+- `AGENTS.md` 1
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/tsv.py` 1
+
+## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
+
+60588 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 2
+- `docs/decisions.md` 2
+- `record.go` 2
+- `render.go` 2
+- `CHANGELOG.md` 1
+- `docs/comprehension-history.md` 1
+- `inline.go` 1
+- `reference.go` 1
+- `shipped.go` 1
+
+## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
+
+2855972 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 25
+- `CHANGELOG.md` 22
+- `cmd/fejkdata/main.go` 20
+- `node.go` 19
+- `testdata/shipped_shape.txt` 19
+- `AGENTS.md` 18
+- `arm.go` 17
+- `DATA-LICENSES.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+
+## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
+
+340530 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 15
+- `docs/decisions.md` 14
+- `README.md` 4
+- `.github/workflows/test.yml` 2
+- `compose.yaml` 2
+- `fejkdata.go` 2
+- `go.mod` 2
+- `release-tooling/publish_release.py` 2
+- `shipped.go` 2
+- `shippedindex.go` 2
+
+## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
+
+91452 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `todo.md` 5
+- `cmd/fejkdata/main_test.go` 3
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `imports_test.go` 2
+- `.dockerignore` 1
+- `.github/workflows/test.yml` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
+
+112728 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `docs/decisions.md` 5
+- `shipped_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data.go` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+- `internal/datafiles/datafiles.go` 1
+
+## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
+
+200600 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 13
+- `todo.md` 8
+- `CHANGELOG.md` 6
+- `cmd/fejkdata/main.go` 6
+- `docs/decisions.md` 6
+- `shipped_test.go` 6
+- `DATA-LICENSES.md` 5
+- `bench_test.go` 5
+- `data_test.go` 5
+- `data.go` 4
+
+## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
+
+87725 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 4
+- `data_test.go` 4
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/source.py` 1
+
+## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
+
+90566 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `data-import/geo-us.py` 6
+- `README.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 3
+- `compose.yaml` 2
+- `data/geo/US/locality.tsv` 2
+- `data_test.go` 2
+- `.gitignore` 1
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
+
+76079 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 10
+- `docs/decisions.md` 6
+- `README.md` 4
+- `AGENTS.md` 2
+- `data-import/geo-se.py` 2
+- `CHANGELOG.md` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+
+## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
+
+102267 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `CHANGELOG.md` 6
+- `todo.md` 6
+- `cli/cli_test.go` 4
+- `docs/decisions.md` 3
+- `cmd/fejkdata/main_test.go` 2
+- `AGENTS.md` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+- `data/modules.go` 1
+
+## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
+
+372546 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 30
+- `todo.md` 10
+- `cli/cli.go` 5
+- `cli/cli_test.go` 3
+- `CHANGELOG.md` 2
+- `data/sv_SE/.fejkdata.json` 2
+- `docs/decisions.md` 2
+- `internal/datafiles/datafiles.go` 2
+- `record.go` 2
+- `compose.yaml` 1
+
+## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
+
+78327 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 8
+- `README.md` 5
+- `CHANGELOG.md` 1
+- `DATA-LICENSES.md` 1
+- `data/misc/car.tsv` 1
+- `docs/decisions.md` 1
+- `docs/research/research-sources-se.md` 1
+- `docs/research/research-sources-world.md` 1
+
+## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
+
+60817 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `shipped_data_test.go` 2
+- `.dockerignore` 1
+- `.gitignore` 1
+- `data_test.go` 1
+- `docs/decisions.md` 1
+- `release-tooling/publish_release.py` 1
+- `shipped_test.go` 1
+
+## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
+
+78241 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 10
+- `todo.md` 7
+- `CHANGELOG.md` 3
+- `docs/decisions.md` 3
+- `compose.yaml` 1
+- `data/en_US/itin.json` 1
+- `data/misc/timezone.json` 1
+- `data/misc/timezone.tsv` 1
+- `go.mod` 1

docs/research/part1-names-finance-company.md

@@ -0,0 +1,148 @@
+# Part 1 — Names, finance, company: sources and rules
+
+Researched 2026-09-17. "Embed" = ship as a data file in an MIT repo. Everything not marked UNVERIFIED was read from the cited page or file this session.
+
+## 1. US person names and identifiers
+
+### SSA baby names (given names by year and sex)
+- Page: https://www.ssa.gov/oact/babynames/limits.html — national zip https://www.ssa.gov/oact/babynames/names.zip (7 MB), state zip https://www.ssa.gov/oact/babynames/state/namesbystate.zip (23 MB), territories 228 kB.
+- Format (per SSA page + readme as mirrored by https://github.com/hackerb9/ssa-baby-names): one `yobYYYY.txt` per year 1880–2025, CSV `name,sex,count`, name 2–15 chars, sex `M`/`F`, sorted by sex then count desc. Names with < 5 occurrences in a year are excluded.
+- Counts: ~100,364 distinct names, ~2.02 M rows across all years (hackerb9 snapshot; year of snapshot UNVERIFIED). Top 1000 names cover 71.5 % of 2025 births (SSA page).
+- Licence: data.gov catalog entry https://catalog.data.gov/dataset/baby-names-from-social-security-card-applications-national-data lists licence **CC0 1.0** (`https://creativecommons.org/publicdomain/zero/1.0/`), publisher SSA. US federal work, no attribution required; embeddable.
+- Note: ssa.gov returns 403 to non-browser user agents (curl/WebFetch); download with a browser UA or via the data.gov link.
+
+### Census 2010 surnames
+- Page: https://www.census.gov/topics/population/genealogy/data/2010_surnames.html
+- Files: full set https://www2.census.gov/topics/genealogy/2010surnames/names.zip (CSV + XLSX, `Names_2010Census.csv`), top 1000 `Names_2010Census_Top1000.xlsx`, docs https://www2.census.gov/topics/genealogy/2010surnames/surnames.pdf.
+- Fields (from surnames.pdf): `name, rank, count, prop100k, cum_prop100k, pctwhite, pctblack, pctapi, pctaian, pct2prace, pcthispanic`. `(S)` = suppressed percentage. Names are UPPERCASE.
+- Count: 162,253 surnames occurring ≥ 100 times (covers 90.1 % of people with a recorded surname); the CSV also has a trailing `ALL OTHER NAMES` row (UNVERIFIED — www2.census.gov returned 403 to curl this session; download with a browser).
+- Licence: US Census Bureau work → US federal government work, public domain in the US (17 U.S.C. §105). Census only asks for citation (https://www.census.gov/about/policies/citation.html: "U.S. Census Bureau, [Table], [Product], [Vintage], [URL], accessed on [date]"). Embeddable; cite source in the data file header.
+
+### Name prefixes / suffixes
+- No authoritative enumerated list exists. USPS Publication 28 §33 (https://pe.usps.com/text/pub28/28c3_011.htm) defines the fields: Name Prefix, First Name, Middle Name or Initial, Surname, **Suffix Title** = "maturity (e.g., JR, SR) and professional (e.g., PHD, DDS) suffixes"; example "MR. WALTER W. WITHERSPOON JR.".
+- Recommendation: hand-curate. Prefixes `Mr., Mrs., Ms., Mx., Dr., Rev.`; suffixes `Jr., Sr., II, III, IV, PhD, MD, DDS, Esq.`. Rule of thumb: generational suffix (Jr/Sr/II–IV) only with a male name in traditional US usage; never combine Jr. with Sr.; at most one generational suffix.
+
+### SSN
+- Structure: `AAA-GG-SSSS` (area 3, group 2, serial 4). No check digit (Wikipedia).
+- Randomization since **2011-06-25** (https://www.ssa.gov/employer/randomization.html): "Previously unassigned area numbers were introduced for assignment excluding area numbers 000, 666 and 900-999." Area no longer geographic.
+- Invalid per SSA POMS RM 10201.035 (https://secure.ssa.gov/apps10/poms.nsf/lnx/0110201035): area `000`, `666`, `900–999`; group `00`; serial `0000`.
+- Generator rule: area ∈ [001,899] \ {666}; group ∈ [01,99]; serial ∈ [0001,9999].
+- Advertising SSNs (https://www.ssa.gov/history/ssn/misused.html): `078-05-1120` (Woolworth wallet card, 1938; > 40,000 people claimed it) and `219-09-9999` (1940 Social Security Board pamphlet). Both are structurally valid — exclude from output.
+- `987-65-4320`–`987-65-4329` "reserved for advertising": repeated by searchbug.com, ssn-check.org, a Missouri state standard (https://oa.mo.gov/sites/default/files/CC-SocialSecurityNumberNamingStandardV050305.pdf, which even misprints it as 987-54-4329). **UNVERIFIED** — not found on any ssa.gov page or POMS this session; Wikipedia no longer carries it. Safe to use these ten as the library's "obviously fake" examples, but do not cite SSA for it.
+
+### ITIN (IRS)
+- IRS Publication 4757 (Rev. 6-2026), p. 5 (https://www.irs.gov/pub/irs-pdf/p4757.pdf): "All valid ITINs are nine-digit numbers in the same format as the SSN (9XX-8X-XXXX), beginning with a "9" and the 4th and 5th digits ranging from 50 to 65, 70 to 88, 90 to 92, and 94 to 99."
+- Group `93` = ATIN (adoption TIN); `89` reserved (Intuit/TaxSlayer support pages, not IRS — UNVERIFIED).
+- Generator rule: `9` + 2 digits + group ∈ {50–65, 70–88, 90–92, 94–99} + 4 digits (serial `0000` not documented as invalid for ITIN; avoid it anyway).
+
+### EIN (IRS)
+- Format `NN-NNNNNNN`. Page: https://www.irs.gov/businesses/small-businesses-self-employed/how-eins-are-assigned-and-valid-ein-prefixes ("The first two digits of your EIN show the IRS campus that assigned it or if you applied online").
+- Valid prefixes (union of the table, 2026-09-17):
+  `01 02 03 04 05 06 10 11 12 13 14 15 16 20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 71 72 73 74 75 76 77 80 81 82 83 84 85 86 87 88 90 91 92 93 94 95 98 99`
+  By campus: Andover 10,12 · Atlanta 60,67 · Austin 50,53 · Brookhaven 01–06,11,13,14,16,21–23,25,34,51,52,54–59,65 · Cincinnati 30,32,35–38,61 · Fresno 15,24 · Kansas City 40,44 · Memphis 94,95 · Ogden 80,90 · Philadelphia 33,39,41–43,46,48,62–64,66,68,71–77,85–88,91–93,98,99 · Internet 20,26,27,33,39,41,42,45–47,81–88,92,93,99 · SBA 31.
+- Not valid: 00, 07, 08, 09, 17, 18, 19, 28, 29, 49, 69, 70, 78, 79, 89, 96, 97. No check digit.
+
+## 2. Finance
+
+### ABA routing number (RTN)
+- Wikipedia https://en.wikipedia.org/wiki/ABA_routing_transit_number: 9 digits; check: `(3(d1+d4+d7) + 7(d2+d5+d8) + (d3+d6+d9)) mod 10 = 0`.
+- First two digits: `00` US Government; `01–12` Federal Reserve districts (normal banks); `21–32` thrifts (historic, still valid); `61–72` non-bank payment processors/clearinghouses; `80` traveler's checks. Other prefixes unused (Wikipedia does not say so explicitly — inferred).
+- Generator rule: prefix ∈ {01–12, 21–32, 61–72}, 6 random digits, compute 9th as `(10 − (3d1+7d2+d3+3d4+7d5+d6+3d7+7d8) mod 10) mod 10`.
+- Open list of real RTNs: the Federal Reserve E-Payments Routing Directory (https://www.frbservices.org/resources/routing-number-directory/index.html) says "may not be sold, re-licensed, or otherwise used for commercial gain" → **not embeddable**. Generate synthetically.
+
+### Payment card numbers (Wikipedia "Payment card number", https://en.wikipedia.org/wiki/Payment_card_number; all Luhn-checked)
+| Network | IIN ranges | Length | CVV |
+|---|---|---|---|
+| Visa | 4 | 13, 16, 19 | 3 |
+| Mastercard | 51–55, 2221–2720 | 16 | 3 |
+| American Express | 34, 37 | 15 | 4 (front) |
+| Discover | 6011, 644–649, 65, 622126–622925 (UnionPay co-brand) | 16–19 | 3 |
+| JCB | 3528–3589 | 16–19 | 3 |
+| Diners Club International | 30 (covers 300–305, 3095), 36, 38, 39 | 14–19 | 3 |
+| Diners Club US & Canada | 55 (Mastercard co-brand) | 16 | 3 |
+| UnionPay | 62 | 16–19 | 3 |
+| Maestro | 5018, 5020, 5038, 5893, 6304, 6759, 6761, 6762, 6763 | 12–19 | 3 |
+| Maestro UK | 6759, 676770, 676774 | 12–19 | 3 |
+| Dankort 5019 · Mir 2200–2204 · RuPay 60,65,81,82,508 · Troy 65,9792 · UATP 1 (15) · Verve 506099–506198, 507865–507964, 650002–650027 (16/18/19) | | | |
+- CVV lengths: https://en.wikipedia.org/wiki/Card_security_code ("three-digit ... Visa, Mastercard, and Discover"; "American Express is a four-digit code on the front").
+- The requested "Maestro 50/56–58/6xxx" is a looser legacy rule; use Wikipedia's explicit IINs.
+- Wikipedia text is CC BY-SA 4.0 — copy the *facts* (not prose) into your table; facts are not copyrightable.
+
+### Published test cards
+- Stripe: https://docs.stripe.com/testing — Visa 4242424242424242, Mastercard 5555555555554444, Amex 378282246310005, Discover 6011111111111117, Diners 3056930009020004 (14), JCB 3566002020360505, UnionPay 6200000000000005; any future expiry, any CVC (4 for Amex).
+- PayPal: https://developer.paypal.com/tools/sandbox/card-testing/ — Visa 4005519200000004, 4012000033330026, 4012000077777777, 4012888888881881, 4217651111111119, 4500600000000061, 4772129056533503, 4915805038587737; Mastercard 2223000048400011; Amex 371449635398431, 376680816376961; Diners 36461510000039, 36461510000013; Maestro 6304000000000000, 5063516945005047; JCB 3636500000000260, 3636500000000989; CUP 6200680000000004, 6200680000000038.
+- Reuse: neither page carries an open licence (Stripe docs are under Stripe's site terms). The numbers themselves are Luhn-valid facts, widely reproduced; embedding the numbers with a "from Stripe/PayPal test docs" note is low-risk but **UNVERIFIED** as an explicit grant. Alternative: generate Luhn-valid numbers from the IIN table above — no dependency on either vendor.
+
+### BIC / SWIFT (ISO 9362)
+- https://en.wikipedia.org/wiki/ISO_9362: 4 letters bank code + 2 letters ISO 3166-1 country + 2 alphanumeric location + optional 3 alphanumeric branch (8 or 11 chars; `XXX` = primary office). Location second char `0` = test BIC, `1` = passive participant, `2` = reverse billing.
+- Generator rule: `[A-Z]{4}[A-Z]{2}[A-Z2-9][A-NP-Z0-9]([A-Z0-9]{3})?` — avoid `0`/`1` as the 8th char for a "live" BIC.
+- SWIFT's BIC Directory is proprietary (SwiftRef licence, redistribution needs a separate "SwiftRef Redistribution License": https://www.swift.com/myswift/ordering/order-products-services/swiftref-redistribution-license) → **not embeddable**.
+- Open alternative: GLEIF BIC-to-LEI mapping, https://www.gleif.org/en/lei-data/lei-mapping/download-bic-to-lei-relationship-files, files at https://mapping.gleif.org/api/v2/bic-lei/ (monthly zip; `LEI-BIC-20260828.zip` → `lei-bic-20260828T000000.csv`, columns `LEI,BIC`, 39,347 rows). Licence: "BIC/LEI Mapping Table License Agreement" (https://www.gleif.org/lei-data/lei-mapping/download-bic-to-lei-relationship-files/2017-12-21_annex-2_bic-to-lei-mapping-table-license-agreement_final.pdf) — a CC0-style grant "for any purpose whatsoever, including ... commercial" but **conditional on reproducing the notice** "SWIFT © and database rights [month year]. All rights reserved. This Mapping Table has been developed by SWIFT. Any use of the Mapping Table ... is subject to the BIC/LEI Mapping Table License Agreement ... For the latest BIC information and updates, always refer to www.swift.com/bic." Embeddable in an MIT repo with that notice in the data file; it is a list of real live BICs (bank names not included — join to GLEIF LEI data if names are wanted).
+
+### ISIN (https://en.wikipedia.org/wiki/International_Securities_Identification_Number)
+- 12 chars: 2-letter country + 9 alphanumeric NSIN + 1 check digit.
+- Check: convert letters A=10…Z=35 (ASCII − 55) to produce a digit string, then Luhn over that string (double every second digit from the right, sum digits, check makes total ≡ 0 mod 10). Pitfall: work on the *expanded* digit string; a transposed letter pair can pass.
+
+### CUSIP (https://en.wikipedia.org/wiki/CUSIP)
+- 9 chars: 6 issuer + 2 issue + 1 check. Values: digits as-is, A=10…Z=35, `*`=36, `@`=37, `#`=38.
+- Check: for i in 1..8, v = value; if i even, v ×= 2; sum += v div 10 + v mod 10; check = (10 − sum mod 10) mod 10.
+- Real CUSIPs are proprietary (CGS/FactSet); generate synthetic ones only.
+
+### IBAN (ISO 13616)
+- Validation (https://en.wikipedia.org/wiki/International_Bank_Account_Number): move first 4 chars to end, letters → 10–35, integer mod 97 must equal 1; max 34 chars; check digits 02–98.
+- SWIFT registry: https://www.swift.com/standards/data-standards/iban-international-bank-account-number, PDF release 101 https://www.swift.com/sites/default/files/files/iban-registry-v101.pdf (release 100 was Oct 2025); TXT also published (URL **UNVERIFIED** — swift.com returns 403/errors to automation this session). Terms: free of charge; no explicit licence text found (**UNVERIFIED**). Country count: 89 countries per Wikipedia (Dec 2024).
+- Embeddable per-country length tables already extracted from the registry:
+  - php-iban `registry.txt` (https://github.com/globalcitizen/php-iban, **LGPL-3.0**): pipe-separated, 121 rows (116 official + unofficial), columns `country_code|country_name|domestic_example|bban_example|bban_format_swift|bban_format_regex|bban_length|iban_example|iban_format_swift|iban_format_regex|iban_length|bban_bankid_start_offset|...|country_sepa|swift_official|...|currency_iso4217|central_bank_url|central_bank_name|membership`. LGPL data in an MIT repo is awkward; use it as a *reference* to build your own table (formats are facts) rather than copying the file.
+  - ibankit-js (https://github.com/koblas/ibankit-js, **Apache-2.0**, registry v95) — same caveat.
+  - Best route: own table `{country, length, bban_format}` derived from the SWIFT PDF (facts), cite SWIFT.
+
+### Currencies (ISO 4217)
+- ISO says use is free: https://www.iso.org/iso-4217-currency-codes.html — "ISO allows free-of-charge use of its country, currency and language codes from ISO 3166, ISO 4217 and ISO 639". Official list from SIX: https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml (+ `.xls`, `list-three` historic). Fields `CtryNm, CcyNm, Ccy, CcyNbr, CcyMnrUnts`; published 2026-01-01; 280 entity rows, 178 distinct codes. No symbols. No licence text inside the XML.
+- Alternatives:
+  - datasets/currency-codes (https://github.com/datasets/currency-codes): **PDDL** (public domain); `data/codes-all.csv` fields `Entity, Currency, AlphabeticCode, NumericCode, MinorUnit, WithdrawalDate`; built from SIX list one + three; no symbols. Embeddable.
+  - npm `currency-codes` v2.2.0 (https://github.com/freeall/currency-codes): **MIT**; fields `code, number, digits, currency, countries[]`; generated from the SIX XML; no symbols. Embeddable.
+  - umpirsky/currency-list (https://github.com/umpirsky/currency-list): **MIT**; code → localised name only (311 entries in `data/en_US/currency.json`, includes historic); no symbols, no numeric codes, no decimals.
+  - Debian iso-codes (https://salsa.debian.org/iso-codes-team/iso-codes): **LGPL-2.1+**; `data/iso_4217.json` has `alpha_3, name, numeric` only (179 entries); no symbols/minor units. LGPL → avoid embedding.
+  - Symbols: none of the above carry them. Unicode CLDR (`common/main/en.xml` currency symbols, Unicode License, MIT-compatible with notice) is the usual source — **UNVERIFIED this session**.
+
+### Crypto addresses
+- Bitcoin (https://en.bitcoin.it/wiki/List_of_address_prefixes): P2PKH version 0x00 → leading `1`; P2SH 0x05 → leading `3`; Base58Check 25–34 chars (alphabet excludes `0OIl`; last 4 bytes = first 4 of double-SHA256 of version+payload). Testnet `m`/`n`, `2`, `tb1`.
+- Bech32 (BIP-173, https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki, BSD-2-Clause): HRP `bc`, separator `1`, charset `qpzry9x8gf2tvdw0s3jn54khce6mua7l`, all-lowercase (or all-uppercase), max 90 chars; P2WPKH `bc1q…` = 42 chars, P2WSH `bc1q…` = 62 chars.
+- Bech32m (BIP-350, https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): witness v1+ (P2TR `bc1p…`, 62 chars), checksum constant `0x2bc830a3` instead of 1.
+- Ethereum EIP-55 (https://eips.ethereum.org/EIPS/eip-55, CC0): `0x` + 40 hex; keccak256 of the lowercase hex (no 0x, as ASCII); for each hex letter, uppercase iff the corresponding hash nibble ≥ 8 (bit 4·i set). Digits unchanged.
+
+## 3. Company
+
+### NAICS 2022 (US Census)
+- Page https://www.census.gov/naics/ ; files: 6-digit list https://www.census.gov/naics/2022NAICS/6-digit_2022_Codes.xlsx (82 kB; **1,012** six-digit codes, 111110…928120, verified by parsing), 2–6 digit https://www.census.gov/naics/2022NAICS/2-6%20digit_2022_Codes.xlsx, structure https://www.census.gov/naics/2022NAICS/2022_NAICS_Structure.xlsx, descriptions https://www.census.gov/naics/2022NAICS/2022_NAICS_Descriptions.xlsx, manual PDF https://www.census.gov/naics/reference_files_tools/2022_NAICS_Manual.pdf.
+- Format: xlsx, column A code (numeric), column B title (trailing spaces in some titles — trim).
+- Licence: no statement on the page or in the manual front matter; US federal government work → public domain in the US (17 U.S.C. §105). Embeddable; cite Census/OMB.
+
+### SEC EDGAR company names
+- `https://www.sec.gov/files/company_tickers.json`: object keyed `"0".."N"` of `{cik_str, ticker, title}`; **10,422** entries (2026-09-17). `company_tickers_exchange.json`: `{"fields":["cik","name","ticker","exchange"],"data":[...]}`.
+- `https://www.sec.gov/Archives/edgar/cik-lookup-data.txt`: 40 MB, **1,059,372** lines, `NAME:CIK:` (`COMPANY NAME:0001234567:`), all filers incl. individuals — filter to the ticker list for company names.
+- Access: declare `User-Agent: Company Name email@domain`, ≤ 10 req/s (https://www.sec.gov/os/webmaster-faq). Licence: "All Government-created content on sec.gov and EDGAR public filing content are free to access and reuse" (same FAQ). Embeddable; the ticker file (~800 kB) is the practical one.
+
+### DUNS
+- https://en.wikipedia.org/wiki/Data_Universal_Numbering_System: 9 digits, no significance; shown `NN-NNN-NNNN` or plain. Had a mod-10 check digit until ~Dec 2006; **dropped** since (expanded the pool by 800 M). So: any 9 digits are structurally valid today; no validation rule to enforce. Optional DUNS+4 suffix (4 alphanumerics) is user-assigned.
+
+### Legal entity suffixes per country
+- GLEIF ISO 20275 Entity Legal Forms code list v1.6 (2026-02-19): page https://www.gleif.org/en/lei-data/code-lists/iso-20275-entity-legal-forms-code-list ; CSV https://www.gleif.org/lei-data/code-lists/iso-20275-entity-legal-forms-code-list/2026-02-19-elf-code-list-v1.6.csv (758 kB; xlsx alongside).
+- Columns: `ELF Code, Country of formation, Country Code (ISO 3166-1), Jurisdiction of formation, Country sub-division code (ISO 3166-2), Entity Legal Form name Local name, Language, Language Code (ISO 639-1), Entity Legal Form name Transliterated name, Abbreviations Local language, Abbreviations transliterated, Date created, ELF Status ACTV/INAC, Modification, Modification date, Reason`.
+- Counts: 4,002 rows; 130 countries; 3,790 ACTV / 212 INAC; 1,518 rows have a local abbreviation (e.g. US `N.A.`, `FSA`; SE `AB`… multiple abbreviations separated by `;`). US has 737 rows (per-state forms). Abbreviations are the field you want for `Inc`, `LLC`, `GmbH`, `AB`, `SAS`, `Pty Ltd`; many rows have none, so filter on non-empty abbreviation and status ACTV.
+- Licence: GLEIF Open Data page (https://www.gleif.org/en/about/open-data): "The data on GLEIF's website is provided under a Creative Commons (CC0) license"; LEI Data Terms of Use: "provided under the CC0 licence, see CC0 1.0 Universal". The ELF page itself names no licence — that CC0 explicitly covers this code list is **UNVERIFIED**, but GLEIF treats all its published data this way. Embeddable.
+
+## Quick embed verdict
+| Item | Embed? | Licence / attribution |
+|---|---|---|
+| SSA baby names | yes | CC0 (data.gov) |
+| Census 2010 surnames | yes | US gov PD; cite Census |
+| EIN prefixes, ITIN/SSN rules | yes (rules) | facts from IRS/SSA |
+| NAICS 2022 | yes | US gov PD |
+| SEC company_tickers.json | yes | free to reuse (SEC) |
+| GLEIF ELF code list | yes | CC0 (GLEIF) |
+| GLEIF BIC-LEI (real BICs) | yes, with mandatory SWIFT notice | custom permissive licence |
+| ISO 4217 (SIX XML / datasets PDDL / npm MIT) | yes | free use (ISO), PDDL, MIT |
+| Fed routing directory, SwiftRef BIC directory, CUSIP lists | no | restricted/proprietary — generate synthetically |
+| php-iban / ibankit registries | copy facts only | LGPL-3.0 / Apache-2.0 |
+| Stripe / PayPal test cards | numbers only, note source | no explicit grant (UNVERIFIED) |

docs/research/part2-internet-vehicle-airline.md

@@ -0,0 +1,118 @@
+# Open data sources: Internet, Vehicle, Airline (researched 2026-09-17)
+
+Verdict key: **EMBED** = safe to vendor into an MIT repo; **EMBED+ATTR** = ok with a notice; **AVOID** = licence incompatible or unclear.
+
+## 4. Internet
+
+### IANA registries (all of them)
+- Licence: IANA/IETF statement at https://www.iana.org/help/licensing-terms — "IANA and IETF intend that the Protocol Registries may be freely used by any party for any purpose", data dedicated under **CC0 1.0**. No attribution required. **EMBED**.
+- All registries below share this licence; ship one `SOURCES`/NOTICE line pointing at that URL.
+
+| Registry | CSV URL | Rows (2026-09-16) | Notes |
+|---|---|---|---|
+| Media types | `https://www.iana.org/assignments/media-types/<type>.csv`, type ∈ application, audio, font, haptics, image, message, model, multipart, text, video (also `example`) | application 1800, audio 165, font 6, haptics 3, image 88, message 27, model 42, multipart 17, text 105, video 97 (header excluded) ≈ **2350** | Fields: `Name,Template,Reference`. Template is the full `type/subtype`; some rows are DEPRECATED/OBSOLETED in Name (filter). Format: RFC 6838 §4.2 — `type "/" subtype`, restricted chars, max 127 chars each, case-insensitive. |
+| HTTP status codes | https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv | 75 rows, **64 assigned** (306 and 418 are "(Unused)"; 104 is TEMPORARY) → 62 usable | Fields: `Value,Description,Reference`. Value is 3 digits 1xx–5xx. |
+| IPv4 special-purpose | https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry-1.csv | 26 blocks | Fields: `Address Block,Name,RFC,Allocation Date,Termination Date,Source,Destination,Forwardable,Globally Reachable,Reserved-by-Protocol`. |
+| IPv6 special-purpose | https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry-1.csv | 27 blocks | Same fields. Contains `2001:db8::/32` and `3fff::/20`. |
+| Service names / ports | https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv | 14,534 rows; 11,731 with both name and a single port; **6,102 distinct named ports**; 6,008 tcp-named; 707 tcp names < 1024 | Fields: `Service Name,Port Number,Transport Protocol,Description,Assignee,Contact,Registration Date,Modification Date,Reference,Service Code,Unauthorized Use Reported,Assignment Notes`. Port Number may be empty, single, or a range `a-b`. Well-known = 0–1023, registered 1024–49151, dynamic 49152–65535 (RFC 6335). ~1.1 MB; embed only the named-port subset. |
+| TLDs | https://data.iana.org/TLD/tlds-alpha-by-domain.txt | **1,401** TLDs (first line is a `# Version …` comment) | Upper-case ASCII; IDNs as `XN--…` punycode. Same IANA CC0 terms (UNVERIFIED that data.iana.org is explicitly covered by the licensing page; it is the same registry data). Lower-case on use. |
+
+### MIME extension mapping: jshttp/mime-db
+- https://github.com/jshttp/mime-db — **MIT** (GitHub API spdx MIT; npm `license: MIT`), version 1.54.0 on npm.
+- `db.json` (CDN: https://cdn.jsdelivr.net/npm/mime-db/db.json): object keyed by lower-case type → `{source, extensions[], compressible, charset}`.
+- Counts (1.54.0): **2,522 types**, **1,015 with extensions**; source = iana 2,136, apache 275, nginx 13, custom 98.
+- Sources: IANA registry (CC0), Apache httpd `mime.types` (Apache-2.0), nginx `mime.types` (BSD-2). **EMBED+ATTR** (keep the MIT copyright notice). Data updates are not semver-breaking per README.
+
+### IPv4 / IPv6 generation rules (RFCs, no data file needed)
+- RFC 5737 documentation: `192.0.2.0/24` (TEST-NET-1), `198.51.100.0/24` (TEST-NET-2), `203.0.113.0/24` (TEST-NET-3). https://www.rfc-editor.org/rfc/rfc5737.html
+- RFC 1918 private: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`. Also useful: `100.64.0.0/10` CGNAT (RFC 6598), `198.18.0.0/15` benchmarking (RFC 2544), `169.254.0.0/16` link-local.
+- IPv6 documentation: `2001:db8::/32` (RFC 3849) and `3fff::/20` (RFC 9637, "expands on the existing 2001:db8::/32 … with the reservation of an additional, larger prefix"). ULA `fc00::/7` (use `fd00::/8`), link-local `fe80::/10`.
+- Validation: generate inside those blocks; avoid network/broadcast (`.0`/`.255`) if the consumer expects host addresses. Format IPv6 per RFC 5952 (lower-case hex, `::` once, longest zero run).
+
+### MAC OUI (IEEE MA-L)
+- CSV: https://standards-oui.ieee.org/oui/oui.csv (3.8 MB, **40,161 rows**, fields `Registry,Assignment,Organization Name,Organization Address`; Assignment is 6 hex chars). Server returns HTTP 418 to non-browser user agents; fetch with a browser UA. Also MA-M `oui28.csv`, MA-S `oui36.csv`, CID `cid.csv`.
+- Licence: no licence on the IEEE pages themselves (site footer "© IEEE – All rights reserved" is generic). Debian's `ieee-data` copyright file (https://metadata.ftp-master.debian.org/changelogs/main/i/ieee-data/unstable_copyright) records IEEE's 2014 statement: *"IEEE does not assert any copyright in the OUI Public Listing or attempt to restrict distribution of the listing in any way. The IEEE Registration Authority does, however, strongly encourage those who use the list to obtain it directly from IEEE…"* — same text in the FSF directory. Debian ships it in `main` on that basis. **EMBED+ATTR** (cite the statement); the org-address column is not needed — keep `Assignment,Organization Name` only (~1 MB → few hundred KB). UNVERIFIED: no current IEEE page restates it (regauth FAQ and IPR pages do not mention it).
+- Alternative needing no data: locally administered unicast MAC — first octet's bit 1 (U/L) = 1, bit 0 (I/G) = 0 → second hex digit ∈ {2, 6, A, E}, e.g. `x2:xx:xx:xx:xx:xx`. Guaranteed never to collide with a vendor OUI. Notations: `01:23:45:67:89:ab`, `01-23-45-67-89-AB`, Cisco `0123.4567.89ab`.
+
+### User-agent strings
+| Source | Licence | Verdict |
+|---|---|---|
+| https://www.useragents.me/ | No licence stated anywhere on the page ("© 2022–2025 Useragents.me"); `/api` returns 404 now; JSON blobs embedded in page, weekly updates | **AVOID** (unlicensed = all rights reserved) |
+| https://explore.whatismybrowser.com/useragents/explore/ + API legal https://developers.whatismybrowser.com/api/about/legal/ | Proprietary: "You may NOT share the unique database download URL, or the downloaded file itself … EXCEPT for making it publicly available on the internet" | **AVOID** |
+| npm `user-agents` (intoli) https://github.com/intoli/user-agents | **BSD-2-Clause**, v2.1.185, daily-rebuilt dataset from Intoli traffic, weighted by frequency; fields `userAgent, platform, vendor, appName, deviceCategory, screenWidth/Height, viewportWidth/Height, connection{…}, oscpu, cpuClass, pluginsLength` | **EMBED+ATTR** (keep BSD notice). Count not stated on the page (UNVERIFIED; dataset is a few thousand rows). |
+| npm `top-user-agents` (microlinkhq) https://github.com/microlinkhq/top-user-agents | **MIT** (GitHub API). Top 100 UA strings from microlink.io traffic, weekly; JSON: https://cdn.jsdelivr.net/gh/microlinkhq/top-user-agents@master/src/index.json (+ desktop.json, mobile.json) | **EMBED+ATTR** — smallest clean option |
+- None of the above is CC0. Alternative: generate UAs from templates (Chrome/Firefox/Safari/Edge strings are fixed-form; only version numbers vary), no data needed.
+
+### Programming languages: GitHub Linguist
+- https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml — **MIT** (repo LICENSE, "Copyright (c) 2017 GitHub, Inc."). **EMBED+ATTR**.
+- **835** top-level entries; **562** with `type: programming` (others: data, markup, prose). Per-entry fields: `type, color, extensions, aliases, tm_scope, ace_mode, language_id, group, interpreters, …`.
+
+### Operating systems
+- No open dataset found beyond awesome-lists. OS names/versions are facts — hand-author a small table (Windows 10/11, Windows Server, macOS versions+names, Ubuntu/Debian/Fedora/Arch/Alpine, iOS, Android, FreeBSD…). `user-agents` (BSD-2) carries `platform`/`oscpu` values if a data-driven list is wanted.
+
+### Database engines / collations
+- Engine names: hand-author (facts).
+- MySQL manual (https://dev.mysql.com/doc/refman/8.4/en/preface.html): Oracle terms — "you may not … copy, reproduce … distribute … any part" except distributing the docs with the software. **AVOID copying tables from MySQL docs.** Instead dump `SHOW COLLATION` from a `mysql:8.4.x` container (server output is factual, not the manual) — ~286 collations / 41 charsets in 8.x (UNVERIFIED count). PostgreSQL docs are under the PostgreSQL licence (permissive, keep notice) — `pg_collation` dump or docs both fine. MariaDB KB is CC BY-SA — avoid.
+
+## 5. Vehicle
+
+### NHTSA vPIC
+- API base `https://vpic.nhtsa.dot.gov/api/vehicles/`, `?format=json|xml|csv`. Endpoints verified: `GetAllMakes` (**12,363 makes**, fields `Make_ID,Make_Name`; 612 KB JSON, mostly small trailer/custom builders — filter with `GetMakesForVehicleType/car` → 195 makes, `/truck` → 207), `GetModelsForMake/honda` (362 models, `Make_ID,Make_Name,Model_ID,Model_Name`), `DecodeWMI/1HG`, `GetWMIsForManufacturer/honda` (45 WMIs, fields `WMI,Name,Country,VehicleType,Id,…`), `GetVehicleVariableList`, `GetVehicleVariableValuesList/<display name>` (must be the display name, URL-encoded, e.g. `Fuel%20Type%20-%20Primary`; the camel-case name returns 0 rows).
+- Value lists (counts): **Fuel Type - Primary 14**, **Body Class 71**, **Transmission Style 12**, **Drive Type 23**, **Vehicle Type 9**. Fields `Id,Name`.
+- Bulk: https://vpic.nhtsa.dot.gov/downloads/ — `vPICList_lite_2026_08.bak.zip` (MS SQL Server 2019+ backup, 190 MB, updated 2026-08-14), also `.plain.zip` / `.custom.zip`. Page says standalone DB "limited to VIN decoding"; makes/models/variables still via API. Contains `Wmi`, `Make`, `Model`, `WMIYearValidChars` tables (UNVERIFIED table names).
+- Licence: FAQ https://vpic.nhtsa.dot.gov/api/home/index/faq — "No, NHTSA is a government agency and the services provided on the API are free for use by the public as an offering as a part of our Open Data initiatives"; no rate-limit number stated, batch jobs asked to run at night. Work of the US federal government → public domain under 17 USC §105; data.gov entry lists licence as "unknown". **EMBED** (note source; no attribution legally required).
+
+### VIN (ISO 3779 / 49 CFR 565, North America)
+- 17 chars from `0-9 A-H J-N P R-Z` (**I, O, Q never**). Positions: 1–3 WMI, 4–8 VDS, 9 check digit, 10 model year, 11 plant, 12–17 serial (last 4 must be digits for US, positions 12–14 alphanumeric — 49 CFR 565.15).
+- Check digit: transliterate `A=1 B=2 C=3 D=4 E=5 F=6 G=7 H=8 J=1 K=2 L=3 M=4 N=5 P=7 R=9 S=2 T=3 U=4 V=5 W=6 X=7 Y=8 Z=9`, digits as-is; weights `8 7 6 5 4 3 2 10 0 9 8 7 6 5 4 3 2`; sum mod 11 → digit, remainder 10 → `X`.
+- Model year (pos 10): excludes `I O Q U Z 0`; `A=2010 … H=2017 J=2018 K=2019 L=2020 M=2021 N=2022 P=2023 R=2024 S=2025 T=2026 V=2027 W=2028 X=2029 Y=2030`, `1=2031…9=2039` (30-year cycle; `1–9` also = 2001–2009). Source: https://en.wikipedia.org/wiki/Vehicle_identification_number (rules are facts; do not copy prose).
+
+### WMI list
+- SAE J1044 / SAE WMI database is paid (https://www.sae.org/standards/j1044_202501-world-manufacturer-identifier) — **AVOID**.
+- Wikipedia "List of WMIs" — CC BY-SA — **AVOID**.
+- Use vPIC: `GetWMIsForManufacturer/<name or id>` per manufacturer, or the `Wmi` table in the standalone `.bak`. Public domain. Country prefix rule (first char): `1,4,5` USA, `2` Canada, `3` Mexico, `J` Japan, `K` Korea, `S` UK, `W` Germany, `Y` Sweden/Finland, `Z` Italy, `L` China (ISO 3780 facts).
+
+### US licence plate formats
+| Source | Licence | Verdict |
+|---|---|---|
+| openalpr `runtime_data/postprocess/us.patterns` (https://github.com/openalpr/openalpr) | **AGPL-3.0** | **AVOID** (291 lines, `@`=letter `#`=digit — useful only as a reference to check hand-authored patterns) |
+| Wikipedia "United States license plate designs and serial formats" | CC BY-SA | **AVOID copying**; formats themselves are facts |
+| jonnii/platekit | MIT, but SVG rendering, no serial patterns | not useful |
+- No MIT/CC0 per-state pattern list found. Recommendation: hand-author a per-state table (facts); current standard passenger formats as listed by Wikipedia (verify against DMV pages before use): AL `0AXXXXX`/`00AXXXX`, AK `ABC 123`, AZ `XXX 1XX`, AR `ABC 12D`, CA `1ABC123` (digit-3 letters-3 digits), CO `ABC-D12`, CT `AB·12345`, DE `123456`, DC `AB-1234`, FL `ABC D12`, GA `ABC1234`, HI `ABC 123`, ID `A 1234U`, IL `AB 12345`, IN `123ABC`, IA `ABC 123`, KS `1234ABC`, KY `ABC123`, LA `123 ABC`, ME `123·ABC`, MD `1AB2345`, MA `1ABC 23`, MI `ABC 1234`, MN `ABC-123`, MS `ABC 123`, MO `AB1 C2D`, MT `0-AB1234`, NE `ABC 123`, NV `123·A45`, NH `123 4567`, NJ `D12-ABC`, NM `123-ABC`, NY `ABC-1234`, NC `ABC-1234`, ND `123 ABC`, OH `ABC 1234`, OK `ABC-123`, OR `123 ABC`, PA `ABC1234`, RI `1AB 234`, SC `123ABC`, SD `0A1 234`, TN `ABC 1234`, TX `ABC-1234`, UT `A12 3BC`, VT `ABC 123`, VA `ABC-1234`, WA `ABC1234`, WV `X1A 2345`, WI `ABC-1234`, WY `1A-123A`. Most states omit I/O/Q from serials (state-specific, UNVERIFIED per state).
+
+## 6. Airline
+
+### OurAirports
+- https://ourairports.com/data/ — "All data is released to the Public Domain"; mirror repo https://github.com/davidmegginson/ourairports-data is **Unlicense** (GitHub API). Nightly updates. **EMBED**.
+- Raw URLs: `https://davidmegginson.github.io/ourairports-data/{airports,airport-frequencies,runways,navaids,countries,regions}.csv`.
+
+| File | Rows | Fields |
+|---|---|---|
+| airports.csv (12.7 MB) | **86,089**; by type: small_airport 42,734, heliport 23,216, closed 13,524, medium 4,106, seaplane_base 1,273, large 1,174, balloonport 62 | `id,ident,type,name,latitude_deg,longitude_deg,elevation_ft,continent,iso_country,iso_region,municipality,scheduled_service,icao_code,iata_code,gps_code,local_code,home_link,wikipedia_link,keywords` |
+| — with `iata_code` | **9,055** (large 1,171, medium 3,398, small 4,233, seaplane 153, heliport 100); `scheduled_service=yes` 4,335; US with IATA 2,036 (873 large/medium) | Suggested subset: `iata_code != '' AND type IN (large_airport, medium_airport)` → 4,569 rows |
+| countries.csv | **249** | `id,code,name,continent,wikipedia_link,keywords` |
+| regions.csv | **3,987** | `id,code,local_code,name,continent,iso_country,wikipedia_link,keywords` (code = ISO 3166-2, e.g. `US-KS`) |
+- No airline file. Derived: datasets/airport-codes on datahub (PDDL) — no need, use upstream.
+
+### OpenFlights
+- https://openflights.org/data.php — airports/airlines/routes/planes under **ODbL 1.0** (+ DbCL); airline/plane data partly from Wikipedia (GFDL/CC BY-SA). 5,888 airlines (2012). **AVOID** (share-alike). npm `airline-codes` (npow, ISC) is a straight OpenFlights sync → inherits ODbL → **AVOID**.
+
+### Airline IATA/ICAO codes — open options
+| Source | Licence | Count | Verdict |
+|---|---|---|---|
+| Wikidata SPARQL (https://query.wikidata.org/sparql) | **CC0** (https://www.wikidata.org/wiki/Wikidata:Licensing) | **2,928** items with IATA code (P229), **3,644** with ICAO code (P230) | **EMBED**. Query: `SELECT ?item ?itemLabel ?iata ?icao ?callsign WHERE { ?item wdt:P229 ?iata . OPTIONAL { ?item wdt:P230 ?icao } OPTIONAL { ?item wdt:P432 ?callsign } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }`. Includes defunct airlines — filter `MINUS { ?item wdt:P576 ?dissolved }` and/or P31 = Q46970 (airline). |
+| OpenTravelData `optd_airlines.csv` (https://github.com/opentraveldata/opentraveldata) | **CC BY 4.0** | 1,620 rows; `^`-separated; fields `pk^env_id^validity_from^validity_to^3char_code^2char_code^num_code^name^name2^alliance_code^alliance_status^type^wiki_link^flt_freq^alt_names^bases^key^version^parent_pk_list^successor_pk_list` | **EMBED+ATTR** (attribution notice required) |
+- Format rules: IATA designator = 2 alphanumeric chars (`[A-Z0-9]{2}`, at least one letter in practice; a third optional char exists in the standard but is unused); "controlled duplicates" share a code between non-overlapping regional carriers. ICAO designator = 3 letters `[A-Z]{3}`, unique. Accounting/prefix code = 3 digits (ticket number prefix, e.g. `016` United).
+
+### Aircraft types
+| Source | Licence | Verdict |
+|---|---|---|
+| ICAO Doc 8643 (https://www.icao.int/operational-safety/doc-8643-aircraft-type-designators) | ICAO copyright notice: "None of the materials … may be used, reproduced or transmitted … without permission in writing from ICAO"; API Data Service is paid (25 free trial calls) | **AVOID** — including GitHub mirrors such as ColtJD45/icao-aircraft-designator-list (MIT-labelled, 7,388 rows, fields `manufacturer,model,type_designator,description,engine_type,engine_count,wtc`) because the underlying data is scraped ICAO content (UNVERIFIED provenance) |
+| OpenSky aircraft database (https://opensky-network.org/data/aircraft; CSV https://s3.opensky-network.org/data-samples/metadata/aircraftDatabase.csv) | "The aircraft database is unlicensed and does not fall under our terms of use … offered as is"; built from registries, openflights.org and ICAO Doc 8643; citation requested; page says it is no longer up to date | **AVOID** ("unlicensed" ≠ open; upstream includes ODbL/ICAO). Fields for reference: `icao24,registration,manufacturericao,manufacturername,model,typecode,serialnumber,linenumber,icaoaircrafttype,operator,operatorcallsign,operatoricao,operatoriata,owner,…,engines,…,categoryDescription` |
+| Wikidata | **CC0** | **550** items with ICAO aircraft type designator (P8305); also IATA aircraft code P9040 (UNVERIFIED property id), manufacturer P176 | **EMBED**. Query: `SELECT ?item ?itemLabel ?icao ?mfrLabel WHERE { ?item wdt:P8305 ?icao . OPTIONAL { ?item wdt:P176 ?mfr } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }` |
+- Format: ICAO type designator 2–4 alphanumerics (`[A-Z0-9]{2,4}`, e.g. `B738`, `A20N`, `E190`); IATA aircraft code 3 alphanumerics (`738`, `32N`).
+
+### Airport / flight / seat format rules
+- IATA airport code `[A-Z]{3}`; ICAO location indicator `[A-Z0-9]{4}` (US `K…`, Canada `C…`, UK `EG…`, Sweden `ES…`). Use OurAirports rows so codes are real.
+- Flight number: 2-char IATA designator + 1–4 digits, no leading zeros in display (`QF9`, `AA1234`); systems cap at 0001–9999. Conventions (not rules): 1–999 mainline, 3000–5999 regional affiliates, ≥6000 codeshares, 8xxx charters, ≥9000 ferry/positioning. ICAO form: 3-letter designator + 1–4 alphanumerics (`AFR1`). Source: https://en.wikipedia.org/wiki/Flight_number.
+- Seat: `row + letter`. Rows typically 1–~60 (A380 up to ~90); many airlines skip 13 (and 14/17 on some carriers). Letters `A–K` **skipping I** (looks like 1/l); narrow-body 3-3 = `ABC DEF` (A/F windows, C/D aisles); wide-body 3-4-3 = `ABC DEFG HJK`, 2-4-2 = `AB DEFG JK`. Regex: `^[1-9][0-9]?[A-HJK]$`.

docs/research/part3-science-text-dates.md

@@ -0,0 +1,149 @@
+# Part 3 — Science/nature, Text, Dates: open data sources for an MIT fake-data library
+
+Researched 2026-09-17. "OK to embed" = redistributable inside an MIT repo with the stated notice. Anything not fetched first-hand is marked **UNVERIFIED**.
+
+## 7. Science / nature
+
+### Periodic table
+
+| Source | Licence | Embed in MIT repo | Fields | Rows | Format |
+|---|---|---|---|---|---|
+| PubChem periodic table — CSV https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/CSV, JSON https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/JSON (page: https://pubchem.ncbi.nlm.nih.gov/periodic-table/) | US-government work, public domain. NLM policy (https://www.ncbi.nlm.nih.gov/home/about/policies/): "Information that is created by or for the US government on this site is within the public domain … may be freely distributed and copied. However, it is requested that in any subsequent use of this work, NLM be given appropriate acknowledgment." | Yes; add an acknowledgment line ("Element data: PubChem/NLM"). | `AtomicNumber, Symbol, Name, AtomicMass, CPKHexColor, ElectronConfiguration, Electronegativity, AtomicRadius, IonizationEnergy, ElectronAffinity, OxidationStates, StandardState, MeltingPoint, BoilingPoint, Density, GroupBlock, YearDiscovered` (17) | 118 (Z=1..118, verified) | CSV; JSON is `{Table:{Columns:{Column:[...]},Row:[{Cell:[...]}]}}` |
+| NIST periodic table https://www.nist.gov/pml/periodic-table-elements | Public domain (17 USC 105); NIST asks for citation (https://www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications) | Yes, but | PDF only (no CSV/JSON) — not usable as a data source | 118 | PDF |
+| Bowserinator/Periodic-Table-JSON https://github.com/Bowserinator/Periodic-Table-JSON | **CC BY-SA 3.0** (LICENSE.md, verified; data derived from Wikipedia) | **No** — ShareAlike is incompatible with MIT redistribution | 33 keys incl. `name, symbol, number, atomic_mass, category, phase, period, group, block, shells, electron_configuration, summary, cpk-hex, …` | **119** (includes hypothetical Ununennium) | JSON, CSV |
+| faker-js `src/locales/en/science/chemical_element.ts` https://github.com/faker-js/faker (branch `next`) | MIT (LICENSE verified) | Yes, keep MIT notice | `{symbol, name, atomicNumber}` | 118 | TS |
+
+Validation rules: `AtomicNumber` 1..118 unique; `Symbol` 1–2 letters, first uppercase, unique; `Name` unique. PubChem leaves numeric fields empty for Z≥104 where unknown; `ElectronConfiguration` for Og is `"[Rn]7s2 7p6 5f14 6d10 (predicted)"` and `StandardState` is `"Expected to be a Gas"` — strip "(predicted)"/"Expected to be a" if you want an enum. `AtomicMass` for unstable elements is the mass number of the most stable isotope as a bare decimal (e.g. `295.216`), no brackets. `CPKHexColor` is 6 hex digits without `#`, empty for some.
+
+### SI units
+
+- Source: BIPM SI Brochure, 9th ed. (2019; current revision v4.01, 2026) https://www.bipm.org/en/publications/si-brochure ; PDF https://www.bipm.org/documents/20126/41483022/SI-Brochure-9-EN.pdf
+- Licence: **CC BY 4.0** — stated on the brochure page and on BIPM's copyright page (https://www.bipm.org/en/copyright): content may be adapted, copied, used in commercial products with "appropriate acknowledgement of the BIPM and its source"; BIPM name/logo must not imply endorsement. Verified via the HTML pages; the PDF's own front-matter licence line is **UNVERIFIED** (could not text-extract the PDF here).
+- Embed: Yes — the unit *names/symbols* are facts (not copyrightable); attribute "SI Brochure, BIPM, CC BY 4.0" anyway.
+- Data (cross-checked against https://en.wikipedia.org/wiki/International_System_of_Units):
+  - 7 base units: second s (time), metre m (length), kilogram kg (mass), ampere A (electric current), kelvin K (thermodynamic temperature), mole mol (amount of substance), candela cd (luminous intensity).
+  - 22 coherent derived units with special names: radian rad, steradian sr, hertz Hz, newton N, pascal Pa, joule J, watt W, coulomb C, volt V, farad F, ohm Ω, siemens S, weber Wb, tesla T, henry H, degree Celsius °C, lumen lm, lux lx, becquerel Bq, gray Gy, sievert Sv, katal kat.
+  - 24 prefixes: quetta Q 10^30, ronna R 10^27, yotta Y 10^24, zetta Z 10^21, exa E 10^18, peta P 10^15, tera T 10^12, giga G 10^9, mega M 10^6, kilo k 10^3, hecto h 10^2, deca da 10^1, deci d 10^-1, centi c 10^-2, milli m 10^-3, micro µ 10^-6, nano n 10^-9, pico p 10^-12, femto f 10^-15, atto a 10^-18, zepto z 10^-21, yocto y 10^-24, ronto r 10^-27, quecto q 10^-30.
+- Format rules for valid output: unit symbols are case-sensitive (`s` vs `S`, `m` vs `M`); prefix and unit symbol are joined without a space (`kJ`, `µs`); a space separates number and unit (`12 kg`), except the degree/minute/second of plane angle; `kg` already carries a prefix — never `µkg`; micro is Greek mu U+03BC (U+00B5 micro sign is the legacy compatibility character); ohm is U+03A9 (U+2126 OHM SIGN is deprecated); `°C` not `° C`.
+- MIT alternative: faker-js `en/science/unit.ts` — 29 `{name, symbol}` objects, MIT.
+
+### Animals by class
+
+- **Wikidata** (https://www.wikidata.org/wiki/Wikidata:Licensing): all structured data **CC0** — no attribution required. Embed: Yes.
+  - Feasibility, live SPARQL (https://query.wikidata.org/sparql), pattern `?t wdt:P31 wd:Q16521; wdt:P105 wd:Q7432; wdt:P171* wd:<class>; wdt:P1843 ?cn FILTER(LANG(?cn)='en')` (taxon, rank species, parent-taxon chain, English common name):
+    - Mammalia Q7377: **5,759** species with ≥1 English common name.
+    - Aves Q5113: **12,432**.
+    - Reptilia Q10811: **16,369** — suspiciously large; Wikidata's parent chain likely pulls birds/Sauropsida in. Use Squamata Q122422 + Testudines Q223044 + Crocodilia Q1387 (or check the chain) instead.
+    - Insecta Q1390 and Plantae Q756: **timed out (502/504) every attempt** — trees too big for the 60 s public endpoint. Options: paginate by order (`P171` one level at a time), use the weekly JSON dump, or take GBIF instead.
+  - Sample rows (mammals): `Notoryctes typhlops` → "Central Desert Marsupial Mole", "Itjaritjari", "Marsupial Mole", "Southern marsupial mole", "Southern Marsupial Mole" — i.e. **several P1843 values per taxon, inconsistent casing, and indigenous-language names tagged `en`**. Validation: pick one name per taxon (prefer the one matching the item's English `rdfs:label`), normalise case, drop names that equal the scientific name.
+- **GBIF Backbone Taxonomy** https://www.gbif.org/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c — licence **CC BY 4.0** (verified via https://api.gbif.org/v1/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c, `license: http://creativecommons.org/licenses/by/4.0/legalcode`); required citation text: "GBIF Secretariat (2023). GBIF Backbone Taxonomy. Checklist dataset https://doi.org/10.15468/39omei". Embed: Yes with that citation. Vernacular names come via `/v1/species/{key}/vernacularNames`, each row from a *different* source checklist with its own licence (some CC BY-NC) — record and check per row. GBIF's site terms page (https://www.gbif.org/terms) was unreachable (timeout) — **UNVERIFIED** beyond the API field.
+- **faker-js** `src/locales/en/animal/*` (MIT; counts = quoted string lines on branch `next`): bird 821, snake 533, dog 493, cow 467, horse 342, rodent 161, insect 129, fish 95, cat 55, cetacean 52, rabbit 49, type 44 (generic: "bat, bear, bee, …, zebra"), pet_name 42, crocodilia 24, bear 8, lion 7. These are breeds/species mixed with common names, no class field; fine as MIT drop-ins. (A WebFetch summariser claimed 1,036 dogs; the regex count is 493 — treat the exact count as UNVERIFIED.)
+
+### Plants
+
+- Same Wikidata approach (CC0) — but the Plantae query timed out; paginate by order or family (`?t wdt:P171 ?family . ?family wdt:P105 wd:Q35409`) or use the dump.
+- GBIF Plantae (kingdomKey 6) vernacular names, CC BY 4.0, same per-row-licence caveat.
+- faker-js has **no** plant list (en locale dirs: airline, animal, app, book, color, commerce, company, database, date, finance, food, hacker, internet, location, lorem, medical, music, person, phone_number, science, team, vehicle, word).
+- USDA PLANTS database (public domain, has common names) — **UNVERIFIED**, not fetched.
+
+### Planets / moons
+
+- NASA NSSDCA Planetary Fact Sheet https://nssdc.gsfc.nasa.gov/planetary/factsheet/ — 10 columns (Mercury, Venus, Earth, Moon, Mars, Jupiter, Saturn, Uranus, Neptune, Pluto) × 20 rows (mass 10^24 kg, diameter km, density, gravity, escape velocity, rotation period, day length, distance from Sun, perihelion, aphelion, orbital period, orbital velocity, inclination, eccentricity, obliquity, mean temp °C, surface pressure, number of moons, rings Y/N, magnetic field Y/N). HTML table only. NASA content "generally not subject to copyright in the United States" (https://www.nasa.gov/nasa-brand-center/images-and-media/); NASA asks to be acknowledged; the NASA insignia must not be used. Embed: Yes with "Source: NASA NSSDCA".
+- Wikidata (CC0): planets = `?p wdt:P31/wdt:P279* wd:Q634; wdt:P397 wd:Q525` → Mercury Q308, Venus Q313, Earth Q2, Mars Q111, Jupiter Q319, Saturn Q193, Uranus Q324, Neptune Q332 (plus hypothetical Theia Q1053432 — filter out). Moons = `?m wdt:P397 <planet>; wdt:P31/wdt:P279* wd:Q2537` (natural satellite; note **Q2199 is "dwarf planet"**, not moon): Saturn 165, Jupiter 85, Uranus 29, Neptune 16, Earth 5, Mars 2. Caveats: Wikidata lags IAU counts (Saturn has 274 recognised moons as of 2025 — UNVERIFIED figure); Earth's 5 includes quasi-satellites; provisional designations (`S/2004 S 3`) are labels too — filter `^S/\d{4}` if you want proper names only.
+
+### Blood type distribution
+
+- **US** (Stanford Blood Center https://stanfordbloodcenter.org/donate-blood/blood-donation-facts/blood-types/, verified; cites AABB Technical Manual 18th ed.): O+ 37.4 %, A+ 35.7 %, B+ 8.5 %, AB+ 3.4 %, O− 6.6 %, A− 6.3 %, B− 1.5 %, AB− 0.6 % (sums to 100.0). Facts — freely embeddable; cite Stanford/AABB.
+- American Red Cross https://www.redcrossblood.org/donate-blood/blood-types.html — **UNVERIFIED**: site returns HTTP 403 (Akamai) to both WebFetch and the headless browser. Search snippets say the Red Cross gives type O by ethnicity (Latino 57 %, African American 51 %, Caucasian 45 %) rather than an 8-way national table.
+- **Global**: no authoritative figure found. Wikipedia "Blood type distribution by country" gives a population-weighted row (O+ 38.4, A+ 27.3, B+ 8.1, AB+ 2.0, O− 13.1, A− 8.1, B− 2.0, AB− 0.01) that its own editors flag "unreliable source"; WorldAtlas gives O+ 42, A+ 31, B+ 15, AB+ 5, O− 3, A− 2.5, B− 1, AB− 0.5 with no source. Recommend shipping only the US table, or per-country tables from Wikipedia (facts; Wikipedia prose is CC BY-SA but the numbers are cited to national blood services — cite those).
+- Validation: ABO ∈ {A, B, AB, O}; Rh ∈ {+, −}; weights must sum to 100 ± rounding.
+
+## 8. Text
+
+### English word lists with part of speech
+
+| Source | Licence | Embed | Counts | Format |
+|---|---|---|---|---|
+| Princeton WordNet 3.0 / 3.1 https://wordnet.princeton.edu/license-and-commercial-use ; download https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz | "WordNet License" (SPDX `WordNet`), BSD-style: permission to use/copy/modify/distribute "for any purpose and without fee or royalty … provided that … the following copyright notice and statements, including the disclaimer … appear on ALL copies … including modifications". Notice text: "WordNet 3.0 Copyright 2006 by Princeton University. All rights reserved." + AS-IS disclaimer; no use of Princeton's name in advertising. | Yes — ship the LICENSE text alongside the data | WordNet 3.0 (wnstats(7WN), verified): unique strings noun 117,798 / verb 11,529 / adj 21,479 / adv 4,481 (total 155,287); synsets 82,115 / 13,767 / 18,156 / 3,621 | WNDB: `index.noun|verb|adj|adv` (one lemma per line: `lemma pos synset_cnt p_cnt [ptr…] sense_cnt tagsense_cnt synset_offset…`), `data.<pos>` (synsets with glosses). Multi-word lemmas use `_` for spaces; lower-case; entries can contain digits/apostrophes/hyphens. |
+| Open English WordNet 2025 https://en-word.net/ ; https://github.com/globalwordnet/english-wordnet | **CC BY 4.0**; cite "Open English WordNet … derived from Princeton WordNet" | Yes, with attribution | 135,969 words, 107,519 synsets (README, verified) | `english-wordnet-2025.zip` (WNDB, 9.2 MB), `english-wordnet-2025.xml.gz` (WN-LMF, 10.8 MB), `english-wordnet-2025-json.zip` (9.5 MB), `.ttl.gz` (16.9 MB) |
+| SCOWL v2 / ESDB https://github.com/en-wl/wordlist (branch v2) ; classic http://wordlist.aspell.net/ | Custom MIT-like (Copyright 2000-2026 Kevin Atkinson: use/copy/modify/distribute/sell "provided that the above copyright notice appears in all copies and that both the above copyright notice and this notice appear in supporting documentation"). Sources 12dicts + ENABLE2K are public domain; lists above size 80 add the UKACD notice; **POS assignments partly from WordNet, so the WordNet notice "MIGHT apply"** (their words). Australian-English parts carry an extra notice. | Yes, ship the Copyright file | Sizes 35 (small), 50 (medium), 60 (spell-check default), 70 (large), 80 (incl. game words), 85 (archaic). No per-size counts published in the v2 README. Classic SCOWL sizes 10–95 — counts **UNVERIFIED** (README not reachable). | v2: `scowl.db` (SQLite), `scowl.txt`, Python `libscowl`. ESDB rows carry POS, spelling (A/B/Z/C/D) and region codes. |
+| 12dicts http://wordlist.aspell.net/12dicts-readme/ | Public domain ("I explicitly release them to the public domain, but request acknowledgment"), **except** `2of12inf` and the `2+2+3*` lists, which depend on AGID and inherit its terms | Yes (PD lists); acknowledge Alan Beale | 3esl ≈22k, 6of12 ≈32k, 2of12 ≈41k, 2of12inf ≈82k, 3of6game ≈65k, 5d+2a ≈68k, 3of6all ≈83k, 2+2+3lem ≈84k, 2of5core ≈4.7k, 6phrase ≈22k, neol2016 ≈600 | Plain text, one word per line; marker suffixes `+ ! ^ &` (not POS). **No POS tags** in any current 12dicts list. |
+| Moby Part-of-Speech II https://www.gutenberg.org/ebooks/3203 ; file https://www.gutenberg.org/files/3203/files/mobypos.txt | Public domain ("Public Domain material by grant from the author, January, 2001", in the package README) | Yes, no notice needed | **233,356** lines (verified) | One entry per line: `word\CODES`, CRLF line ends (e.g. `A-line\NA`, `a tempo\Avh`). Codes in priority order: `N` noun, `p` plural, `h` noun phrase, `V` verb (usu. participle), `t` transitive verb, `i` intransitive verb, `A` adjective, `v` adverb, `C` conjunction, `P` preposition, `!` interjection, `r` pronoun, `D` definite article, `I` indefinite article, `o` nominative. Many entries are phrases, proper nouns, or 1990s-era; non-ASCII entries use a legacy 8-bit encoding (e.g. `a bon march\v` lost its `é`) — restrict to ASCII `[a-z]+` for a clean list. |
+| faker-js `en/word/*` (MIT) | MIT | Yes | adjective 1000, noun 1000, verb 1000, adverb 325, preposition 109, conjunction 51, interjection 46 | TS string arrays |
+
+### Word frequency lists
+
+| Source | Licence | Verdict |
+|---|---|---|
+| Google Books Ngram data v2/v3 https://storage.googleapis.com/books/ngrams/books/datasetsv3.html | **CC BY 3.0** ("This compilation is licensed under a Creative Commons Attribution 3.0 Unported License") | Embeddable with attribution. Format `ngram TAB year TAB match_count TAB volume_count`; 1-gram files split by leading letter, GBs each — aggregate offline to a top-N list. |
+| wordfreq https://github.com/rspeer/wordfreq | Code **Apache 2.0** (not MIT). Data: "may be redistributed under a Creative Commons Attribution-ShareAlike 4.0 license" (mix of Google Books, Wikipedia, OpenSubtitles, SUBTLEX, Leeds, ParaCrawl, Twitter); maintainer notes the CSV export "does not follow the CC-By-SA license". Project in sunset mode (data snapshot ≈2021). | **Avoid** embedding the data (ShareAlike). |
+| Peter Norvig count_1w.txt https://norvig.com/ngrams/ | Page says "Code … under the MIT license"; the **data** comes from the Google Web 1T corpus distributed by LDC under LDC terms — no data licence stated. | **UNVERIFIED / avoid**. 333,333 words, `word TAB count`, lowercase. |
+| hermitdave/FrequencyWords https://github.com/hermitdave/FrequencyWords | "MIT License for code. CC-by-sa-4.0 for content." | **Avoid** (ShareAlike). |
+
+### Lorem ipsum
+
+- Canonical paragraph (Letraset, 1966; public domain — garbled Cicero): "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+- Source: Cicero, *De finibus bonorum et malorum* 1.10.32–33 (45 BC), public domain; the "dolorem ipsum" fragment begins "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit…". Reference: https://en.wikipedia.org/wiki/Lorem_ipsum
+- MIT word pool alternative: faker-js `en/lorem/word.ts` — 999 words.
+
+### Quote collections
+
+- Project Gutenberg #27889 https://www.gutenberg.org/ebooks/27889 — Bartlett, *Familiar Quotations*, **9th edition** (title page "NINTH EDITION", copyright 1875/1882/1891/1903) — not the 1919 10th. "Public domain in the USA." Plain text 3.1 MB, HTML zip 50 MB, EPUB.
+- Project Gutenberg #16732 https://www.gutenberg.org/ebooks/16732 — an early (Hurst & Co. reprint) edition, no edition number given. PD in USA.
+- 10th edition (1914/1919, ed. Nathan Haskell Dole) exists on archive.org (https://archive.org/details/bartlettsfamilia0000john_o4b4) — not found on Gutenberg; **UNVERIFIED** availability as clean text.
+- Either Bartlett text needs parsing (quote / author / source headings); no structured file. Wikiquote is CC BY-SA — avoid.
+- Validation: strip Gutenberg header/footer (`*** START OF THE PROJECT GUTENBERG EBOOK` … `*** END …`) before extraction; Gutenberg's trademark terms only bind if you keep the "Project Gutenberg" name on the output — strip it.
+
+### Colour names
+
+| Source | Licence | Embed | Count / format |
+|---|---|---|---|
+| CSS Color Module Level 4 §named colors https://www.w3.org/TR/css-color-4/#named-colors | W3C Document License (2023) https://www.w3.org/copyright/document-license-2023/ — copy/distribute allowed for any purpose; derivatives allowed only "to facilitate implementation of the technical specifications"; notice required: "Copyright © 2023 W3C®. This software or document includes material copied from or derived from [title and URI]." | Yes — the name→hex table is factual data and is copied into every browser; include the W3C notice line with the spec title + URI. | **148** names (verified: aliceblue … yellowgreen, incl. `rebeccapurple #663399`); all names lowercase ASCII, hex 6 lowercase digits. |
+| XKCD colour survey https://xkcd.com/color/rgb.txt | **CC0** (file header: `# License: https://creativecommons.org/publicdomain/zero/1.0/`) | Yes, no notice needed | **949** lines (verified). Format `name TAB #rrggbb TAB` (note the trailing tab). Max name length 26; all hex valid lowercase; no duplicate names. Contains crude names ("poo", "baby poop green", "diarrhea", "vomit") — apply a blocklist. |
+| faker-js `en/color/human.ts` | MIT | Yes | 31 names |
+| Pantone | Proprietary | **Avoid** | — |
+
+## 9. Dates
+
+### IANA tz database
+
+- Source https://www.iana.org/time-zones ; files https://data.iana.org/time-zones/tzdb/ (current version **2026d**, verified). LICENSE: "Unless specified below, all files in the tz code and data (including this LICENSE file) are in the public domain" (only `date.c`, `newstrftime.3`, `strftime.c` are BSD-3). Embed: Yes, no notice needed.
+- `zone1970.tab` — **312** data rows; tab-separated; UTF-8; `#` comments. Columns: (1) comma-separated ISO 3166-1 alpha-2 codes of countries overlapping the zone, most-populous first; (2) coordinates in ISO 6709 `±DDMM±DDDMM` or `±DDMMSS±DDDMMSS` (lat then lon, no separator); (3) TZ name (`Europe/Zurich`); (4) comment, present iff the country has several zones. One row per timezone where civil time has agreed since 1970.
+- `zone.tab` — **418** rows, older one-country-per-row table (column 1 is a single country code); kept for backward compatibility. Prefer `zone1970.tab`.
+- `backward` — **252** `Link TARGET LINK-NAME` lines mapping old/merged names (`US/Eastern`, `Asia/Calcutta`, `Europe/Kiev`) to current ones; also a few `Zone` entries. Use it to (a) accept aliases as input and (b) never emit them as output.
+- UTC offsets: do **not** embed — offsets depend on DST rules that change several times a year. Derive at runtime: `new Intl.DateTimeFormat('en-US', { timeZone, timeZoneName: 'longOffset' }).formatToParts(date)` → part `timeZoneName` = `GMT-05:00` (or `GMT` for zero) — strip `GMT`, treat bare `GMT` as `+00:00`. Validation of a generated zone name: `Intl.supportedValuesOf('timeZone')` (canonical names only; whether links like `US/Eastern` appear depends on the ICU build) or `try { new Intl.DateTimeFormat(undefined, { timeZone }) } catch { invalid }`, which accepts links too.
+
+### ISO 8601 layouts (https://en.wikipedia.org/wiki/ISO_8601)
+
+| Layout | Extended | Basic | Example |
+|---|---|---|---|
+| Calendar date | `YYYY-MM-DD` | `YYYYMMDD` | 2009-01-06 |
+| Ordinal date | `YYYY-DDD` | `YYYYDDD` | 1981-095 |
+| Week date | `YYYY-Www-D` | `YYYYWwwD` | 2009-W01-1 (weeks start Monday, week 1 contains 4 Jan) |
+| Time | `hh:mm:ss[.fff]` | `hhmmss` | 13:47:30 (24 h; `T` prefix optional standalone) |
+| UTC / offset | `Z` or `±hh:mm` | `±hhmm`, `±hh` | 2007-04-05T14:30Z, 2024-06-01T09:00-05:00 |
+| Date-time | date `T` time offset | | 2007-04-05T14:30:00+02:00 |
+| Duration | `PnYnMnDTnHnMnS` / `PnW` | | P3Y6M4DT12H30M5S |
+| Interval | start`/`end, start`/`duration, duration`/`end | | 2007-03-01T13:00:00Z/2008-05-11T15:30:00Z |
+| Recurring | `Rn/`interval | | R5/2008-03-01T13:00:00Z/P1Y2M10DT2H30M |
+
+RFC 3339 profile (what JSON/APIs usually mean): extended forms only, `T`/`Z` may be lowercase, space allowed instead of `T`, `-00:00` = unknown offset, no durations/intervals/week/ordinal dates. Generate RFC 3339 by default.
+
+### Unicode CLDR (locale month/weekday names, date patterns)
+
+- Licence: **Unicode License v3** (SPDX `Unicode-3.0`), OSI-approved 2023-11-17 (https://opensource.org/license/unicode-3-0). Text https://www.unicode.org/license.txt — MIT-style: use/copy/modify/sell "provided that either (a) this copyright and permission notice appear with all copies of the Data Files or Software, or (b) this copyright and permission notice appear in associated Documentation"; no use of the Unicode name in advertising. Embed: Yes; put the notice in LICENSE/NOTICE.
+- npm: `cldr-core`, `cldr-dates-full`, `cldr-numbers-full` (peer of dates), etc.; current **48.2.0** (CLDR 48, verified via npm registry, `license: Unicode-3.0`); repo https://github.com/unicode-org/cldr-json .
+- Path is **`cldr-dates-full/main/<locale>/ca-gregorian.json`** (not `dates/gregorian.json`), rooted at `main.<locale>.dates.calendars.gregorian`. Keys (verified for `en`): `months`, `days`, `quarters`, `dayPeriods`, `eras`, `dateFormats`, `dateSkeletons`, `timeFormats`, `timeSkeletons`, `dateTimeFormats`, `dateTimeFormats-atTime`, `dateTimeFormats-relative`.
+  - `months.{format|stand-alone}.{abbreviated|narrow|wide}` keyed `"1".."12"`; `days.{format|stand-alone}.{abbreviated|narrow|short|wide}` keyed `sun..sat`.
+  - `dateFormats`: en = full `EEEE, MMMM d, y`, long `MMMM d, y`, medium `MMM d, y`, short `M/d/yy`.
+  - `timeFormats`: en = full `h:mm:ss a zzzz`, long `h:mm:ss a z`, medium `h:mm:ss a`, short `h:mm a` — **the `a` is preceded by U+202F NARROW NO-BREAK SPACE**; `*-alt-ascii` variants use a plain space. Pick one spelling (ASCII) and ignore the other.
+  - `dateTimeFormats.{full|long|medium|short}` = glue like `{1}, {0}` ({1} date, {0} time); `availableFormats` = skeleton→pattern map (`yMMMd` → `MMM d, y`), plus `intervalFormats`, `appendItems`.
+  - `dayPeriods.format.abbreviated`: `am: AM`, `pm: PM`, plus `midnight`, `noon`, `morning1`… variants.
+- Pattern letters follow LDML (https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table): `y` year, `M`/`L` month (format/stand-alone), `d` day, `E`/`c` weekday, `h` 1–12, `H` 0–23, `a` AM/PM, `z`/`zzzz` zone name, `x`/`X` ISO offset; literal text in single quotes. Same patterns drive `Intl.DateTimeFormat` internally, so runtime `Intl` can replace embedding for month/weekday names if bundle size matters.
+
+## Summary of licence verdicts
+
+- Embed freely (PD/CC0): PubChem periodic table, NIST, NASA fact sheet, Wikidata, Moby POS, 12dicts (PD lists), Bartlett via Gutenberg, XKCD colours, IANA tz, Lorem ipsum/Cicero.
+- Embed with notice: WordNet 3.x (WordNet License), Open English WordNet (CC BY 4.0), SCOWL/ESDB (custom MIT-like), GBIF backbone (CC BY 4.0 + citation), BIPM SI (CC BY 4.0), CSS named colours (W3C notice), CLDR (Unicode-3.0), Google Books ngrams (CC BY 3.0), faker-js lists (MIT).
+- Avoid: Bowserinator periodic table (CC BY-SA 3.0), wordfreq data (CC BY-SA 4.0), hermitdave (CC BY-SA 4.0), Norvig count_1w (LDC-derived, unstated), Wikiquote (CC BY-SA), Pantone.
+- UNVERIFIED: American Red Cross figures (site blocks fetches), any global blood-type table, BIPM PDF front-matter licence line, classic SCOWL per-size counts, USDA PLANTS, IAU moon counts, exact faker dog count, Bartlett 10th-edition text availability.

docs/research/part4-iso-commerce-books.md

@@ -0,0 +1,139 @@
+# Part 4 — ISO lists, commerce, books/music/food
+
+Researched 2026-09-17. "Embed?" = can the data ship inside an MIT-licensed repo. Anything marked UNVERIFIED was not confirmed against the primary source.
+
+## 10. ISO lists
+
+### ISO 3166-1 countries
+
+| Source | Licence | Embed in MIT repo? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [datasets/country-codes](https://github.com/datasets/country-codes) | PDDL (README: "Public Domain Dedication and License"; GitHub API detects no licence file) | Yes, no attribution required. README caveat: ISO itself says its list is "for internal use and non-commercial purposes free of charge" — the repo argues no rights subsist in a list of facts | 249 | ISO3166-1-Alpha-2/-Alpha-3/-numeric, official_name_en, UNTERM names (ar/zh/en/fr/ru/es short+formal), Dial, TLD, Capital, Continent, Languages, ISO4217 currency code/name/numeric/minor_unit, Region/Sub-region/Intermediate names+codes (M49), FIPS, IOC, FIFA, MARC, ITU, WMO, GAUL, EDGAR, Geoname ID, wikidata_id, CLDR display name, is_independent, LDC/LLDC/SIDS flags | CSV `data/country-codes.csv` |
+| [annexare/Countries](https://github.com/annexare/Countries) (npm `countries-list` 3.4.1, 2026-07-14) | MIT | Yes; keep MIT notice | 252 countries, 185 languages (incl. non-ISO entries like AC, XK) | per country: name, native, phone[] (calling codes), continent, capital, currency[], languages[], optional alias[], partOf; separate ISO 4217 currency table (name, native, symbol, numeric, decimals); languages: name, native | TS source, exported JSON/CSV/SQL |
+| [Debian iso-codes](https://salsa.debian.org/iso-codes-team/iso-codes) | LGPL-2.1-or-later (Debian copyright file) | Not safely as embedded data in an MIT repo — LGPL applies to the data files; you would have to ship it as a separately licensed component with LGPL notice. Avoid unless accepting that | 3166-1: 249 | alpha_2, alpha_3, numeric, name, official_name, common_name, flag | JSON `data/iso_3166-1.json` (raw URL needs `?inline=false`) |
+| [mledoze/countries](https://github.com/mledoze/countries) | ODbL-1.0 | No — share-alike + attribution; derived databases must be ODbL. Avoid | ~250 | common/official names, cca2/cca3/ccn3/cioc, tld, currencies, idd (calling codes), capital, region/subregion, languages, translations, latlng, borders, area, demonyms, flag emoji, UN member | JSON/CSV/XML/YAML |
+| [restcountries](https://gitlab.com/restcountries/restcountries) | MPL-2.0 (source repo; hosted API v5 is proprietary/API-key) | File-level copyleft: a copied data file stays MPL and must carry the notice; OK to ship next to MIT code but not to relicense. Prefer PDDL/MIT sources | 250+ | name, tld, cca2, ccn3, cca3, cioc, fifa, independent, status, unMember, currencies, idd, capital, capitalInfo, altSpellings, region, subregion, continents, languages, translations, latlng, landlocked, borders, area, flag (emoji), demonyms, flags, coatOfArms, population, maps, gini, car, postalCode, startOfWeek, timezones | JSON `src/main/resources/countriesV3.1.json` |
+| [lukes/ISO-3166-Countries-with-Regional-Codes](https://github.com/lukes/ISO-3166-Countries-with-Regional-Codes) | CC BY-SA 4.0 (LICENSE.md) | No — share-alike. Avoid | 249 | name, alpha-2, alpha-3, country-code, iso_3166-2, region, sub-region, intermediate-region + codes | CSV/JSON/XML (all, slim-2, slim-3) |
+| [stefangabos/world_countries](https://github.com/stefangabos/world_countries) | Data: CC BY-SA 4.0 (README, GitHub licence detection); npm `world_countries_lists` package.json says LGPL-3.0-or-later — contradictory | No — share-alike either way. Avoid | 249 world / 193 UN; subdivisions 5046 | id (numeric), alpha2, alpha3, name (37 languages); subdivisions: country, code, name, name_en, type, parent | JSON/CSV/PHP/SQL/XML |
+| [i18n-iso-countries](https://www.npmjs.com/package/i18n-iso-countries) 7.14.0 | MIT | Yes | 78 language files | alpha2, alpha3, numeric, localized names | JSON per language |
+| Wikipedia ISO 3166-1 | CC BY-SA 4.0 | No (share-alike) | — | — | — |
+| [ISO OBP](https://www.iso.org/obp) | Proprietary; ISO grants free use only for "internal use and non-commercial purposes" (quoted in datasets/country-codes README) | No | — | — | — |
+
+- Flag emoji: derive, no data needed — for alpha-2 `XY`, emit U+1F1E6 + (X − 'A') and U+1F1E6 + (Y − 'A') (regional indicator symbols).
+- Recommendation: `datasets/country-codes` (PDDL) covers every requested field (name, alpha2, alpha3, numeric, calling code, TLD, capital, currency, languages) in one CSV; flag emoji derived. `annexare` (MIT) is the fallback for native names/calling codes if PDDL's ISO caveat worries you; it lacks TLD.
+- Validation rules: alpha-2 `^[A-Z]{2}$`, alpha-3 `^[A-Z]{3}$`, numeric `^\d{3}$` (zero-padded, keep as string); calling code 1–3 digits, NANP countries share `1`; TLD `^\.[a-z]{2}$` (ccTLD = lowercase alpha-2, exceptions: GB uses `.uk`).
+
+### ISO 639 languages
+
+| Source | Licence | Embed? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [langs](https://www.npmjs.com/package/langs) 2.0.0 (2017, unmaintained) | MIT | Yes | 184 | name, local, 1 (639-1), 2, 2T, 2B, 3 | `data.js` array |
+| [iso-639-1](https://www.npmjs.com/package/iso-639-1) 3.1.6 (2026-07-02) | MIT | Yes | 183 | code, name, nativeName | `src/data.js` |
+| annexare/Countries languages | MIT | Yes | 185 | code (639-1), name, native | TS/JSON |
+| Debian iso-codes 639-2 / 639-3 / 639-5 | LGPL-2.1+ | See above (LGPL) | 639-2: 487; 639-3: 7923 | 639-2: alpha_2, alpha_3, bibliographic, name, common_name; 639-3: + inverted_name, scope, type | JSON |
+| [SIL ISO 639-3 tables](https://iso639-3.sil.org/code_tables/download_tables) | SIL terms: may incorporate into software (commercial or not) with attribution to iso639-3.sil.org, must not modify identifiers, and product "does not provide a means to redistribute the code set" | Risky — a git repo with the table IS a means to redistribute; avoid embedding the full table. Codes themselves are facts | 7927 | Id, Part2b, Part2t, Part1, Scope, Language_Type, Ref_Name, Comment | tab-delimited `iso-639-3.tab` |
+| [Library of Congress ISO 639-2](https://www.loc.gov/standards/iso639-2/ascii_8bits.html) | US federal government work → public domain in the US | Yes | ~487 (matches Debian 639-2 count) | pipe-delimited: alpha3-B, alpha3-T, alpha2, English name, French name — UNVERIFIED: loc.gov is behind a Cloudflare bot check (WebFetch, curl and Playwright all blocked); field layout from memory | `ISO-639-2_utf-8.txt` |
+
+- Recommendation: `langs` or `iso-639-1` (MIT) for 639-1 names; both are small and derived from public code tables. Anything 639-3-sized is LGPL or SIL-restricted.
+- Rules: 639-1 `^[a-z]{2}$`, 639-2/3 `^[a-z]{3}$`; 639-2 has B/T pairs (e.g. `fre`/`fra`, `ger`/`deu`) — pick one consistently (T codes = 639-3).
+
+### ISO 3166-2 subdivisions
+
+| Source | Licence | Embed? | Rows | Fields |
+|---|---|---|---|---|
+| Debian iso-codes 3166-2 | LGPL-2.1+ | LGPL caveat | 5046 | code, name, type, parent |
+| [olahol/iso-3166-2.json](https://github.com/olahol/iso-3166-2.json) | package.json says ISC; no LICENSE file; GitHub detects none; data scraped from eQuest xls (provenance unclear); last push 2021 | No — unclear licence and provenance | 237 countries / 3807 divisions | `{alpha2: {name, divisions: {code: name}}}` |
+| [esosedi/3166](https://github.com/esosedi/3166) (npm `iso3166-2-db` 2.3.11) | MIT | Yes; data mixes GeoNames (CC BY 4.0), Wikipedia (CC BY-SA), OSM (ODbL) — the MIT label may not survive scrutiny for the Wikipedia/OSM-derived parts. UNVERIFIED how much is derived from each | "all countries and regions" (count not stated) | ISO 3166-2 + FIPS codes, names in 12 languages, admin level, GeoNames/OSM/Wikipedia/WOF ids |
+| stefangabos subdivisions | CC BY-SA 4.0 | No | 5046 | country, code, name, name_en, type, parent |
+| [US Census state.txt](https://www2.census.gov/geo/docs/reference/state.txt) | US government work → public domain | Yes | 57 (50 states + DC + 6 territories) | STATE (FIPS), STUSAB, STATE_NAME, STATENS |
+
+- Recommendation: for an en_US library, ship only the US list (Census/USPS, public domain: 50 states + DC, optionally territories) — ISO 3166-2 code = `US-` + USPS abbreviation. Skip the ~5000-row world list; every open compilation is LGPL, CC BY-SA or unclear.
+- Rule: `^[A-Z]{2}-[A-Z0-9]{1,3}$`.
+
+## 11. Commerce
+
+### Word lists (product name / adjective / material)
+
+- [@faker-js/faker](https://github.com/faker-js/faker) 10.6.0 — MIT (LICENSE covers code and locale data; copyright Faker contributors 2022-2025 and Marak Squires 2011-2020; acknowledges Ruby Faker and Perl Data::Faker, both MIT). No separate data licence.
+- Provenance policy in [CONTRIBUTING.md](https://github.com/faker-js/faker/blob/next/CONTRIBUTING.md): "Faker must not contain copyrighted materials"; facts (finite known lists) are OK; compilations must not be copied from a single source (Wikipedia, "most popular" articles, government sites) because "a compilation of facts can be copyrighted" — contributors are told to draw on multiple sources and their own judgement.
+- Verdict: reusing faker's `en` commerce lists (`product_name.adjective/material/product`, `department`) is fine under MIT — keep the MIT notice with the copied data. The lists are short curated words (no external dataset). Nothing in the repo or docs restricts locale-data reuse beyond MIT.
+
+### Garment and shoe sizes
+
+- Letter sizes XS–XXL: EN 13402-3 letter codes with chest/bust ranges (Wikipedia [EN 13402](https://en.wikipedia.org/wiki/EN_13402), CC BY-SA — but the table is a handful of facts; the standard itself is paywalled). Men chest / women bust cm: XXS 70–78 / 66–74, XS 78–86 / 74–82, S 86–94 / 82–90, M 94–102 / 90–98, L 102–110 / 98–107, XL 110–118 / 107–119, XXL 118–129 / 119–131, 3XL 129–141 / 131–143.
+- US numeric women's sizes (0–20, even) come from ASTM D5585 (paywalled); no open dataset. Just generate the even-number series.
+- Shoe sizes: no open dataset found (search hits are ML demos and retailer charts). Use the formulas from Wikipedia [Shoe size](https://en.wikipedia.org/wiki/Shoe_size) (facts, not copyrightable): EU (Paris point) ≈ 1.5 × foot cm + 2 (i.e. 3/20 × mm + 2); UK adult ≈ 3 × foot in − 23; US men ≈ 3 × foot in − 22 = UK + 1; US women (common) = US men + 1.5 (FIA scale: 3 × foot in − 21); Mondopoint = foot length mm. ISO/TS 19407:2023 has the official conversion tables (paywalled). Generate from a foot length (men 24–31 cm, women 21–27 cm) in half-size steps so all systems agree.
+
+### Check-digit rules (all verified against Wikipedia articles; algorithms are facts)
+
+| Code | Rule |
+|---|---|
+| EAN-13 / UPC-A / GTIN-8/12/13/14 | Weights 3,1,3,1… starting from the rightmost data digit (the one next to the check digit); check = (10 − sum mod 10) mod 10. UPC-A = EAN-13 with a leading 0 (GS1 prefix of a 12-digit GTIN is `0` + first two digits). GTIN-14 first digit is an indicator 1–8 (packaging level) or 9 (variable measure); 0 is not valid there. |
+| UPC-A number system (first digit) | 0,1,6,7,8,9 regular; 2 variable-weight; 3 drugs (NDC); 4 in-store/local; 5 coupons. |
+| ISBN-10 | Σ(weights 10..2 × first 9 digits) + check ≡ 0 mod 11; check = (11 − sum mod 11) mod 11, 10 → `X`. |
+| ISBN-13 | EAN-13 with prefix 978 or 979. 979 groups in use: 979-8 USA, 979-10 France, 979-11 Korea, 979-12 Italy. 978 groups 0 and 1 = English-language. For an en_US generator: `978-0…`, `978-1…`, `979-8…`. The official range file (`https://www.isbn-international.org/export_rangemessage.xml`) is XML only and the site says "You must not republish material from our site … without our written permission" — do not embed it; group/registrant length only matters for hyphenation, not validity. |
+| ISSN | Format `NNNN-NNNC`; weights 8..2 over first 7 digits, sum mod 11; check = 0 if remainder 0 else 11 − remainder; 10 → `X`. EAN-13 form: `977` + 7 ISSN digits (no check) + 2 variant digits + EAN check. |
+| IMEI | 15 digits: TAC 8 (RBI 2 + 6) + serial 6 + Luhn check (ISO/IEC 7812; from the right, double every second digit, sum digits, total ≡ 0 mod 10). SVN never enters the check. IMEISV = 14 digits + 2-digit SVN, no check. RBI `00` = test IMEI (Wikipedia [Reporting Body Identifier](https://en.wikipedia.org/wiki/Reporting_Body_Identifier)); live RBIs: 01 CTIA/PTCRB (US), 35 TÜV SÜD/BABT (UK), 86 TAF (China), 99 GHA. For fake data use TAC `00xxxxxx` so it can never match a real handset — UNVERIFIED: third-party pages claim the GSMA TS.06 test-TAC shape is `00 44` + 4 digits; the TS.06 PDF could not be text-extracted here. |
+| ASIN | 10 chars `[A-Z0-9]{10}`; non-books start `B0` + 8 alphanumerics (`^B0[A-Z0-9]{8}$`); books reuse their ISBN-10 verbatim. No check digit (Wikipedia). |
+| SKU | Not standardised (Wikipedia). Convention only: 8–12 uppercase alphanumerics, e.g. `[A-Z]{3}-\d{4}-[A-Z]{2}`; avoid leading 0 and O/I if you want scanner-safe. |
+
+### GS1 prefixes
+
+- Official table: [gs1.org/standards/id-keys/company-prefix](https://www.gs1.org/standards/id-keys/company-prefix) (rendered via Playwright; WebFetch gets 403). Copyright notice ([gs1.org/terms-use](https://www.gs1.org/terms-use)): reproduction allowed only "in unaltered form … for your personal, non-commercial use or use within your organisation"; "You are not permitted to re-transmit, distribute or commercialise the information or material without seeking prior written approval from GS1." → Do not copy the GS1 page. The prefix-to-country mapping is factual and also on Wikipedia ([List of GS1 country codes](https://en.wikipedia.org/wiki/List_of_GS1_country_codes), CC BY-SA 4.0 — avoid verbatim copy). Safe path: embed only the US/special ranges you need, written from the facts below.
+- Ranges relevant to an en_US generator (from the GS1 page): 001–019, 030–039, 060–139 GS1 US (UPC-A compatible); 020–029 restricted circulation within a region; 040–049 restricted within a company; 050–059 GS1 US reserved; 200–299 restricted circulation (region); 952 "used for demonstrations and examples of the GS1 system"; 977 ISSN; 978–979 ISBN; 980 refund receipts; 981–983 coupons (common currency); 990–999 coupons. Prefixes do not identify country of origin. Selected others: 300–379 France, 400–440 Germany, 450–459 & 490–499 Japan, 500–509 UK, 690–699 China, 730–739 Sweden, 750 Mexico, 754–755 Canada, 760–769 Switzerland, 800–839 Italy, 840–849 Spain, 870–879 Netherlands, 880–881 South Korea, 890 India, 930–939 Australia, 940–949 New Zealand.
+- Generator rule: for fake UPC/EAN use prefix `952` (GS1 demo) or a `2xx` restricted-circulation prefix — never a real company prefix; for US-looking UPC-A use number system 0–1/6–8 with a random 5-digit manufacturer code and accept that it may collide with real products.
+
+## 12. Books, music, food
+
+### Project Gutenberg catalog
+
+- URL: `https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv` (20 MB) / `pg_catalog.csv.gz` (5.3 MB), regenerated weekly; also RDF (`rdf-files.tar.bz2`, 121 MB) and MARC.
+- Fields: `Text#, Type, Issued, Title, Language, Authors, Subjects, LoCC, Bookshelves` (comma CSV, quoted; Authors as `Last, First, birth-death`, `;`-separated).
+- Count (2026-09-17): 79 381 rows; Type: Text 78 130, Sound 1 114, Dataset 89, Image 33, other 12; Language: en 62 860, fr 4 190, fi 3 684, de 2 425, it 1 110.
+- Licence: each RDF record carries `<cc:Work><cc:license rdf:resource="https://creativecommons.org/publicdomain/zero/1.0/"/>` → catalog metadata is CC0 1.0 (verified on `cache/epub/1/pg1.rdf`). The CSV itself has no header notice; treat as CC0 by the same source. Terms of use: do not hammer their servers (bulk feeds are the sanctioned path), "Project Gutenberg" is a trademark — royalties for commercial use of the *name*; do not brand the data file with it beyond a source citation.
+- Embed: yes (CC0). For a fake-data library ship a filtered subset (Type=Text, Language=en, title + first author, maybe 5–10k rows) rather than 20 MB.
+
+### Music genres and instruments
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| [MusicBrainz genre list](https://musicbrainz.org/genres) | Genre is a core entity (schema doc lists 13 core entities incl. Genre) → core data dump `mbdump.tar.bz2` is CC0. The genre→entity *associations* come via user tags (CC BY-NC-SA 3.0) — do not use those | Yes, names only | 2 202 (`/ws/2/genre/all?fmt=json`, `genre-count`) | WS API needs a descriptive User-Agent; list is a JSON array of `{id, name, disambiguation}` |
+| [Discogs data dumps](https://data.discogs.com/) | CC0 | Yes | genre/style vocab is embedded in release XML (monthly dumps, GBs) — no standalone style list; UNVERIFIED count (~15 genres, ~600 styles from memory) | Impractical to extract; prefer MusicBrainz |
+| ID3v1 genres | Names 0–79 from the 1999 ID3v1 spec, 80–125 Winamp, 126–191 Winamp 5.6 (2010) — a de-facto spec table, treated as public domain facts | Yes | 192 (0–191) | Wikipedia [List of ID3v1 genres](https://en.wikipedia.org/wiki/List_of_ID3v1_genres) (CC BY-SA page; the list itself is a spec enumeration). id3.org returned HTTP 500 during research — UNVERIFIED against the original spec page |
+| Wikidata instruments | CC0 | Yes | 9 194 items `P31/P279* Q34379` with English labels; 2 372 with a Hornbostel–Sachs number (P1762) | Filter by P1762 for a clean orchestral/folk instrument list |
+| Wikidata music genres | CC0 | Yes | 6 619 (`Q188451`) | Noisier than MusicBrainz |
+
+### USDA FoodData Central
+
+- Downloads: [fdc.nal.usda.gov/download-datasets](https://fdc.nal.usda.gov/download-datasets/) (April 2026 release):
+  - Foundation Foods: JSON 459 KB zip / 6.5 MB; CSV 3.7 MB zip / 32 MB — 394 foods (API `totalHits`, dataType=Foundation).
+  - SR Legacy (final, April 2018): JSON 12.3 MB zip / 205 MB; CSV 6.7 MB zip / 54 MB — 7 793 foods.
+  - FNDDS 2021-2023: CSV 200 MB zip / 1.6 GB — 5 432 foods.
+  - Branded: CSV 428 MB zip / 2.9 GB — 433 403 foods.
+  - Full: CSV 460 MB zip / 3.1 GB.
+- Licence ([API guide](https://fdc.nal.usda.gov/api-guide/)): "USDA FoodData Central data are in the public domain and they are not copyrighted. They are published under CC0 1.0 Universal (CC0 1.0)". No permission needed; USDA *requests* listing FoodData Central as source and notifying them. Suggested citation: "U.S. Department of Agriculture, Agricultural Research Service. FoodData Central, 2019. fdc.nal.usda.gov."
+- Embed: yes (CC0). Ship a derived list (SR Legacy `description` + `food_category`) not the raw dump.
+- CSV structure (UNVERIFIED — the field-description PDF could not be text-extracted; from memory): `food.csv` (fdc_id, data_type, description, food_category_id, publication_date), `food_category.csv` (id, code, description), `nutrient.csv` (id, name, unit_name, nutrient_nbr, rank), `food_nutrient.csv` (id, fdc_id, nutrient_id, amount, …), `food_portion.csv`, `measure_unit.csv`, `sr_legacy_food.csv` (fdc_id, NDB_number), `foundation_food.csv`.
+
+### Dishes, cuisines, drinks
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| Wikidata dishes (`Q746549`) | CC0 | Yes | 7 480 with English label | SPARQL `?i wdt:P31/wdt:P279* wd:Q746549` |
+| Wikidata cuisines (`Q1968435`) | CC0 | Yes | 219 | |
+| Wikidata cocktails (`Q134768`) | CC0 | Yes | 305 | |
+| Wikidata beer (`Q44` subclass tree) / wine (`Q282`) | CC0 | Yes | 420 / 3 025 | Wine tree is mostly appellations/brands; filter by P279 depth |
+| IBA official cocktails | IBA site "© IBA 2026 – All rights reserved"; Wikipedia list CC BY-SA 4.0 | Names only are facts (102 cocktails: 34 Unforgettables, 34 Contemporary Classics, 34 New Era, 2024 revision); do not copy recipes/descriptions | 102 | Safest: take the names from Wikidata (`P31 Q134768`) rather than the IBA page |
+| [Open Brewery DB](https://github.com/openbrewerydb/openbrewerydb) | MIT (LICENSE, © 2025 Open Brewery DB) | Yes | 11 931 breweries (US 8 308, DE 1 445, AU 514, BE 478, CA 283, NZ 243) | `breweries.csv`: id (UUID), name, brewery_type (micro 5 906, brewpub 3 947, closed 644, planning 635, regional 240, contract 208, large 137, proprietor, taproom, bar, nano, cidery, beergarden), address_1..3, city, state_province, postal_code, country, phone, website_url, longitude, latitude. Real businesses — fine for names, but generating "fake" data with real addresses/phones may be undesirable |
+| Wikipedia lists (cuisines, dishes, IBA) | CC BY-SA 4.0 | No verbatim copies | | |
+
+## Summary of safe picks
+
+- Countries: `datasets/country-codes` (PDDL) or `annexare/Countries` (MIT); flag emoji derived from alpha-2.
+- Languages: `langs` / `iso-639-1` (MIT, 639-1 only).
+- Subdivisions: US-only list from Census (public domain); skip world list.
+- Commerce words: faker `en` lists (MIT). Check digits: implement from the rules above; GS1 prefix table: hand-write the few ranges needed, use `952`/`2xx` for generated barcodes; ISBN range file: do not embed.
+- Books: Gutenberg `pg_catalog.csv` (CC0), filtered.
+- Music: MusicBrainz genre names (CC0, 2 202), ID3v1 list (192), Wikidata instruments (CC0).
+- Food: USDA FDC SR Legacy/Foundation descriptions (CC0, cite USDA), Wikidata dishes/cuisines (CC0), Open Brewery DB (MIT).
+- Avoid: mledoze (ODbL), lukes (CC BY-SA), stefangabos (CC BY-SA/LGPL), Debian iso-codes (LGPL), SIL 639-3 table (no-redistribution clause), GS1/ISBN-International pages (all rights reserved), Wikipedia tables verbatim (CC BY-SA), MusicBrainz tag associations (CC BY-NC-SA).

docs/research/research-age-bands.md

@@ -0,0 +1,10 @@
+# Population by age band — research 2026-10-04
+
+Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
+
+| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
+|---|---|---|---|---|---|---|---|
+| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
+| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
+
+At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.

docs/research/research-geo-se.md

@@ -0,0 +1,114 @@
+# Open data sources for real Swedish geographic and postal-address data
+
+Researched 2026-09-16. Goal: Sweden → län → kommun → tätort/postort → street, with real postal codes, embeddable in an MIT-licensed library.
+
+## TL;DR
+
+| Need | Best source | Licence | Embed in MIT repo? |
+|---|---|---|---|
+| Län + kommun names/codes | SCB `kommunlankod-2026.xlsx` | CC0 1.0 | Yes, no attribution needed |
+| Population per kommun (weighting) | SCB PxWeb API `BE0101A/BefolkningNy` | CC0 1.0 | Yes |
+| Tätorter (2,017) with code, kommun, län, population | SCB open geodata WFS/GeoPackage `stat:Tatorter_2023` + PxWeb `MI0810A/LandarealTatortN` | CC0 1.0 | Yes |
+| Streets + house numbers + postnummer + postort + kommun + coordinates (~3.9M points) | Lantmäteriet *Belägenhetsadress Nedladdning, vektor* (STAC, GeoPackage per kommun) | CC BY 4.0 **plus** personal-data terms, purpose review, Basic-auth download | **Not as-is** — raw redistribution is bound by the personal-data terms; a derived aggregate (street ↔ postnummer ↔ postort, no house numbers) is the realistic path but needs a stated purpose approved by Lantmäteriet. Legal review needed. |
+| postnummer → postort (list only) | GeoNames `SE.zip` (18,887 rows, 1,780 postorter) | CC BY 4.0 | Yes with attribution — but source is an undated third-party file, likely stale |
+| Street names per kommun/postort without Lantmäteriet | Trafikverket NVDB *Gatunamn* (Lastkajen) | CC0 1.0 | Yes, but no postnummer; needs account to download |
+| Street names | OpenStreetMap | ODbL 1.0 | Effectively no (share-alike on the extracted database) |
+| Municipal address files | OpenAddresses `sources/se/*` (16 kommuner) | Mixed: CC0 (Göteborg, Malmö), CC BY 4.0 (Helsingborg), unspecified (Stockholm 2016 dump, Uppsala) | Per source |
+| Postal code register (authoritative) | PostNord → Postnummerservice Norden AB / Geposit AB | Commercial; "resale/sublicensing not permitted" | No |
+
+## 1. Lantmäteriet
+
+Since 2025-02-03 the EU High Value Datasets (HVD) products are fee-free but Lantmäteriet explicitly says they are *not* "öppna data" because conditions attach ([Avgiftsfria produkter](https://www.lantmateriet.se/sv/geodata/vara-produkter/avgiftsfria-produkter/)). Two tiers exist:
+
+- **Öppna data** — licence "Creative Commons, CC0" ([Öppna data](https://www.lantmateriet.se/sv/geodata/vara-produkter/avgiftsfria-produkter/oppna-data/)). Verified on Geotorget product pages for *Topografi 50/100/250/1M Nedladdning, vektor* ("Villkor: Creative Commons, CC0", "Juridisk prövning: Nej", GeoPackage, download in Geotorget or via the *Geotorget Nedladdning* API).
+- **Värdefulla datamängder (HVD)** — licence **CC BY 4.0** per the terms document *Användningsvillkor för värdefulla datamängder* (DNR LM2025/009266 v1.0, 2025-02-01, [PDF](https://www.lantmateriet.se/globalassets/geodata/geodataprodukter/anvandningsvillkor_for_vardefulla_datamangder.pdf)). Products with personal data use *Användningsvillkor för värdefulla datamängder som innehåller personuppgifter* (DNR LM2025/009269 v1.1, 2025-03-04, [PDF](https://www.lantmateriet.se/globalassets/geodata/geodataprodukter/anvandningsvillkor_for_vardefulla_datamangder_pu.pdf)).
+
+Per-product status (Geotorget product pages, rendered 2026-09-16):
+
+| Product | Fee | Terms | Legal review | Access | Format |
+|---|---|---|---|---|---|
+| [Belägenhetsadress Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/belagenhetsadress-nedladdning-vektor-api) | 0 | HVD terms **with personal data** | **Yes** | STAC API `https://api.lantmateriet.se/stac-vektor/v1`, collection `belagenhetsadresser` | GeoPackage, one zip per kommun |
+| [Belägenhetsadress Direkt](https://geotorget.lantmateriet.se/geodataprodukter/belagenhetsadress-direkt-api) | 0 | HVD terms with personal data | Yes | REST `https://api.lantmateriet.se/distribution/produkter/belagenhetsadress/v4.2` (OAuth2 via API portal) | JSON / XML |
+| [Belägenhetsadress Nedladdning, Inspire](https://www.lantmateriet.se/sv/geodata/vara-produkter/produktlista/belagenhetsadress-nedladdning-inspire/) | 0 | HVD terms with personal data | Yes (3–5 working days extra for security review) | Atom feed, login | GML; SWEREF 99 TM / ETRS89; semi-annual |
+| [Ortnamn Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/ortnamn-nedladdning-vektor-api) | 0 | HVD terms (no personal data) | No | STAC collection `ortnamn`, one national zip (58.5 MB) | GeoPackage |
+| [Ortnamn Direkt](https://geotorget.lantmateriet.se/geodataprodukter/ortnamn-direkt-api) | 0 | HVD terms | No | REST `…/distribution/produkter/ortnamn/v2.2` | JSON / XML |
+| Kommun, län och rike (administrativ indelning) | 0 | STAC says `CC-BY-4.0`; a Lantmäteriet HVD presentation (GISS, 2025-03-04) tables it as **CC0** ("CC0 används som licens för tjänster som är gemensamma för HVD och NGP … Kommun, Län och Rike") — **conflicting, unverified which prevails** | No | STAC collection `kommun-lan-rike` (zip ~6 MB, yearly + "aktuell"); OGC API Features `https://api.lantmateriet.se/ogc-features/v1/administrativ-indelning` (collections `kommuner`, `lan`, `rike`, plus `-2025`/`-2026`; items endpoint returns 401) | GeoPackage / GeoJSON |
+| [Topografi 10 Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/topografi-10-nedladdning-vektor) | 0 | HVD terms with personal data | **Yes** | Geotorget download / API | GeoPackage |
+| Topografi 50 / 100 / 250 / 1M Nedladdning, vektor | 0 | **CC0** | No | Geotorget download / *Geotorget Nedladdning* API | GeoPackage |
+
+Notes:
+- *Administrativ indelning* is not a standalone Geotorget product; it is the STAC collection `kommun-lan-rike`, the OGC API above, an [Inspire download](https://www.lantmateriet.se/sv/geodata/vara-produkter/produktlista/administrativ-indelning-nedladdning-inspire/) (Atom/WCS), or a layer in Topografi 100/250/1M (CC0). Topografi 1M also carries "tätortsgränser och tätortssymboler".
+- The STAC catalog root and collection/item listings are public (HTTP 200, no auth); it self-describes: "avgiftsfria och får användas enligt creative commons licens CC BY 4.0. För vissa informationsmängder kommer din användning att prövas juridiskt … och du behöver då godkänna särskilda användningsvillkor." Asset downloads on `dl1.lantmateriet.se` return **401 Basic** — credentials are the Geotorget account (private person) or a system account (organisation). Private accounts are "endast för privat bruk" ([konto-privatperson](https://geotorget.lantmateriet.se/konto-privatperson)); an MIT library is not private use, so apply as an organisation.
+- **Record count.** 290 STAC items (one per kommun), total 324 MB zipped. Smallest Bjurholm 143 KB, largest Stockholm 9.1 MB, Göteborg 8.9 MB, Malmö 4.9 MB. Fastighetsregistret held 3,882,361 valid addresses in 2022 (2,886,796 street addresses + 964,737 rural) per [SCB/LM presentation](https://kartografiska.se/wp-content/uploads/5C_Foretagsadresser_SCB_LM.pdf); the OSM import thread cites 3.7 M. So ~3.9 M address points, not 4–5 M.
+- **Fields** (JSON schema `belagenhetsadress-4.2.2.json`, matches *Nationell specifikation Adress* v1.0.1): `kommunkod`, `kommunnamn`, `kommundel` (geografisk kommundel), `adressomrade` + `adressomradestyp` (gatuadressområde / metertalsadressområde / byadressområde), `gardsadressomrade`, `adressplatsnummer`, `bokstavstillagg`, `lagestillagg`/`lagestillaggsnummer`, `adressplatstyp`, `adressplatsbeteckning`, `postnummer`, `postort` (both "beslutas av PostNord AB", only on current addresses), `popularnamn`, `distriktskod`/`distriktsnamn`, `adressattAnlaggning`, `anmarkningstyp`/`anmarkningstext`, `registerenhetsreferens` (property link), `objektidentitet`, `objektstatus`, `versionGiltigFran`, `geometri` (SWEREF 99 TM point). No län field — derive from kommunkod (first two digits).
+- Lantmäteriet treats addresses as personal data via Fastighetsregisterlagen § 6: usage only for the purpose approved in the application, EU/EEA-only storage, spot checks, revocation (terms PU §§ 3.3–4.2). The OSM community reports Lantmäteriet refused ODbL compatibility and OpenAddresses ([#7657](https://github.com/openaddresses/openaddresses/issues/7657), [#7608](https://github.com/openaddresses/openaddresses/issues/7608)) has not been able to include it; the OSM thread ([127427](https://community.openstreetmap.org/t/lantmateriet-belagenhetsadress/127427), 91 posts, last 2026-07-25) says one mapper got approval for a full 3.7 M import but no formal import has happened.
+- **Attribution text required (CC BY 4.0 + terms § 3.1):** product name as data source, "©Lantmäteriet", whether the data was processed, and that CC BY 4.0 applies — may be placed in accompanying documentation/metadata.
+- Ortnamn: name types include `Tätort` ("minst 200 invånare"), Bebyggelse, Trakt, Kyrka, Anläggning, nature names; Ortnamnsregistret holds ~980,000 names (Lantmäteriet, figure not re-verified). Not delivered as CSV.
+
+## 2. SCB (Statistics Sweden)
+
+- **Licence.** Statistics and geodata published as open data: "Creative Commons 0 1.0 Universal, CC0" ([användningsvillkor](https://www.scb.se/om-scb/om-scb.se-och-anvandningsvillkor)); since 2021-07-01. Optional credit "Källa: SCB" / "Source: Statistics Sweden"; do not cite SCB as source for data you have processed. API limit 10 requests / 10 s per IP.
+- **Län and kommuner with codes.** 21 län, 290 kommuner ([lan-och-kommuner](https://www.scb.se/hitta-statistik/regional-statistik-och-kartor/regionala-indelningar/lan-och-kommuner/)). Download: `https://www.scb.se/contentassets/7a89e48960f741e08918e489ea36354a/kommunlankod-2026.xlsx` (18 KB, HTTP 200). Codes: län 2 digits (01 Stockholm … 25 Norrbotten), kommun 4 digits = län + 2 (0180 Stockholm).
+- **Population per kommun.** PxWeb API `https://api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0101/BE0101A/BefolkningNy` — region dimension has 312 values (riket, 21 län, 290 kommuner), years 1968–2024 (verified 2026-09-16). Also `BE0101C/BefArealTathetKon` (density, 1991–2025).
+- **Tätorter.** 2,017 tätorter in 2023 (WFS `resultType=hits` on `stat:Tatorter_2023` → `numberOfFeatures="2017"`; SCB also states 2,017). Open geodata: `https://geodata.scb.se/geoserver/stat/wfs` (WFS/WMS) and GeoPackage download ([statistiska tätorter](https://www.scb.se/vara-tjanster/oppna-data/oppna-geodata/statistiska-tatorter/)). Attributes: `tatortskod`, `tatort`, `kommun`, `kommunnamn`, `lan`, `lannamn`, `area_ha`, `bef` (population), `ar`. Population per tätort also in PxWeb `MI0810/MI0810A/LandarealTatortN` (2,827 region codes, every 5 years to 2023). Also `Smaorter_2023`, `DeSO_2025`, `RegSO_2025`.
+
+## 3. Postal codes
+
+- **Owner.** PostNord Sverige owns and operates the postnummer system; PTS is supervisory authority; a Postnummerråd approves changes 3–4 times a year (sv.wikipedia). Register management is delegated to **Postnummerservice Norden AB**, machine access via **Geposit AB**. Postnummerservice's [terms](https://postnummerservice.se/en/information/kopvillkor): "free use within your own organization", "resale of data is not permitted", "sublicensing to third parties is not permitted" — unusable for an open library.
+- **No open list from PostNord, Lantmäteriet or Digg.** dataportal.se search for "postnummer" (2026-09-16) yields only Postnummerservice's own commercial catalogue entry ("Svenska postnummer och postorter", published 2012), Helsingborg's municipal postnummer polygons, and Lantmäteriet's Inspire address entry. The old dataportal community called the postnummer system "slarvats bort till ett privat bolag". GitHub CSVs (`zegl/sweden-zipcode`, `lapplandi/sveriges-postnummer`, `beshrkayali/sverige_postnummer`) state no licence or provenance — do not use.
+- **Lantmäteriet addresses carry `postnummer` + `postort`** on every current address point (set by PostNord), so the address product is the only lawful open-ish route to a *complete, current* postnummer ↔ postort ↔ street mapping.
+- **GeoNames `SE.zip`** ([readme](https://download.geonames.org/export/zip/readme.txt)): CC BY 4.0 ("a link on your website to www.geonames.org is ok"). Downloaded and counted: **18,887 rows, 18,887 distinct postnummer, 1,780 postorter, 22 admin1 values (21 län + blank), 285 admin2 (kommun code) values; 3,055 rows lack kommun; accuracy: 16,115 rows = 4 (centroid of postal code area), 389 = 1, 2,383 blank**. Fields: country, postal code (`111 64` with space), place name, admin1 name/code (län), admin2 name/code (kommun 4-digit), admin3, lat, lon, accuracy. The GeoNames sources page lists Sweden's source as `www.pellesoft.se/upload/program/prg00745.zip` with **no date** (URL now dead) — a hobbyist upload, so freshness is unknown; the row count (18.9 k) exceeds today's ~17 k total codes, indicating retired codes are still present. The zip is rebuilt daily (Last-Modified 2026-09-16) but that says nothing about the underlying data. 96 KB zipped.
+- **Counts.** Postnummerservice: "~17 000 postal codes", "10 500 deliverable postal codes", "1 743 postal cities"; Kartanalys: 10,839 geographic postnummer, 1,740 postorter; Wikipedia: >16,100 areas (2008). Changes quarterly.
+- **Structure** (sv/en Wikipedia): 5 digits written `NNN NN`. Lower numbers further south, except 1xx xx = Stockholm. First digit/tens: 10–19 Stockholm, 20–29 Skåne (Malmö 20–21), 30–39 southern Sweden, 40–49 Göteborg area (Göteborg 40–41), 50–59, 60–69, 70–79, 80–89, 90–99 progressively north to Norrbotten (98x). Postorter come in 2-, 3- or 5-position sizes; in two-position places the 3rd digit (in three-position the 4th) marks delivery type: 0 = boxes/postal, 1 = boxes/business, 2–4 and 6–7 = street delivery, 5 = rural (lantbrevbäring), 8 = reply mail, 9 = competitions/temporary — with exceptions in big cities. Box codes are therefore not street-deliverable and should be excluded when generating street addresses.
+
+## 4. Streets
+
+- **Lantmäteriet** — see § 1; `adressomrade` with `adressomradestyp = gatuadressområde` is the street name.
+- **Trafikverket NVDB** — CC0 1.0 Universal since 2017-04-03 ([trafikverket.se](https://www.trafikverket.se/e-tjanster/hamta-data-fran-trafikverket/) links `creativecommons.org/publicdomain/zero/1.0/deed.sv`; [OSM wiki](https://wiki.openstreetmap.org/wiki/Sweden/trafikverket)). Data product **Gatunamn** = "det officiellt adressbildande namnet på gatan" (municipal decision); Trafikverket is cleaning it in 2026 so it holds only names used in street addresses ([Vägnamn i NVDB](https://www.nvdb.se/sv/aktuellt/nyhetsarkiv/2026/vagnamn-i-nvdb/)). Download via Lastkajen (registration with e-mail + accept licence; Shapefile/GeoPackage), or the Datautbytesportal API. Gives street ↔ kommun (and geometry) but **no postnummer/postort**. Record count not published on the pages checked.
+- **OpenStreetMap** — ODbL 1.0. A list of street names extracted from OSM is a Derivative Database: "Where you make our data or any Derivative Database available to others, it must continue to be licensed under the ODbL" ([OSMF FAQ](https://osmfoundation.org/wiki/Licence/Licence_and_Legal_FAQ)); only *Produced Works* (maps etc.) escape share-alike. Embedding an OSM-derived JSON in an MIT repo therefore forces that data file to ODbL with attribution "© OpenStreetMap contributors" + link to openstreetmap.org/copyright, and downstream consumers inherit share-alike on the data. Not recommended.
+- **OpenAddresses** — 16 Swedish sources, all municipal (`sources/se/municipality_of_*.json`: Alingsås, Gislaved, Göteborg, Helsingborg, Höganäs, Kalmar, Kristinehamn, Malmö, Nacka, Sävsjö, Stockholm, Uppsala, Västerås, Vaxholm, Växjö, Österåker). OA does not relicense; licence is per source. Checked: Göteborg **CC0 1.0** (attribution "Göteborgs stad", shapefile, `gatunamn`), Malmö **CC0 1.0** (CSV with `ADRESSOMR`, `ADRESSPLAT`, `POSTNR`, `POSTORT`, cached 2024-11), Helsingborg **CC BY 4.0** ("Helsingborgs stad", GeoJSON with Gatunamn/Postnummer/Stad), Uppsala attribution required, licence name not stated (ArcGIS REST), Stockholm **no licence field**, data is a 2016 cache of a defunct WS. Usable for a few big cities only; no national coverage.
+- **Unique street-name counts.** SCB (Lägenhetsregistret 2019, [artikel](https://www.scb.se/hitta-statistik/artiklar/2021/pa-ringvagen-bor-det-flest/)): "Antalet gator eller vägar med en eller flera adresser är 399 175 i hela landet" — this is streets counted per place, not unique names. Most frequent: Ringvägen 205, Skogsvägen 204, Björkvägen 200, Skolgatan 196, Storgatan 180 (in 180 kommuner, 55,360 residents). A national unique-name count was not found; expect well under 399 k (probably 100–200 k) — **unverified**.
+
+## 5. Sizes
+
+| Level | Count | Source |
+|---|---|---|
+| Län | 21 | SCB |
+| Kommuner | 290 | SCB; 290 STAC items |
+| Tätorter (2023) | 2,017 | SCB WFS hits |
+| Postorter | ~1,740–1,780 | Postnummerservice 1,743; Kartanalys 1,740; GeoNames 1,780 |
+| Postnummer | ~17,000 total, ~10,500–10,840 deliverable/geographic | Postnummerservice, Kartanalys |
+| Streets with addresses (per place) | 399,175 (2019) | SCB |
+| Unique street names | not published; likely 100–200 k | unverified |
+| Address points | ~3.9 M (3,882,361 in 2022) | Lantmäteriet/SCB |
+| Lantmäteriet address GeoPackages | 324 MB zipped, 290 files | STAC |
+| GeoNames SE.zip | 96 KB (18,887 rows) | download |
+| Ortnamn national GeoPackage | 58.5 MB zipped; ~980 k names | STAC; Lantmäteriet |
+
+Embedding suggestion (size-driven, before licence): län + kommun + population (~20 KB) and tätorter with population (~100 KB) are trivially embeddable under CC0. A postnummer→postort table is ~300 KB raw. A national (street, postort, postnummer-range) table at ~400 k rows is roughly 15–25 MB JSON, ~3–5 MB gzipped — optional download pack territory; a default pack could carry the top N streets per kommun (e.g. 20 × 290 ≈ 6 k rows, <300 KB).
+
+## 6. Licence compatibility with an MIT repo
+
+| Source | Licence | Embed? | Required notice |
+|---|---|---|---|
+| SCB codes, population, tätorter | CC0 1.0 | Yes | None; optional "Källa: SCB" |
+| Lantmäteriet Topografi 50/100/250/1M (admin boundaries, tätort symbols) | CC0 | Yes | None (courtesy credit suggested) |
+| Lantmäteriet Ortnamn | CC BY 4.0 + HVD terms (no personal data, no review) | Yes, data file stays CC BY 4.0 inside the MIT repo | "Källa: Ortnamn Nedladdning, vektor, ©Lantmäteriet, bearbetad, CC BY 4.0" (product name, ©Lantmäteriet, processed-flag, CC BY 4.0) |
+| Lantmäteriet kommun-län-rike | CC BY 4.0 per STAC (CC0 per LM slide — unresolved) | Yes | Same attribution form as above unless CC0 is confirmed |
+| Lantmäteriet Belägenhetsadress | CC BY 4.0 + personal-data terms, purpose approval, EU-only storage, revocable | **Only with an approved application** whose stated purpose covers deriving and publishing an aggregated street/postnummer/postort list under CC BY 4.0. Raw points (house numbers, property links) should not be redistributed. Get written confirmation from geodatasupport@lm.se; treat as unresolved until then. | Product name, ©Lantmäteriet, "bearbetad", CC BY 4.0 |
+| GeoNames SE | CC BY 4.0 | Yes | Link/credit to www.geonames.org; note staleness |
+| Trafikverket NVDB Gatunamn | CC0 1.0 | Yes | None |
+| OpenStreetMap | ODbL 1.0 | Not practical (share-alike on the data) | "© OpenStreetMap contributors" + ODbL if ever used |
+| OpenAddresses Göteborg, Malmö | CC0 1.0 | Yes | None (Göteborg asks credit "Göteborgs stad") |
+| OpenAddresses Helsingborg | CC BY 4.0 | Yes | "Helsingborgs stad" |
+| OpenAddresses Stockholm/Uppsala | unspecified | No | — |
+| PostNord/Postnummerservice files | commercial, no redistribution | No | — |
+
+CC BY 4.0 and CC0 data files can sit in an MIT repo; the MIT licence covers the code, and a `DATA-LICENSES.md`/NOTICE lists each dataset with its licence and the attribution above. CC BY 4.0 § 2(a)(5)(B) (no technical protection measures) is irrelevant for a plain JSON file.
+
+## Could not verify
+- Whether the kommun-län-rike collection is CC0 or CC BY 4.0 (sources disagree).
+- Whether Lantmäteriet will approve an application whose purpose is publishing a derived open dataset; OSM/OpenAddresses experience suggests refusal is likely.
+- GeoNames Sweden source date; assume stale.
+- National count of unique street names; Gatunamn (NVDB) record count.
+- Exact number of active postnummer today (three sources: ~17 k total / 10.5–10.8 k deliverable).

docs/research/research-geo-world.md

@@ -0,0 +1,208 @@
+# Open geographic / postal-address data outside Sweden — for fejkdata (MIT)
+
+Researched 2026-09-16. "Embeddable in MIT repo" below means: the data licence permits redistribution and commercial use with at most an attribution notice, so the data can ship inside an MIT-licensed package with a separate `DATA-LICENSES` file. Anything share-alike (ODbL, CC BY-SA) is flagged as **not** embeddable. Items marked *(unverified)* were not confirmed against a primary source.
+
+## 0. Summary table
+
+| Country | Official open address register | Licence | Addresses | Postcode in register | Postcode data open? | Embed in MIT? |
+|---|---|---|---|---|---|---|
+| US | DOT National Address Database (NAD) | US public domain | ~80 M (Jun 2026) | ZIP (optional field; fill rate *unverified*) | ZCTA yes (public domain); USPS publishes no open ZIP file | Yes |
+| Norway | Kartverket Matrikkelen – Adresse | CC BY 4.0 | 2.58 M (Jun 2024) | Yes (postnummer + poststed) | Yes: Bring postnummerregister, NLOD 2.0 | Yes |
+| Denmark | Danmarks Adresseregister (DAR) via DAWA / Datafordeler | CC BY 4.0 | ~3.8 M | Yes | Yes (in DAR; 1,089 postnumre) | Yes |
+| Finland | DVV building addresses (final release Feb 2025) + Posti PCF/BAF | CC BY 4.0 / Posti terms (free; redistribute with terms attached) | ~5 M buildings | Yes | Yes (Posti PCF/BAF) | Yes (Posti: attach terms) |
+| UK (GB) | No open *address* register. OS Open UPRN (points only), OS Open Names (streets/places/postcodes), Code-Point Open (postcodes) | OGL v3 | 40 M UPRNs (no addresses); 870 k roads; 1.7 M postcodes | n/a | Yes for GB (Code-Point Open, OGL); Northern Ireland restricted | Yes (GB only; no house numbers) |
+| Germany | No national register. 15 Länder publish Hauskoordinaten/Gebäudereferenzen (dl-de/by-2-0, dl-de/zero-2-0); Bavaria missing | mixed, mostly permissive | ~19 M points across Länder | Yes in Hauskoordinaten | **No** — Deutsche Post DATAFACTORY is proprietary; open PLZ lists are OSM-derived (ODbL) | Partly (Länder files yes; a national PLZ list is the problem) |
+| Netherlands | BAG (Kadaster/PDOK) | CC0 1.0 | ~9.9 M | Yes | Yes (in BAG) | Yes |
+| France | Base Adresse Nationale (BAN) | Licence Ouverte / Etalab 2.0 | >25 M | Yes | Yes (BAN; La Poste "base officielle des codes postaux", open licence) | Yes |
+| Spain | Catastro INSPIRE Addresses (AD) + INE callejero | Catastro own licence (free, attribution) / INE legal notice | ~15.7 M points (mailwoman count) | Yes (AD:PostCode) | Correos' postcode layer closed since 2017; postcodes still appear per address in Catastro AD | Probably (read Catastro licence PDF first) |
+| Australia | G-NAF (Geoscape via data.gov.au) | EULA based on CC BY 4.0 + "no mail-out" restriction | 15.95 M (Aug 2026) | Yes | In G-NAF; Australia Post file is non-commercial only | Yes (with EULA notice) |
+| Canada | StatCan National Address Register (NAR) 2024; ODA v1 (2021) | StatCan Open Licence / OGL-Canada | 17.1 M / ~10 M | NAR: MAIL_POSTAL_CODE "for most addresses" | Canada Post's postal code file is **proprietary**; only what NAR/ODA carry | Yes (NAR); postcode completeness unverified |
+
+## 1. Per-country detail
+
+### United States
+
+**Official address register — National Address Database (NAD), US DOT**
+- Licence: US public domain (`http://www.usa.gov/publicdomain/label/1.0/`), stated on data.gov: https://catalog.data.gov/dataset/national-address-database-nad-text-file. No attribution required. Embeddable.
+- Size: ~80 M records; current release compiled 2026-06-30, dataset updated 2026-09-03. Sources: https://data.transportation.gov/dataset/National-Address-Database-NAD-Text-File/fc2s-wawr/about_data, https://www.placekey.io/datasets/national-address-database (80 M; "24 fields" in the text export).
+- Coverage: aggregated from state/local providers; a mix of fully, partially and non-participating states (placekey page; the DOT page https://www.transportation.gov/gis/national-address-database blocks scrapers, so the coverage map was not read).
+- Formats: zipped flat text and File Geodatabase. The text file is ordered by spatial cluster, not by OID.
+- Fields (NAD schema https://www.transportation.gov/sites/dot.gov/files/2023-07/NAD_Schema_202304.pdf, blocked for bots; list taken from the FGDC content-requirements doc https://www.fgdc.gov/organization/working-groups-subcommittees/address-sc/220810-nad-content-requirements-approved.pdf): address number parts (`AddNum_Pre`, `Add_Number`, `AddNum_Suf`), street name parts (`St_PreDir`, `St_PreTyp`, `St_Name`, `St_PosTyp`, `St_PosDir`, `StNam_Full`), sub-address (`Building`, `Floor`, `Unit`, `Room`), place names (`Inc_Muni`, `Post_City`, `Census_Plc`, `Uninc_Comm`), `County`, `State`, `Zip_Code`, `Plus_4`, `Longitude`, `Latitude`, `NatGrid`, `Addr_Type`, `Placement`, `NAD_Source`, `DataSet_ID`. FGDC: "A Zip Code is recommended but not mandatory"; a complete place name and state are mandatory. Expect some records without ZIP.
+- Census Address Count Listing files give housing-unit counts per block, not addresses: https://census.gov/geographies/reference-files/2025/geo/addcountlisting.html. The Census MAF itself is not public.
+
+**Administrative hierarchy — Census Bureau (public domain)**
+- States: 50 + DC + PR + 4 island areas = 56 FIPS state codes ("States & Equivalents: 56" in the 2020 tallies). ANSI list: https://www.census.gov/library/reference/code-lists/ansi/ansi-codes-for-states.html (also carries FM, MH, PW, UM).
+- Counties: 3,144 in the 50 states + DC; 3,234 including PR and island areas (2020 tallies https://www.census.gov/geographies/reference-files/time-series/geo/tallies.html; https://en.wikipedia.org/wiki/List_of_United_States_counties_and_county_equivalents). Connecticut switched to 9 planning regions as county equivalents in 2022 — use a current vintage.
+- Places: 19,734 incorporated places + 12,454 CDPs (2020 tallies, incl. PR/island areas); "approximately 19,500" incorporated places in the 50 states (https://www.census.gov/library/stories/2020/05/america-a-nation-of-small-towns.html).
+- TIGER/Line 2025 shapefiles and the pipe-delimited **Gazetteer files** (2026 vintage: states, counties, county subdivisions, places, ZCTAs; GEOID, name, area, centroid lat/lon; **no population**): https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html. Gazetteer excludes island areas, includes PR.
+- Population per place: **SUB-EST2025** CSVs (incorporated places + MCDs, April 2020 base through July 2025, released May 2026): https://www.census.gov/data/tables/time-series/demo/popest/2020s-total-cities-and-towns.html. Public domain. The right file for population-weighted draws.
+
+**Postal codes**
+- USPS publishes no open ZIP list file; counts quoted 41,541–41,695 active ZIPs (https://facts.usps.com/42000-zip-codes/ says 41,554). Commercial "USPS-licensed" databases exist.
+- Census **ZCTA**: 33,791 ZCTAs (2020), public domain, in TIGER + Gazetteer + relationship files (ZCTA↔place, ZCTA↔county): https://www.census.gov/programs-surveys/geography/technical-documentation/records-layout/2020-zcta-record-layout.html. PO-box-only ZIPs have no ZCTA (https://en.wikipedia.org/wiki/ZIP_Code_Tabulation_Area).
+- GeoNames `US.txt` (CC BY 4.0): ZIP → place name, state, county, lat/lon. Count not stated on the site *(≈41 k, unverified)*.
+
+**Street names**
+- TIGER/Line `featnames` relationship files (per county) hold every feature name with prefix/suffix parts; joined to `edges` and `addrfeat` (address ranges). Public domain. No published count of unique street names *(unverified; must be computed)*.
+- NAD `St_Name`/`StNam_Full` distinct per `Post_City` is the cheaper source for "real streets in Denver".
+
+**Sizes for the embed-vs-download decision (US)**
+
+| Level | Count | Source |
+|---|---|---|
+| States + DC + PR + island areas | 56 | Census tallies 2020 |
+| Counties / equivalents | 3,144 (50+DC) / 3,234 (with PR + islands) | Census tallies, Wikipedia |
+| Incorporated places | 19,734 (all) / ~19,500 (50+DC) | Census tallies |
+| CDPs | 12,454 | Census tallies |
+| ZCTAs / USPS ZIPs | 33,791 / ~41,500 | Census tallies / USPS facts |
+| Address points (NAD) | ~80 M | DOT |
+| OpenAddresses US collections | NE 2.35 GB + South 7.11 GB + West 8.54 GB + Midwest 4.86 GB (zipped CSV); 2,381 US sources | https://batch.openaddresses.io/ (2026-09) |
+| Unique street names | unknown; compute from NAD/TIGER | — |
+
+### Norway
+- **Matrikkelen – Adresse** (Kartverket). Licence CC BY 4.0 (https://data.norge.no/en/datasets/8f1151bf-ad63-3e47-adbb-143f876776ee/matrikkelen-adresse; Geonorge register https://register.geonorge.no/inspire-statusregister/matrikkelen-adresse/73bc0329-faac-419b-80ff-d113c0ffe6a0). Formats: CSV, SOSI, GML, SQL/PostGIS, FGDB, Atom feed, REST API `https://api.kartverket.no/adresser/v1/`. Files per kommune (daily) and fylke/country (weekly).
+- Counts (Kartverket "Antall vegadresser og matrikkeladresser", extract 2024-06-01, https://www.kartverket.no/datakvalitet/rapporter/adresse/Antall%20vegadresser%20og%20matrikkeladresser.xlsx): **2,577,825 addresses** — 2,536,663 vegadresser (street addresses) + 41,162 matrikkeladresser (cadastral, no street name).
+- Fields (CSV): adressenavn (street), nummer, bokstav, postnummer, poststed, kommunenummer, kommunenavn, grunnkrets, tettsted, coordinates (EPSG:25833 / 4258); bruksenhetsnummer only in non-CSV formats. Fylke = first two digits of kommunenummer.
+- Hierarchy: 15 fylker + Svalbard, 357 kommuner (2024). Kartverket/SSB code lists are open *(SSB Klass API licence NLOD, unverified)*.
+- Postal codes: Bring/Posten **Postnummerregister** — postnummer, poststed, kommunenummer, kommunenavn, kategori; TAB text + xlsx; **NLOD 2.0** per data.norge.no (https://data.norge.no/en/datasets/5e6847ba-156d-4e14-85d3-8d7f8b727523/postnummer-i-norge). Updated annually (Sept/Oct). Embeddable.
+- OpenAddresses: 17 `no/` sources.
+
+### Denmark
+- **Danmarks Adresseregister (DAR)**, served by **DAWA** (`https://api.dataforsyningen.dk/`) and Datafordeler. Licence **CC BY 4.0**, credit "Klimadatastyrelsen" (https://datafordeler.dk/vejledning/brugervilkaar/danmarks-adresseregister-dar/). Embeddable.
+- Counts: ~3.8 M adresser (Klimadatastyrelsen DAWA report); DAWA today: **1,089 postnumre, 53,976 vejnavne, 99 kommuner, 5 regioner** (queried 2026-09-16 via `?format=csv`).
+- Fields (DAWA `adresser`): vejnavn, husnr, etage, dør, postnr, postnrnavn, supplerende bynavn, kommunekode/-navn, regionskode/-navn, sogn, koordinater (ETRS89/WGS84), DAR ids. Downloads: CSV, JSON, GeoJSON, per kommune or whole country; `adgangsadresser` = access address (building entrance, no floor/door). Docs: https://dawadocs.dataforsyningen.dk/dok/adresser.
+- Hierarchy: 5 regioner → 98 kommuner (99 rows incl. Christiansø) → postnumre/byer. All in DAWA.
+- Postcodes: in DAR (source PostNord, updated on change), same licence.
+- OpenAddresses: `dk/countrywide` (1 source).
+
+### Finland
+- No single address-register download today. Sources:
+  - **DVV (Digital and Population Data Services Agency) building address open data** — CC BY 4.0; final official release 2025-02-14, distribution ended 2025-03-14; archived copies (CSV/GeoPackage, ~5 M buildings: street fi/sv, house number, postal code, municipality, WGS84) at https://markuskainu.fi/posts/2025-02-03-dvv-rakennusten-osoitetiedot/. Successor: **SYKE/Ryhti building addresses** via OGC API Features, CC BY 4.0 (https://avoindata.suomi.fi/data/fi/dataset/syke-rakennusten-osoitteet-ogc-api-features) — page blocked the fetch; verify fields and bulk download before relying on it.
+  - **NLS (Maanmittauslaitos)** topographic database address points — CC BY 4.0 (https://www.maanmittauslaitos.fi/en/opendata-licence-cc40); address points computed from road-name data.
+  - **Digiroad** (Fintraffic since 2026-01-01) — CC BY 4.0; road names + address number ranges (https://vayla.fi/en/transport-network/data/digiroad).
+- **Posti postal code files** (https://www.posti.fi/en/for-businesses/customer-support/postal-code-services; directory https://www.posti.fi/webpcode/): **PCF** (postal code, names fi/sv + abbreviations, type code, region code/name, municipality code/name fi/sv, language code; daily), **BAF** (every street per postal code with odd/even building-number ranges + municipality; weekly; excludes Åland), **POM** (monthly changes). Fixed-width `.dat`, Unix LF. Terms (service description 2015-01-01): "The files are freely downloadable … The data can be disclosed to third parties but it must be ensured that the recipient of the data is aware of the terms of use for the service as well as the download date." No licence name, no fee, no commercial restriction stated. Embeddable if the terms PDF + download date ship with the data.
+- Hierarchy: Statistics Finland classifications, CC BY 4.0: Municipalities 2026 https://stat.fi/en/luokitukset/kunta/kunta_1_20260101, Regions 2026 https://stat.fi/en/luokitukset/maakunta/maakunta_1_20260101. Posti PCF also carries municipality + region codes.
+- OpenAddresses: `fi/countrywide-fi`, `fi/countrywide-sv` + 38 municipal sources.
+
+### United Kingdom (Great Britain; NI separate)
+- No open full-address register (AddressBase is paid). Open pieces, all **OGL v3**, attribution "Contains OS data © Crown copyright and database right [year]" (https://wiki.openstreetmap.org/wiki/Ordnance_Survey_OpenData_Licence; OS moved to OGL v3 in 2015):
+  - **OS Open Names** — "over 870 000 named and numbered roads, nearly 44 000 settlements and over 1.6 million postcodes"; 34 attributes incl. `name1`, `name2` (Welsh/Gaelic), `type`, `local_type` (populatedPlace: City/Town/Village/Hamlet/Suburban Area/Other Settlement; transportNetwork: Named Road, Numbered Road, Section Of …; other: Postcode), `postcode_district`, `populated_place`, `district_borough`, `county_unitary`, `region`, `country`, `GEOMETRY_X/Y`. CSV/GML/GeoPackage, quarterly. https://docs.os.uk/os-downloads/products/addresses-and-names-portfolio/os-open-names. Gives street → settlement → district → county → region → country, but **no house numbers** and only the postcode district per road.
+  - **Code-Point Open** — ~1.7 M GB postcode units; fields Postcode, Positional_quality_indicator, Eastings, Northings, Country_code, NHS codes, Admin_county_code, Admin_district_code, Admin_ward_code. No street or locality names. GB only; NI excluded. https://docs.os.uk/os-downloads/products/areas-and-zones-portfolio/code-point-open.
+  - **OS Open UPRN** — ~40 M UPRN + coordinates, no addresses. https://www.ordnancesurvey.co.uk/products/os-open-uprn.
+  - **ONS Postcode Directory (ONSPD)** — postcode → all admin geographies; OGL, but requires Royal Mail attribution ("Contains Royal Mail data © Royal Mail copyright and database right [year]") and NI (BT) postcodes are end-user-licence only (no commercial redistribution): https://www.ons.gov.uk/methodology/geography/licences. Drop NI rows.
+- Hierarchy codes: ONS GSS codes (countries, regions, counties/UAs, districts, wards) via the ONS Open Geography Portal, OGL. Population: ONS mid-year estimates (OGL) at local-authority level; settlement population must come from GeoNames/Wikidata.
+- OpenAddresses: 0 `gb/` sources.
+- Practical: a deliverable GB address needs street + number + postcode unit; open data gives street + postcode *district* only. "Real street in a real town with a plausible district" is achievable; a real full postcode for that street is not.
+
+### Germany
+- **No national open address register.** BKG sells "Georeferenzierte Adressdaten" commercially. Open data is per Land: Hauskoordinaten / Gebäudereferenzen from 15 Länder under dl-de/by-2-0, dl-de/zero-2-0 or CC BY 4.0; **Bavaria missing** (~13 M people) — https://github.com/sister-software/mailwoman/issues/2300 (19.3 M permissive points vs 20.3 M in an ODbL build). Example: NRW "Gebäudereferenzen NW", dl-de/zero-2-0, semi-annual, https://open.nrw/dataset/172a0ba8-d470-47c1-ac89-b85f8190ac7e. Thuringia is dl-de/by-2-0. OpenAddresses lists 14 `de/*/statewide` sources (bb, bw, hb, he, hh, mv, ni, nw, rp, sh, sl, sn, st, th) — no `by`; Berlin appears as a Geoportal source *(check)*.
+- Hierarchy: **BKG VG250** (Land → Regierungsbezirk → Kreis → Verwaltungsgemeinschaft → Gemeinde with AGS keys; VG250-EW adds population), dl-de/by-2-0: https://gdz.bkg.bund.de/index.php/default/open-data/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html. **Destatis Gemeindeverzeichnis GV100/GV-ISys** (all Gemeinden with AGS, population, area, PLZ of the Verwaltungssitz), quarterly ASCII/Excel: https://www.destatis.de/DE/Themen/Laender-Regionen/Regionales/Gemeindeverzeichnis/_inhalt.html — licence dl-de/by-2-0 *(not stated on that page; Destatis default, unverified)*. 16 Länder, ~400 Kreise, ~10,800 Gemeinden *(approx.)*.
+- **Postal codes are the blocker**: Deutsche Post **DATAFACTORY** is proprietary and licensed for internal use only (https://www.deutschepost.de/de/d/deutsche-post-direkt/datafactory.html; a FragDenStaat request for the data was refused). Open PLZ lists (OpenPLZ API, yetzt/postleitzahlen, Geofabrik postcode polygons) are **OSM-derived → ODbL** (OpenPLZ sources page https://www.openplzapi.org/en/sources/: streets and PLZ from OSM, municipalities from Destatis GV100). GeoNames `DE.txt` provenance not stated *(unverified)*. The Länder Hauskoordinaten carry PLZ per address, so a PLZ↔Ort↔Straße table can be derived legitimately for the 15 covered Länder; GV100 gives one PLZ per Gemeinde seat.
+- OpenAddresses: 32 `de/` sources.
+
+### Netherlands
+- **BAG** (Basisregistratie Adressen en Gebouwen), Kadaster. Licence **CC0 1.0** (https://data.overheid.nl/en/dataset/basisregistratie-adressen-en-gebouwen--bag-; OSM wiki: "released under a Public Domain license"). Embeddable, no attribution.
+- ~9.9 M addresses (mailwoman count). Object types: panden, verblijfsobjecten, **nummeraanduidingen** (huisnummer + huisletter + toevoeging + postcode), **openbare ruimten** (streets), **woonplaatsen**. Fields yield street, number, postcode (4 digits + 2 letters), woonplaats, gemeente (CBS code), RD/WGS84 point.
+- Download: BAG Extract XML (monthly full + mutations) and BAG GeoPackage via PDOK Atom `https://service.pdok.nl/lv/bag/atom/bag.xml`; REST API `https://api.pdok.nl/lv/bag/`. data.overheid.nl mentions distribution fees for some Kadaster products; the PDOK extract/GeoPackage is free.
+- Hierarchy: 12 provincies → 342 gemeenten (2025) → ~2,500 woonplaatsen. CBS gebiedsindelingen (CC BY 4.0); CBS "Kerncijfers wijken en buurten" for population.
+- Postcodes: in BAG since 2012 (CC0). PostNL's product is proprietary but unnecessary.
+- OpenAddresses: `nl/countrywide` (1 source).
+
+### France
+- **Base Adresse Nationale (BAN)** — **Licence Ouverte / Etalab 2.0** since 2020-01-01 (https://fr.wikipedia.org/wiki/Base_adresse_nationale; https://adresse.data.gouv.fr/outils/telechargements). Embeddable with attribution.
+- >25 M addresses. Downloads: CSV national / per département / per commune, "CSV with BAN ids", addok format; daily updates with weekly/monthly archives; also MVT/WFS/WMS. CSV fields `id, id_fantoir, numero, rep, nom_voie, code_postal, code_insee, nom_commune, code_insee_ancienne_commune, nom_ancienne_commune, x, y, lon, lat, type_position, alias, nom_ld, libelle_acheminement, nom_afnor, source_position, source_nom_voie, certification_commune, cad_parcelles` *(from BAN docs, not re-verified this session)*.
+- Hierarchy: INSEE **Code Officiel Géographique** (18 régions, 101 départements, ~34,900 communes), Licence Ouverte; INSEE populations légales per commune (Licence Ouverte).
+- Postal codes: in BAN; also La Poste **"Base officielle des codes postaux"** (commune → code postal → libellé d'acheminement, INSEE code), open licence, https://datanova.laposte.fr/datasets/laposte-hexasmal and https://www.data.gouv.fr/datasets/base-officielle-des-codes-postaux.
+- OpenAddresses: 107 `fr/` sources (one per département, fed weekly from BAN).
+
+### Spain
+- **Catastro INSPIRE Addresses (AD)** — per-municipality GML via Atom `http://www.catastro.minhap.es/INSPIRE/Addresses/ES.SDGC.AD.atom.xml`, refreshed ~6-monthly; WFS `http://ovc.catastro.meh.es/INSPIRE/wfsAD.aspx`. Fields: ThoroughfareName, address number, `AD:PostCode` (5 digits), AdminUnitName (municipio, provincia), point at building entrance or parcel centroid. Coverage: 95% of Catastro territory, **excludes País Vasco and Navarra** (own cadastres; Navarra has OpenAddresses source `es/nc/statewide`). Licence: "licencia de cesión de derechos que se obtendrá de manera automática", free, attribution "© Dirección General del Catastro" (https://www.catastro.hacienda.gob.es/webinspire/documentos/Conjuntos%20de%20datos.pdf; https://www.catastro.hacienda.gob.es/webinspire/index.html). mailwoman labels it CC BY 4.0 (15.66 M points); the Catastro PDF does not say "CC BY" — **read the "Descripción de la licencia" PDF before embedding**.
+- **INE Callejero del Censo Electoral** — VIAS, TRAMOS, PSEUDOVIAS, UNIDADES POBLACIONALES; ASCII ZIP, national + per province, Jan/Jul each year, `https://www.ine.es/prodyser/callejero/caj_esp/caj_esp_MMYYYY.zip` (https://datos.gob.es/en/catalogo/ea0010587-callejero-de-censo-electoral). Licence = INE aviso legal (free reuse with source citation *(unverified; page blocked)*). Whether TRAMOS carry código postal: *(unverified)*.
+- **CartoCiudad (IGN/CNIG)** — addresses, postcodes, toponyms, admin units; CC BY 4.0 (https://datos.gob.es/es/aplicaciones/direcciones-postales-de-cartociudad-espana). Its postcode polygon layer came from Correos and was **withdrawn in 2017** (Correos now sells it, ~€6k/yr) — https://www.nosolosig.com/articulos/asi-hice-el-mapa-de-los-codigos-postales-de-espana-con-sig-y-datos-abiertos. Per-address postcodes remain in Catastro AD and CartoCiudad addresses.
+- Hierarchy: 17 CCAA + 2 ciudades autónomas → 50 provincias → 8,132 municipios; INE padrón gives population per municipio; INE codes are open.
+- OpenAddresses: `es/countrywide` (Catastro) + `es/nc/statewide` + 4 others.
+
+### Australia
+- **G-NAF** (Geoscape via data.gov.au, https://data.gov.au/data/dataset/geocoded-national-address-file-g-naf). August 2026: **15,949,543 addresses** (15,108,510 principal). Quarterly. PSV tables (~5 GB unpacked, many tables to join) + **G-NAF Core** single simplified table; GDA94 or GDA2020.
+- Licence: **G-NAF End User Licence Agreement, "based on CC BY 4.0"**, plus: must not be used to compile addresses for sending mail unless each is verified against a secondary source; must follow the Australian Privacy Principles. Preferred attribution: "Incorporates or developed using G-NAF © Geoscape Australia licensed by the Commonwealth of Australia under the Open Geo-coded National Address File (G-NAF) End User Licence Agreement." Embeddable with the EULA text and that notice; the mail restriction is irrelevant to fake data but must be passed on.
+- Fields: number first/last with prefix/suffix, flat/level, street name + type + suffix, locality, state, postcode, lat/lon, mesh block; LGA via ABS codes.
+- Hierarchy: 8 states/territories → LGAs (~560) → localities (~15,000 in the G-NAF locality table). **ABS ASGS Edition 3** boundaries/allocation files, CC BY 4.0: https://www.abs.gov.au/statistics/standards/australian-statistical-geography-standard-asgs/edition-3-july-2021-june-2026/access-and-downloads/digital-boundary-files. Population: ABS ERP by SA2/LGA (CC BY 4.0).
+- Postcodes: in G-NAF. Australia Post's postcode file is free only as a **non-commercial PDF**; CSV products are paid (https://auspost.com.au/business/services/data-services/address-data/postcode-data). ABS "Postal Areas" (2,644 POAs, CC BY 4.0) are mesh-block approximations.
+- OpenAddresses: `au/countrywide` + 7 statewide + councils (58 sources).
+
+### Canada
+- **National Address Register (NAR)**, Statistics Canada — December 2024 release, **17.1 M** records, all 13 provinces/territories; two CSVs per province (addresses, locations). Licence **Statistics Canada Open Licence** (use, reproduce, distribute, sell, sublicense; attribution "Adapted from Statistics Canada, National Address Register, 2024. This does not constitute an endorsement by Statistics Canada of this product."): https://www.statcan.gc.ca/en/reference/licence; user guide https://www150.statcan.gc.ca/n1/pub/46-26-0002/462600022024002-eng.htm. Fields: `CIVIC_NO, CIVIC_NO_SUFFIX, OFFICIAL_STREET_NAME/TYPE/DIR, APT_NO_LABEL, MAIL_STREET_NAME/TYPE/DIR, MAIL_MUN_NAME, MAIL_PROV_ABVN, MAIL_POSTAL_CODE, CSD_CODE, CSD_ENG_NAME, CSD_FRE_NAME, PROV_CODE, REPPOINT_LATITUDE/LONGITUDE, BG_X/Y, BU_USE`. Postal code present "for most addresses" — completeness per province unverified. Embeddable.
+- **Open Database of Addresses (ODA) v1** (2021), ~10 M records from 99 datasets, OGL-Canada, CSV per province, postal code where the provider had it: https://www.statcan.gc.ca/en/lode/databases/oda. Older; prefer NAR.
+- Hierarchy: StatCan Standard Geographical Classification 2021 — 13 provinces/territories → 293 census divisions → ~5,160 census subdivisions (municipalities); population per CSD from Census 2021. StatCan Open Licence.
+- **Postal codes are proprietary**: Canada Post asserts copyright, sells the file, and sued Geocoder.ca in 2012 (https://opennorth.ca/resources/open-postal-code-data/). StatCan's PCCF is licensed from Canada Post and not redistributable. GeoNames `CA.txt` = **FSA (first 3 chars) only**. Only NAR/ODA `MAIL_POSTAL_CODE` gives open per-address postal codes.
+- OpenAddresses: `ca/countrywide` + 221 sources; Canada collection 1.25 GB.
+
+## 2. Cross-country hierarchy sources
+
+| Source | Licence | Content | Notes |
+|---|---|---|---|
+| Debian **iso-codes** (`iso_3166-1.json`, `iso_3166-2.json`; XML deprecated) | LGPL-2.1 (https://salsa.debian.org/iso-codes-team/iso-codes) | All ISO 3166-1 countries and 3166-2 subdivisions with codes, types, parents; 90+ translations | LGPL on a data file is awkward inside MIT; ship it as a separately-licensed data file or consume it only at build time. ISO's own OBP is not open. |
+| **GeoNames** `admin1CodesASCII.txt`, `admin2Codes.txt` | CC BY 4.0 | admin1/admin2 names + geonameids per country | admin1 codes follow ISO 3166-2 for some countries and FIPS/numeric codes for others. |
+| `amckenna41/iso3166-2` (PyPI) | MIT (code); data compiled from Wikipedia/RestCountries | 250 countries, >5,000 subdivisions with names, lat/lon | Convenient; provenance mixed. |
+| Per-country official lists (section 1) | see country | authoritative codes + population | Preferred for the 11 target countries. |
+
+## 3. Localities with population
+
+| Source | Licence | Size | Fit |
+|---|---|---|---|
+| **GeoNames** `cities500/1000/5000/15000` + `allCountries` (https://download.geonames.org/export/dump/) | CC BY 4.0 (attribution to geonames.org) | cities500 ≈185 k, cities1000 ≈130 k, cities5000 ≈50 k, cities15000 ≈25 k; fields geonameid, name, asciiname, alternatenames, lat/lon, feature class/code (PPL, PPLA…), country, admin1–4 codes, population, timezone | Best single cross-country locality+population source; population uneven (many zeros). Embeddable with attribution. |
+| **Census SUB-EST2025** (US) | public domain | ~19.5 k places | Authoritative US population. |
+| **Wikidata** (P1082 population, P131 located-in, P281 postal code) | CC0 | all | No attribution; needs a SPARQL/dump pipeline; quality varies. |
+| **Who's On First** (https://www.whosonfirst.org/docs/licenses/) | CC0 for WOF's own work; 312 sources ranging CC0…CC BY 3.0/4.0, OGL | localities, regions, counties with hierarchy + population where sourced | Attribution list is per source; no share-alike found. Heavy (one GeoJSON per record). |
+| **dr5hn/countries-states-cities-database** (https://github.com/dr5hn/countries-states-cities-database) | **ODbL 1.0** | 250 countries, 5,299 states, 153,765 cities, 844,248 postcodes (125 countries) | **Share-alike → not embeddable in MIT.** Community-maintained; README: data "may contain errors or lag behind geopolitical changes"; issue tracker shows large re-parenting fixes (8,727 French cities). Not a source of truth. |
+
+## 4. Postal codes — GeoNames and per-country openness
+
+- **GeoNames postal** (https://download.geonames.org/export/zip/, CC BY 4.0): TSV fields `country code, postal code, place name, admin name1, admin code1, admin name2, admin code2, admin name3, admin code3, latitude, longitude, accuracy (1–6)`. ~100 countries. Readme caveats: Canada and Netherlands partial codes only (full NL/CA files separate); UK from Royal Mail (© Royal Mail 2022 — attribution needed); Ireland/Malta first letters only; Argentina 5 chars; Brazil only `-000` codes; Chile/China partial. Lat/lon is algorithmic (matched to a GeoNames toponym or averaged from neighbours) — a locality centroid, not the postcode's. Per-country provenance is not documented, so DE's origin is unknown.
+
+| Country | Open? | Best open source |
+|---|---|---|
+| US | Yes (ZCTA, public domain); USPS list not a file | Census ZCTA + GeoNames US.txt |
+| Norway | Yes (NLOD 2.0) | Bring postnummerregister; also in Matrikkelen |
+| Denmark | Yes (CC BY 4.0) | DAWA `postnumre` (1,089) |
+| Finland | Yes (Posti terms) | Posti PCF (codes) + BAF (streets per code) |
+| UK | GB yes (OGL, Code-Point Open); NI restricted | Code-Point Open + OS Open Names |
+| Germany | **No** (Deutsche Post proprietary); OSM lists are ODbL | Derive PLZ↔Ort↔Straße from Länder Hauskoordinaten (15 Länder); GV100 for seat PLZ |
+| Netherlands | Yes (CC0, in BAG) | BAG |
+| France | Yes (Licence Ouverte) | La Poste hexasmal + BAN |
+| Spain | Polygon layer closed since 2017; per-address codes open via Catastro AD (own licence) | Catastro AD `AD:PostCode`; INE callejero |
+| Australia | In G-NAF (EULA/CC BY 4.0); Australia Post file paid/non-commercial | G-NAF locality table |
+| Canada | **No** (Canada Post proprietary; GeoNames FSA only) | NAR `MAIL_POSTAL_CODE` where present |
+
+## 5. OpenAddresses and OpenStreetMap
+
+**OpenAddresses** (https://openaddresses.io/, https://batch.openaddresses.io/)
+- Global collection 48.39 GB (2026-09-07); US in 4 regional collections (table in section 1); Canada 1.25 GB. 3,282 sources (1,586 flagged "need attention"). Sources per country in the batch list: US 2,381, CA 222, FR 107, AU 58, FI 40, DE 32, NO 17, ES 6, DK 1, NL 1, GB 0.
+- Licensing: the repo LICENSE (BSD-3) covers source definitions only. **Each source keeps its own licence**, recorded in the source JSON and `state.txt`; "most sources only require attribution", but several are share-alike (ODbL, CC BY-SA) — e.g. OSM-derived and some municipal sources (https://geocode.earth/docs/reference/data_sources/). For MIT: filter by the per-source licence field, keep PD/CC0/CC BY/OGL/dl-de sources, generate the attribution file from `state.txt`. The old results.openaddresses.io coverage page is frozen at mid-2021.
+- The official registers above are themselves the OA sources for NO/DK/FI/NL/FR/ES/AU (and NAD/state sources for the US); going to the register directly gives cleaner licensing and fresher data. OA's value is one uniform CSV schema (`LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID,HASH`).
+
+**OpenStreetMap** — ODbL 1.0
+- Any *substantial* extract (systematic, >100 features, or an area with >1,000 inhabitants) is a Derivative Database; distributing it — including embedded in a library — requires offering it under ODbL (share-alike) plus attribution. OSMF guideline: https://osmfoundation.org/wiki/Licence/Community_Guidelines/Substantial_-_Guideline ("village map OK, town map not OK"; repeated small extractions count as one).
+- Consequence: an embedded street/postcode list derived from OSM (OpenPLZ, Geofabrik postcode polygons, yetzt/postleitzahlen, OSM-derived dr5hn rows) cannot ship inside an MIT package without that data file being ODbL and tracked as a separate database; downstream users mixing it into their databases inherit share-alike. Keep OSM out of the default; at most an *optional* download pack clearly labelled ODbL.
+
+## 6. Embed vs. optional download — sizes and a recommendation
+
+**US (ships by default with en_US)**
+- Embed (small, public domain): 56 states (+ USPS abbreviations, FIPS), 3,234 counties (GEOID, name, state), ~19.5 k incorporated places with population + county + lat/lon (SUB-EST2025 joined to the Gazetteer), 33,791 ZCTAs with the ZCTA→place/county relationship, and a compact **street-name pool per place** derived from NAD (distinct `St_Name`+`St_PosTyp` per `Post_City`/`State`, top-N by frequency). Places+ZCTAs ≈ 2–4 MB gzipped; a street pool for all ~19.5 k places would run to tens of MB — cap to the ~1,000 largest places or ~20 streets per place for the default pack.
+- Download pack: full NAD (~80 M rows, multi-GB), TIGER `featnames`/`addrfeat` per county, OA US collections (~23 GB zipped). Real house numbers + ZIP+4 come only from here.
+- Deliverability caveat: NAD ZIP is optional, and place naming mixes `Post_City` (USPS city) with `Inc_Muni`; use `Post_City` + `Zip_Code` for mailing realism.
+
+**Other countries** — same shape: embed hierarchy + localities-with-population + postcode↔locality table (a few MB per country: NO's 2.6 M addresses collapse to ~4.5 k postcodes and ~90 k street names; DK 1,089 postcodes / 54 k street names; NL/FR/AU similar), with street pools for the largest localities; keep full address points as download packs fetched from the official Atom/CSV endpoints at runtime, never vendored (G-NAF ~5 GB, BAN CSV ~2 GB, BAG ~2 GB, NAR ~1 GB).
+
+**Licence file needed in the repo** (`DATA-LICENSES.md`): CC BY 4.0 (GeoNames, Kartverket, Klimadatastyrelsen, Statistics Finland, DVV/SYKE, NLS, ABS, CartoCiudad), NLOD 2.0 (Bring), OGL v3 (OS, ONS), Licence Ouverte 2.0 (BAN, La Poste, INSEE), G-NAF EULA notice, StatCan Open Licence notice, dl-de/by-2-0 and dl-de/zero-2-0 (BKG, Länder), Posti terms + download date, Catastro "© Dirección General del Catastro". CC0/public domain (US Census, DOT NAD, BAG, Wikidata) need nothing but are worth listing.
+
+## 7. Open questions / unverified
+- NAD ZIP fill rate and the list of non-participating states (DOT site blocks fetches; read the NAD coverage map manually).
+- Unique US street-name count — compute from NAD once downloaded.
+- Catastro licence text ("Descripción de la licencia" PDF) — confirm redistribution inside a software package.
+- INE aviso legal wording; whether callejero TRAMOS include código postal.
+- Destatis GV100 licence statement; Berlin/Bayern Hauskoordinaten status.
+- SYKE/Ryhti address API fields and bulk-download availability (avoindata.fi blocked the fetch).
+- NAR `MAIL_POSTAL_CODE` completeness per province.
+- GeoNames DE/US postal-code provenance and counts.
+- BAN CSV field list re-verification.

docs/research/research-libraries.md

@@ -0,0 +1,287 @@
+# Fake-data libraries: data-category inventory
+
+Researched 2026-09-16. Sources: official docs and current `master`/`main`/`next` source trees. "Big four" = @faker-js/faker (JS), Faker (Python), gofakeit (Go), Datafaker (Java). Abbreviations in tables: **JS**, **PY**, **GO**, **DF**, **BG** (Bogus), **MM** (Mimesis), **CH** (Chance), **RB** (Ruby faker).
+
+## 1. Category modules and their generators
+
+Legend: names are the library's own identifiers (camelCase = JS/DF/BG, snake_case = PY/MM/RB, PascalCase = GO). Columns list what each library offers under that module; a `—` means the module does not exist there.
+
+### person
+
+| Lib | Generators |
+|---|---|
+| JS | firstName, lastName, middleName, fullName, prefix, suffix, sex, sexType, gender, bio, jobArea, jobDescriptor, jobTitle, jobType, zodiacSign (all name methods take `sex: female|male|generic`) |
+| PY | first_name/_female/_male/_nonbinary, last_name/_female/_male/_nonbinary, name/_female/_male/_nonbinary, prefix/_female/_male/_nonbinary, suffix/_female/_male/_nonbinary, language_name; separate providers: job (job, job_female, job_male), ssn (ssn; per-locale variants: vat_id, itin, ein, nif, cpf, personnummer …), passport (passport_number, passport_dob, passport_owner), profile (profile → job, company, ssn, residence, current_location, blood_group, website, username, name, sex, address, mail, birthdate; simple_profile) |
+| GO | Person (struct), Name, NamePrefix, NameSuffix, FirstName, MiddleName, LastName, Gender, Age, Ethnicity, SSN, EIN, Hobby, SocialMedia, Bio, Contact, Email, Phone, PhoneFormatted, Teams |
+| DF | Name: name, nameWithMiddle, fullName, firstName, femaleFirstName, maleFirstName, lastName, prefix, suffix, title, username; Demographic: race, educationalAttainment, demonym, sex, maritalStatus; Gender; Pronouns; BloodType; Mbti; Zodiac; Relationship; Hobby; Mood; Job; Passport; DrivingLicense; IdNumber (valid/invalid per country: US SSN, SE, ZA, SG FIN/UIN, CN, PT NIF, MX, PL PESEL, KR RRN, GE); CPF/CNPJ (BR); Nigeria; Australia |
+| BG | Name: FirstName, LastName, FullName, Prefix, Suffix, FindName, JobTitle, JobDescriptor, JobArea, JobType; Person object (Gender, FirstName, LastName, FullName, UserName, Avatar, Email, DateOfBirth, Address, Phone, Website, Company); country extensions: Ssn/Ein (US), Personnummer/Samordningsnummer (SE), Cpr (DK), Henkilotunnus (FI), Fodselsnummer (NO), Pesel/Nip/Regon (PL), CodiceFiscale (IT), Cpf/Cnpj (BR), Sin (CA), Nino (GB), Cnp (RO), NationalNumber (BE), Nif/Nipc (PT) |
+| MM | Person: first_name, surname/last_name, patronymic, full_name, name, title, username, password, email, birthdate, gender, gender_symbol, gender_code, sex, height, weight, blood_type, occupation, nationality, university, academic_degree, language, phone_number, telephone, identifier |
+| CH | first, last, name, prefix, suffix, gender, age, birthday, ssn, cpf, cf (IT codice fiscale), profession |
+| RB | Name (name, name_with_middle, first_name, male_first_name, female_first_name, neutral_first_name, last_name, prefix, suffix, initials), Gender, Demographic, IDNumber (per-country), Job, Relationship, Blood, DrivingLicence, Avatar, ChileRut, SouthAfrica, NationalHealthService |
+
+### address / location
+
+| Lib | Generators |
+|---|---|
+| JS (location) | buildingNumber, cardinalDirection, city, continent, country, countryCode, county, direction, language, latitude, longitude, nearbyGPSCoordinate, ordinalDirection, postalAddress, secondaryAddress, state({abbreviated}), street, streetAddress, timeZone, zipCode({state, format}) |
+| PY (address) | address, building_number, city, city_suffix, country, country_code, current_country, current_country_code, postcode, street_address, street_name, street_suffix; locale extras e.g. en_US: city_prefix, secondary_address, administrative_unit, state_abbr, zipcode_plus4, postcode_in_state, military_ship/state/apo/dpo; ja_JP: prefecture, city, town, chome, ban, gou, building_name; fr_FR: department, department_name, department_number, region; es_ES: region/autonomous_community; de_DE: city_with_postcode, street_suffix_short/long; en_GB: county; separate **geo** provider: coordinate, latitude, longitude, latlng, local_latlng(country), location_on_land (real geonames.org places with tz) |
+| GO (address) | Address (struct), City, Country, CountryAbr, State, StateAbr, Street, StreetName, StreetNumber, StreetPrefix, StreetSuffix, Unit, Zip, Latitude, LatitudeInRange, Longitude, LongitudeInRange |
+| DF (Address) | streetName, streetAddressNumber, streetAddress, secondaryAddress, zipCode, postcode, eircode, zipCodePlus4, zipCodeByState, countyByZipCode, streetSuffix, streetPrefix, citySuffix, cityPrefix, city, cityName, state, stateAbbr, latitude, longitude, latLon, lonLat, timeZone, country, countryCode, buildingNumber, fullAddress, mailBox; plus Country (name, code2/3, capital, currency, flag), Nation (nationality, language, capital, flag), Compass, Mountain, Planet, Space, Locality (locale codes) |
+| BG (Address) | ZipCode, City, StreetAddress, CityPrefix, CitySuffix, StreetName, BuildingNumber, StreetSuffix, SecondaryAddress, County, Country, FullAddress, CountryCode, State, StateAbbr, Latitude, Longitude, Direction, CardinalDirection, OrdinalDirection; premium Bogus.Locations: GPS, altitude, depth, geohash |
+| MM (Address) | street_number, street_name, street_suffix, secondary_address, address, state/region/province/federal_subject/prefecture (abbr), postal_code/zip_code, country, country_code (A2/A3/numeric), country_emoji_flag, default_country, city, latitude, longitude, coordinates (DMS option), continent, calling_code/isd_code, iata_code, icao_code |
+| CH | address, street, city, zip, postal (CA), postcode (GB), state, province, country, areacode, phone, latitude, longitude, altitude, depth, coordinates, geohash, locale |
+| RB | city, street_name, street_address, secondary_address, building_number, mail_box, community, zip_code(state_abbreviation:), zip, postcode, time_zone, street_suffix, city_suffix, city_prefix, state, state_abbr, country, country_by_code, country_name_to_code, country_code, country_code_long, latitude, longitude, full_address, full_address_as_hash; Travel: airport, train_station; Locations: australia |
+
+### company
+
+| Lib | Generators |
+|---|---|
+| JS | name, buzzAdjective, buzzNoun, buzzPhrase, buzzVerb, catchPhrase, catchPhraseAdjective, catchPhraseDescriptor, catchPhraseNoun (locale data: adjective, descriptor, noun, legal_entity_type, name_pattern) |
+| PY | company, company_suffix, catch_phrase, bs; locale extras (e.g. it_IT/pl_PL company_vat, nl_NL, ru_RU large_company, etc.) |
+| GO | Company, CompanySuffix, BS, Blurb, BuzzWord, Slogan, Job, JobDescriptor, JobLevel, JobTitle |
+| DF | Company: name, suffix, industry, profession, buzzword, catchPhrase, bs, logo, domainName, url; Business; Brand; IndustrySegments; Marketing; Team; Restaurant; Subscription; Twitter; University; Educator |
+| BG | CompanySuffix, CompanyName, CatchPhrase, Bs |
+| MM | Finance.company, company_type |
+| CH | company, profession |
+| RB | Company (name, suffix, industry, profession, type, catch_phrase, buzzword, bs, logo, ein, duns_number, swedish_organisation_number, czech_organisation_number, french_siren/siret, norwegian/australian/spanish/polish/russian/south_african/brazilian ids …), Business, Marketing, IndustrySegments, Team, University, Educator, Restaurant, Subscription, Construction |
+
+### internet
+
+| Lib | Generators |
+|---|---|
+| JS | displayName, domainName, domainSuffix, domainWord, email, exampleEmail, emoji, httpMethod, httpStatusCode, ip, ipv4, ipv6, jwt, jwtAlgorithm, mac, password, port, protocol, url, userAgent, username |
+| PY | email, safe_email, free_email, company_email, ascii_* variants, free_email_domain, domain_name, domain_word, safe_domain_name, tld, hostname, dga, http_method, http_status_code, iana_id, image_url, ipv4, ipv4_network_class, ipv4_private, ipv4_public, ipv6, mac_address, nic_handle(s), port_number, ripe_id, slug, uri, uri_extension, uri_page, uri_path, url, user_name; user_agent provider (chrome, firefox, safari, opera, internet_explorer, platform tokens); emoji provider |
+| GO | URL, UrlSlug, DomainName, DomainSuffix, IPv4Address, IPv6Address, MacAddress, HTTPStatusCode(Simple), HTTPMethod, HTTPVersion, LogLevel, UserAgent (+Chrome/Firefox/Opera/Safari/API), Username, Password, Emoji (+22 category fns), InputName, Svg |
+| DF | Internet: username, emailAddress(name), safeEmailAddress, emailSubject, domainName/Word/Suffix, url, webdomain, image, httpMethod, password(opts), port, macAddress, ipV4Address, privateIpV4Address, publicIpV4Address, ipV4Cidr, ipV6Address, ipV6Cidr, slug, uuidv3/4/7, userAgent, botUserAgent; Http; Domain; Sip; Emoji; SlackEmoji; Credentials; Aws; Azure; Hashing; Fingerprint; Computer; Device; Camera; Drone; Robin |
+| BG | Avatar, Email, ExampleEmail, UserName, UserNameUnicode, DomainName, DomainWord, DomainSuffix, Ip, Port, IpAddress, IpEndPoint, Ipv6, UserAgent, Mac, Password, Color, Protocol, Url, UrlWithPath, UrlRootedPath |
+| MM | Internet: content_type, dsn, http_status_message/code, http_method, ip_v4/v6 (+object, cidr, with_port, special), cloud_region, asn, mac_address, hostname, url, uri, query_string/parameters, tld, user_agent, port, path, slug, public_dns, http_request/response_headers |
+| CH | avatar, color, domain, email, fbid, google_analytics, hashtag, ip, ipv6, klout, tld, twitter, url |
+| RB | Internet (email, username, password, domain_name, ip_v4/v6, mac_address, url, slug, user_agent, uuid, bot_user_agent …), Internet::HTTP, Omniauth, Stripe, X (Twitter), Avatar, Placeholdit, LoremFlickr |
+
+### finance / payment / bank
+
+| Lib | Generators |
+|---|---|
+| JS | accountName, accountNumber, amount, bic, bitcoinAddress, creditCardCVV, creditCardIssuer, creditCardNumber, currency, currencyCode, currencyName, currencyNumericCode, currencySymbol, ethereumAddress, iban, litecoinAddress, pin, routingNumber, transactionDescription, transactionType |
+| PY | bank: aba, bank, bank_country, bban, iban, swift, swift8, swift11; credit_card: credit_card_number, credit_card_provider, credit_card_expire, credit_card_security_code, credit_card_full; currency: currency, currency_code, currency_name, currency_symbol, cryptocurrency(_code/_name), pricetag |
+| GO | Price, CreditCard, CreditCardCvv, CreditCardExp, CreditCardNumber, CreditCardType, Currency, CurrencyLong, CurrencyShort, AchRouting, AchAccount, BitcoinAddress, BitcoinPrivateKey, BankName, BankType, Cusip, Isin |
+| DF | Finance: creditCard(type), bic, iban(country), ibanSupportedCountries, usRoutingNumber; Money: currency, currencyCode, currencyNumericCode, currencySymbol; Currency; Stock; FinancialTerms; CryptoCoin; Coin; Business |
+| BG | Account, AccountName, Amount, TransactionType, Currency, CreditCardNumber, CreditCardCvv, BitcoinAddress, EthereumAddress, RoutingNumber, Bic, Iban; GB SortCode, VatNumber |
+| MM | Finance: bank, currency_iso_code, currency_symbol, cryptocurrency_iso_code/_symbol, price, price_in_btc, stock_ticker, stock_name, stock_exchange; Payment: cid, bitcoin_address, ethereum_address, credit_card_network/number/expiration_date, cvv, credit_card_owner |
+| CH | cc, cc_type, currency, currency_pair, dollar, euro, exp, exp_month, exp_year |
+| RB | Finance (credit_card, vat_number, ticker, stock_market), Bank (name, swift_bic, iban, account_number, routing_number, bsb_number), Currency, Crypto, Blockchain (bitcoin, ethereum, tezos, aeternity), Invoice, Stripe, Coin |
+
+### commerce / product
+
+| Lib | Generators |
+|---|---|
+| JS | department, isbn, price, product, productAdjective, productDescription, productMaterial, productName, upc |
+| PY | barcode: ean, ean8, ean13, localized_ean*; isbn: isbn10, isbn13; sbn; (no product/commerce provider in core; community Ecommerce provider) |
+| GO | Product (struct), ProductName, ProductDescription, ProductCategory, ProductFeature, ProductMaterial, ProductUPC, ProductAudience, ProductDimension, ProductUseCase, ProductBenefit, ProductSuffix, ProductISBN |
+| DF | Commerce: department, productName, material, brand, vendor, price(min,max), promotionCode; Barcode; Code (isbn10/13, gtin8/13, ean, asin, imei); Appliance; GarmentSize; Size; Tire; House; ElectricalComponents |
+| BG | Department, Price, Categories, ProductName, Color, Product, ProductAdjective, ProductMaterial, Ean8, Ean13 |
+| MM | Code: locale_code, issn, isbn, ean, imei, pin |
+| CH | — |
+| RB | Commerce (color, department, material, product_name, price, promotion_code, brand, vendor), Barcode, Code (isbn, ean, asin, imei, npi, nric, rut, sin), Appliance, Device, Camera, House |
+
+### vehicle / transport
+
+| Lib | Generators |
+|---|---|
+| JS | bicycle, color, fuel, manufacturer, model, type, vehicle, vin, vrm; airline: aircraftType, airline, airplane, airport, flightNumber, recordLocator, seat |
+| PY | automotive: license_plate, vin (per-locale plate formats) |
+| GO | Car (struct), CarMaker, CarModel, CarType, CarFuelType, CarTransmissionType; Airline*: AircraftType, Airplane, Airport, AirportIATA, FlightNumber, RecordLocator, Seat |
+| DF | Vehicle: vin, manufacturer, make, model(make), makeAndModel, style, color, upholstery(+Color/Fabric), transmission, driveType, fuelType, carType, engine, carOptions, standardSpecs, doors, licensePlate(state); Aviation; Transport; DrivingLicense |
+| BG | Vin, Manufacturer, Model, Type, Fuel; GbRegistrationPlate |
+| MM | Transport: manufacturer, car, airplane, vehicle_registration_code(locale) |
+| CH | — |
+| RB | Vehicle (vin, manufacture, make, model, make_and_model, style, color, transmission, drive_type, fuel_type, car_type, engine, car_options, standard_specs, doors, door, year, mileage, license_plate, singapore_license_plate, version), Drone, Travel::Airport, Travel::TrainStation |
+
+### lorem / text / words
+
+| Lib | Generators |
+|---|---|
+| JS | lorem: lines, paragraph(s), sentence(s), slug, text, word(s); word: adjective, adverb, conjunction, interjection, noun, preposition, sample, verb, words; hacker: abbreviation, adjective, ingverb, noun, phrase, verb; string: alpha, alphanumeric, binary, fromCharacters, hexadecimal, nanoid, numeric, octal, sample, symbol, ulid, uuid |
+| PY | lorem: word(s), sentence(s), paragraph(s), text(s), get_words_list; misc: password, md5, sha1, sha256, binary, boolean, null_boolean, csv/dsv/psv/tsv, fixed_width, json, json_bytes, image, tar, zip, uuid4 |
+| GO | Word families: Noun* (Common/Concrete/Abstract/Collective*/Countable/Uncountable/Proper/Determiner), Verb* (Action/Linking/Helping/Transitive/Intransitive), Adverb* (Manner/Degree/Place/Time*/Frequency*), Preposition* (Simple/Double/Compound), Adjective* (8 kinds), Pronoun* (8 kinds), Connective* (6 kinds), Word, Interjection, Sentence, Paragraph, LoremIpsum{Word,Sentence,Paragraph}, Question, Quote, Phrase{,Noun,Verb,Adverb,Preposition}, Comment; Hacker*, Hipster{Word,Sentence,Paragraph}; Letter(N), Digit(N), Vowel, Lexify, Numerify, RandomString, ShuffleStrings |
+| DF | Lorem (word(s), sentence(s), paragraph(s), characters, fixedString, maxLengthSentence, supplemental), Text (character, upper/lowercase, text with symbol rules), Verb, Word, Hacker, Hipster, Shakespeare, Yoda, Joke, ChuckNorris, Matz, NatoPhoneticAlphabet, FunnyName, Marketing |
+| BG | Lorem: Word(s), Letter, Sentence(s), Paragraph(s), Text, Lines, Slug; Hacker: Abbreviation, Adjective, Noun, Verb, IngVerb, Phrase; Rant: Review(s) |
+| MM | Text: alphabet, level, text, sentence, title, words, word, quote, color, hex_color, rgb_color, answer, emoji |
+| CH | paragraph, sentence, syllable, word, character, letter, string |
+| RB | Lorem (word(s), character(s), sentence(s), paragraph(s), question(s), paragraph_by_chars, multibyte), Markdown, Hipster, Hacker, Adjective, Verbs, Quote, Quotes::Shakespeare/Chiquito/Rajnikanth, ChuckNorris, Emotion, Source (code snippets), Html, Json, Types, Alphanumeric, String, Boolean, NatoPhoneticAlphabet |
+
+### date / time
+
+| Lib | Generators |
+|---|---|
+| JS | anytime, between, betweens, birthdate({mode: age|year}), future, past, recent, soon, month, weekday, timeZone (locale data: month, weekday) |
+| PY | date_time: date, date_time, date_object, time, time_object, date_between(_dates), date_time_between(_dates), date_this_century/decade/month/year, date_time_this_*, date_time_ad, date_of_birth(minimum_age, maximum_age), future_date(time), past_date(time), iso8601, unix_time, time_delta, time_series, timezone, pytimezone, am_pm, century, year, month, month_name, day_of_month, day_of_week |
+| GO | Date, PastDate, FutureDate, DateRange, NanoSecond, Second, Minute, Hour, Month, MonthString, Day, WeekDay, Year, TimeZone, TimeZoneAbv, TimeZoneFull, TimeZoneOffset, TimeZoneRegion |
+| DF | DateAndTime: future, past, between, birthday, birthdayLocalDate, duration, period; Time; TimeAndDate |
+| BG | Past, PastOffset, Soon, SoonOffset, Future, FutureOffset, Between, BetweenOffset, Recent, RecentOffset, Timespan, Month, Weekday |
+| MM | Datetime: date, datetime, time, timestamp, formatted_*, week_date, day_of_week, month, year, day_of_month, timezone, gmt_offset, periodicity, future/past_date(time), duration, bulk_create_datetimes |
+| CH | ampm, date, hammertime, hour, millisecond, minute, month, second, timestamp, timezone, weekday, year |
+| RB | Date (between, between_except, forward, backward, birthday, in_date_period, on_day_of_week_between), Time (between, between_dates, forward, backward) |
+
+### phone
+
+| Lib | Generators |
+|---|---|
+| JS | number({style: human|national|international}), imei |
+| PY | phone_number, country_calling_code, msisdn (per-locale formats) |
+| GO | Phone, PhoneFormatted |
+| DF | PhoneNumber: cellPhone, cellPhoneInternational, phoneNumber, phoneNumberInternational, phoneNumberNational, extension, subscriberNumber |
+| BG | PhoneNumber, PhoneNumberFormat |
+| MM | Person.phone_number/telephone (mask), Address.calling_code |
+| CH | phone, areacode |
+| RB | PhoneNumber (phone_number, cell_phone, country_code, phone_number_with_country_code, cell_phone_with_country_code, cell_phone_in_e164, area_code, exchange_code, subscriber_number, extension) |
+
+### science / medical
+
+| Lib | Generators |
+|---|---|
+| JS | chemicalElement, unit; medical locale data (en only) |
+| PY | — (community: Healthcare, Biology, Geoscience, Scientific) |
+| GO | — |
+| DF | Science: element, elementSymbol, unit, scientist, tool, quark, leptons, bosons; Medical, Medication, Disease, MedicalProcedure, CareProvider, Observation, BloodType, Measurement, Weather, Planet, Space, Mountain, Cannabis, LargeLanguageModel |
+| BG | premium Bogus.Healthcare |
+| MM | Science: rna_sequence, dna_sequence |
+| CH | — |
+| RB | Science (element, element_symbol, element_state, element_subcategory, scientist, modifier, tool), Space, Measurement, Cannabis, Medical (NationalHealthService) |
+
+### music / animal / food / books / entertainment
+
+| Lib | Generators |
+|---|---|
+| JS | music: album, artist, genre, songName; animal: bear, bird, cat, cetacean, cow, crocodilia, dog, fish, horse, insect, lion, petName, rabbit, rodent, snake, type; food: adjective, description, dish, ethnicCategory, fruit, ingredient, meat, spice, vegetable; book: author, format, genre, publisher, series, title |
+| PY | — (core has none; community Music, Sci Fi) |
+| GO | Song, SongName, SongArtist, SongGenre; PetName, Animal, AnimalType, FarmAnimal, Cat, Dog, Bird; Fruit, Vegetable, Breakfast, Lunch, Dinner, Snack, Dessert, Drink; Beer{Alcohol,Blg,Hop,Ibu,Malt,Name,Style,Yeast}; Book, BookTitle, BookAuthor, BookGenre; Movie, MovieName, MovieGenre; Celebrity{Actor,Business,Sport}; Minecraft (18 fns) |
+| DF | Music: instrument, key, chord, genre; RockBand; Artist; Kpop; Hololive; Animal: name, scientificName, genus, species; Cat, Dog, Horse; Food: ingredient, allergen, spice, dish, fruit, vegetable, sushi, measurement; Apple, Beer, Cheese, Coffee, Dessert, IceCream, Tea; Book, Movie, Show, OscarMovie; ~75 entertainment franchises, ~31 videogames, 9 sports |
+| BG | Music (premium Hollywood: movies, TV, actors) |
+| MM | Food: vegetable, fruit, dish, spices, drink |
+| CH | animal (with type), tv, radio, rpg, dice |
+| RB | Music (+11 bands), Creature (animal, bird, cat, dog, horse), Food, Beer, Coffee, Tea, Dessert, Book, Books (4 series), Movie/Movies (17), TvShows (39), Games (26), JapaneseMedia (10), Sports (6), Fantasy::Tolkien, Religion::Bible, Superhero, DcComics, Kpop, Ancient, GreekPhilosophers, Cosmere |
+
+### color / image / system / database / misc
+
+| Lib | Generators |
+|---|---|
+| JS | color: cmyk, colorByCSSColorSpace, cssSupportedFunction, cssSupportedSpace, hsl, human, hwb, lab, lch, rgb, space; image: avatar, avatarGitHub, dataUri, personPortrait, url, urlLoremFlickr, urlPicsumPhotos; system: commonFileExt/Name/Type, cron, directoryPath, fileExt/Name/Path/Type, mimeType, networkInterface, semver; database: collation, column, engine, mongodbObjectId, type; git: branch, commitDate, commitEntry, commitMessage, commitSha; number: bigInt, binary, float, hex, int, octal, romanNumeral; datatype.boolean; science.unit |
+| PY | color: color, color_name, safe_color_name, hex_color, safe_hex_color, rgb_color, rgb_css_color, color_hsl/hsv/rgb/rgb_float; file: file_extension, file_name, file_path, mime_type, unix_device, unix_partition; python: pybool, pydecimal, pydict, pyfloat, pyint, pyiterable, pylist, pyobject, pyset, pystr, pystr_format, pystruct, pytuple, enum; doi; emoji; user_agent |
+| GO | Color, HexColor, RGBColor, HSLColor, SafeColor, NiceColors; Image, ImageJpeg, ImagePng; CSV, JSON, XML, SQL, FileExtension, FileMimeType, Template, Markdown, EmailText, FixedWidth; ID, UUID; Error* (9 kinds); AppName, AppVersion, AppAuthor; Language, LanguageAbbreviation, LanguageBCP, ProgrammingLanguage; School; Gamertag, Dice; Bool, Weighted, FlipACoin; Number/Int*/Uint*/Float* ranges |
+| DF | Color, Image, File, App, ProgrammingLanguage, LanguageCode, Number, Bool, Unique, Options, Photography, Military, OlympicSport, Weather, Construction, Community, Chiquito … (263 providers total) |
+| BG | Images: DataUri, PicsumUrl, PlaceholderUrl, LoremFlickrUrl; System: FileName, DirectoryPath, FilePath, CommonFileName, MimeType, CommonFileType/Ext, FileType, FileExt, Semver, Version, Exception, AndroidId, ApplePushToken, BlackBerryPin; Database: Column, Type, Collation, Engine; Randomizer (numbers, chars, Guid, Hash, Enum, WeightedRandom, Shuffle …) |
+| MM | Hardware: resolution, screen_size, cpu, cpu_frequency, generation, cpu_codename, ram_type, ram_size, ssd_or_hdd, graphics, manufacturer, phone_model; Development: software_license, calver, version, stage, programming_language, os, boolean, system_quality_attribute; File, BinaryFile, Path, Cryptographic (uuid, hash, token, mnemonic), Numeric, Choice |
+| CH | bool, falsy, floating, integer, natural, prime, hex, guid, hash, coin, dice, normal (Gaussian), n, unique, weighted, android_id, apple_token, bb_pin, wp7_anid, wp8_anid2 |
+| RB | Color, File, Computer, Device, ProgrammingLanguage, App, Number, Boolean, Hash/Crypto, Json, Html, Markdown, Military, Compass, Coin, Mountain, Nation, Space, Time, Types, Slack Emoji, Vulnerability identifier |
+
+Module presence summary:
+
+| Module | JS | PY | GO | DF | BG | MM | CH | RB |
+|---|---|---|---|---|---|---|---|---|
+| person | x | x | x | x | x | x | x | x |
+| address | x | x | x | x | x | x | x | x |
+| company | x | x | x | x | x | (finance) | x | x |
+| internet | x | x | x | x | x | x | x | x |
+| finance | x | x | x | x | x | x | x | x |
+| commerce/product | x | barcode only | x | x | x | code only | — | x |
+| vehicle | x | plate/vin | x | x | x | x | — | x |
+| airline | x | — | x | x | — | iata/icao | — | x |
+| lorem/words | x | x | x | x | x | x | x | x |
+| date | x | x | x | x | x | x | x | x |
+| phone | x | x | x | x | x | x | x | x |
+| science | x | — | — | x | — | dna | — | x |
+| medical | data only | — | — | x | premium | — | — | x |
+| music | x | — | x | x | — | — | — | x |
+| animal | x | — | x | x | — | — | x | x |
+| food | x | — | x | x | — | x | — | x |
+| book | x | isbn | x | x | — | — | — | x |
+| color | x | x | x | x | x | text | x | x |
+| image | x | x | x | x | x | binaryfile | avatar | x |
+| system/file | x | x | x | x | x | x | — | x |
+| database | x | — | sql | — | x | dsn | — | — |
+| git | x | — | — | — | — | — | — | — |
+| hacker/hipster | x | — | x | x | x | — | — | x |
+| national IDs | — | ssn per locale | SSN/EIN | IdNumber | ext. pkgs | identifier | ssn/cpf/cf | IDNumber |
+| pop-culture | — | — | minecraft, movie | ~110 | premium | — | tv, rpg | ~85 |
+| hardware | — | — | — | Computer/Device | — | x | — | Computer/Device |
+| weather/space | — | — | — | x | — | — | — | Space |
+
+## 2. Locale coverage and address data structure
+
+### Locale counts
+
+| Lib | Locales | Structure |
+|---|---|---|
+| JS | 77 dirs under `src/locales` (incl. `base`, `en_BORK`, `en_AU_ocker`) | One dir per locale, one file per module per key (`en/location/city_name.ts`). Fallback chain per Faker instance: `[de_CH, de, en, base]`; first locale that defines a key wins. `base` holds locale-independent data (ISO codes, time zones). A key may be explicitly `null` to mean "not applicable" (e.g. no zip codes for HK) so it does not fall through to `en`. https://fakerjs.dev/guide/localization.html |
+| PY | 126 locale codes listed; per provider: address 67, person 86, ssn 56 | Python subclasses per `providers/<provider>/<locale>/__init__.py` overriding tuples/formats of the base. Missing provider for a locale falls back to `en_US`. Coverage is uneven (`am_ET` = phone only). Multi-locale `Faker(['it_IT','en_US'])` with optional weights. https://faker.readthedocs.io/en/master/locales.html |
+| GO | 1 (English/US). No locale API; `Language`/`LanguageBCP` only emit language codes | Static Go maps in `data/*.go`. Open issue #352 "generate data for specific country/language" unanswered. https://github.com/brianvoe/gofakeit/issues/352 |
+| DF | 127 yml files in `src/main/resources` (incl. 49 country-only files like `_SE.yml` and 78 language(-region) files); README says "60+" | One YAML per locale (`sv-SE.yml`) under `sv-SE: faker: address: …`; `en/` split into 264 per-provider files. Locale chain `de_CH → de → en`, key-by-key. Values are templates `#{Name.first_name}`. Custom yml via `faker.addPath(locale, path)`. Country file (`_SE.yml`) layered over language file. |
+| BG | 50 (`data/*.locale.json`; README says 46) | Verbatim copy of faker.js (v5-era) locale JSON, merged with `data_extend/*.locale.json` by a gulp task, shipped as BSON. Falls back to `en`. https://github.com/bchavez/Bogus/wiki/Creating-Locales |
+| MM | 47 (53 dirs incl. `global`, `int`, `bin`, template) | Per locale six JSON files: `address, datetime, finance, food, person, text`. Everything else (internet, payment, code, hardware…) is locale-independent. |
+| CH | 1 (en; `postal` CA and `postcode` GB formats exist, `locale()` only returns codes) | Data inline in `chance.js`. |
+| RB | 58 yml files ("over 40" in README) | One YAML per locale `lib/locales/<code>.yml` (`en` split into `en/*.yml`), I18n gem, key-level fallback to `en`. |
+
+### Address data: real or synthetic, and correlation
+
+| Lib | Cities | Street names | Postcodes | Hierarchy / correlation |
+|---|---|---|---|---|
+| JS | `en`: real list `city_name` (~1000 US cities) is one of 5 `city_pattern`s; the other 4 are `{prefix} {firstName}{suffix}` style synthetic. `de`: 327 real cities; `sv`: no `city_name`, pattern-only synthetic. Varies per locale. | `en`: `street_pattern` = `{firstName} {street_suffix}` / `{lastName} …` (synthetic); `de`: 1800 real street names (from one NRW region). | Format masks (`#####`, `#####-####`). `en_US` has `postcode_by_state` so `zipCode({state:'CA'})` gives a state-prefixed range. | None between city↔state↔zip. `county` is a flat list. `timeZone` is per-locale. `nearbyGPSCoordinate` is the only "correlated" geo function. Issue #983: en_CA returned US cities; PR #2141 added real cities for ZA locales — real-city coverage is locale-by-locale volunteer work. |
+| PY | Per locale. `en_US`/`en_GB`: synthetic (`{city_prefix} {first_name}{city_suffix}`). `de_DE` 400+ real, `sv_SE` 45 real, `pl_PL` real, `ja_JP` real prefectures/cities/towns, `es_ES` real provinces + regions. | `en_US`: `{first_name} {street_suffix}`; `sv_SE`: prefix+suffix ("Björkgatan"); `de_DE`: synthetic name+Straße; `pl_PL`: real street-name pools. | Masks. `en_US`: `postcode_in_state` with real per-state numeric ranges (still random within range, "not correlated with cities"). `fr_FR`: postcode built from a real department number (correlated to department, not to city). `en_GB`: real outward-area letters but random assembly. `sv_SE`: `%####`. | No country→state→city hierarchy anywhere; `ja_JP` has prefecture–city pairs in data but methods pick independently. `geo.local_latlng(country)` returns real geonames places (name, lat, lon, country, tz) — the only real, correlated location record in the library. `address_formats` are weighted (`25.0` standard vs `1.0` military). |
+| GO | Real list of major US cities | Real-looking suffix lists; StreetName is a pool | `#####` mask | None: `Address()` picks city, state, zip independently; lat/lon are uniform random on the globe, not near the city. Open issue #196 "Better address" unanswered. |
+| DF | `en`: `city` formats are all synthetic (`#{city_prefix} #{Name.first_name}#{city_suffix}`); `cityName` draws from a separate real list. `sv-SE`: real `city_name` list; `de`: synthetic. | `en`: `#{Name.first_name} #{street_suffix}`; `de`: real `street_root` list combined randomly. | Masks; `en-US` `postcode_by_state` (`350##`), `zipCodeByState`, `countyByZipCode` (en-US only), `eircode` (IE). | None by default. Issue #1551 (Locale.CHINA yields "福建省南京市") open as "proposal". Issue #1477 unresolved directive for city names in some locales. |
+| BG | Whatever faker.js v5 had per locale (`en`: synthetic patterns only — real `city_name` list postdates the copy) | synthetic | Masks; `en_US` `postcode_by_state` | None. Issue #481 "Invalid Zip Codes (US)" open. Issue #342 "City returns people's names" (side effect of `{firstName}` patterns). |
+| MM | Real lists per locale (`en`: real US cities; `sv`: real municipalities incl. small towns) | `en`: real San Francisco street names; `sv`: real Swedish street names | `postal_code_fmt` mask per locale; `sv` has none | `state` (name+ISO 3166-2 abbr) real, but no link to city or postcode; `default_country` tied to locale. Open question #1584 on choosing default city per locale. |
+| CH | Synthetic (3-syllable word) | Synthetic word + real suffix | Real-format masks | None |
+| RB | `en`: synthetic patterns (`city_with_state` exists but is `city, state` random) ; some locales have real lists | `{first_name} {street_suffix}` | `#####`; `zip_code(state_abbreviation:)` uses per-state pattern (`900##`); doc says "may not be an actual state zip" | `full_address_as_hash` returns fields but does not correlate them. Issue #2881 (state vs state_abbr mismatch, country vs country_code mismatch) open; #1958 asked for geocodable city+zip; #2899 fake zips closed "not planned". |
+
+**Which libraries correlate any address fields at all:** Python Faker (`fr_FR` postcode↔department, `en_US` zip↔state range, `geo.local_latlng` real records), faker-js/Datafaker/Bogus/Ruby (`zip↔state` range only, en_US), Mimesis (locale↔country only). None ships a country→region→city→postcode hierarchy or city↔lat/lon.
+
+### Other localized fields (what varies per locale)
+
+- JS: `sv` overrides color, commerce, company, date, internet, location, person, phone_number, team; `en` defines 24 modules; `base` 9. So finance/vehicle/animal/food/science are effectively English-only.
+- PY: per-locale providers exist for address, automotive (plate formats), bank (IBAN/BBAN/SWIFT by country), company (VAT IDs), currency, date_time (month/day names), internet (TLDs, free email domains, user-name formats), job, lorem (word lists), person, phone_number, ssn (national IDs with check digits), passport, color, barcode (EAN prefixes), misc.
+- DF `de.yml` overrides: address, company, compass, creature, lorem, hipster, name, color, commerce, book, university, chuck_norris, space, music, games, food, simpsons, dr_who, vehicle. Country files (`_SE.yml`) mainly phone_number and address formats.
+- MM: only address, datetime, finance, food, person, text are localized.
+
+### Name weighting
+
+| Lib | Weighted by frequency? |
+|---|---|
+| PY | Yes, default `use_weighting=True`. `en_US`: ~500 female + ~500 male first names weighted from SSA decade tables, ~1000 surnames weighted by US Census; `sv_SE`: ~1000/1000/1000 weighted from Skatteverket/ISOF. Plus weighted `address_formats`, `random_element(OrderedDict)`. |
+| JS | No. Plain lists (`en`: ~650 female, ~550 male, ~400 generic first names). `helpers.weightedArrayElement` exists for user data only. |
+| GO | No. ~1500 first, ~500 last, plain slices. `Weighted(options, weights)` for user data. |
+| DF | No. `en`: ~1200 male, ~4400 female first names, ~1000 last, unweighted. |
+| BG | No (faker.js data); `Randomizer.WeightedRandom` for user data. |
+| MM | No. `en`: ~4000 female, ~3000 male, ~2500 surnames, unweighted. |
+| CH | No; `weighted()` helper only. |
+| RB | No. |
+
+## 3. Common complaints (issues and posts)
+
+- **Uncorrelated address parts** — the single most repeated request across every library:
+  - faker-ruby #1958 "Need an address object that has a street, city, zip that correlate" (geocoding breaks) https://github.com/faker-ruby/faker/issues/1958; #2881 "Consistent addresses" (state ≠ state_abbr, country ≠ country_code) https://github.com/faker-ruby/faker/issues/2881; #2899 "Fake Zip Codes" closed not-planned https://github.com/faker-ruby/faker/issues/2899; #275 zips like 90099 https://github.com/stympy/faker/issues/275
+  - fzaninotto/Faker #625 "Faking Legitimate Addresses" https://github.com/fzaninotto/Faker/issues/625
+  - java-faker #378 "Italy, Lima" / "USA, Moscow" https://github.com/DiUS/java-faker/issues/378; Datafaker #1551 wrong province–city for China https://github.com/datafaker-net/datafaker/issues/1551
+  - gofakeit #196 "city may not be in the same state as the zip" https://github.com/brianvoe/gofakeit/issues/196
+  - Bogus #481 invalid US zips https://github.com/bchavez/Bogus/issues/481
+  - Python Faker #697 "city in country" https://github.com/joke2k/faker/issues/697; #1812 `states_postcode` fails for territories
+- **Synthetic city names** ("East Jarretmouth", "Nord Müller-stadt"): faker-js #2021 merged `city`/`cityName` and documented that only some patterns are real https://github.com/faker-js/faker/issues/2021; Bogus #342 `Address.City` returns people's names https://github.com/bchavez/Bogus/issues/342; faker-js PR #2141 / #2127 / #3792 adding "real" or "more realistic" cities per locale.
+- **Localized data falling through to English**: faker-js #983 en_CA cities were US cities https://github.com/faker-js/faker/issues/983; Datafaker #1477 unresolved directive for city names https://github.com/datafaker-net/datafaker/issues/1477; #1358 ru-MD city names; Python #2432 vi_VN outdated administrative units.
+- **Fields within one record don't match**: Python #1185 profile name vs username/email https://github.com/joke2k/faker/issues/1185; #1420 email consistent with name https://github.com/joke2k/faker/issues/1420 (both stale, no maintainer answer); gofakeit `Person()` picks gender and first name independently and email from a fresh name.
+- **No frequency weighting / flat distributions**: blood types come out uniform when O+ and A+ are >70% of population (https://www.statology.org/how-to-validate-enhance-faker-profile-data-generation/); Python's `use_weighting` is the only built-in answer and only some locales supply weights; #1815 weighting silently dropped after adding a provider https://github.com/joke2k/faker/issues/1815.
+- **Formats that produce impossible values**: Python #1849 US phone numbers that can't exist; #1868 fr_FR postcodes with <5 digits; faker-js #1159 unrealistic BIC; #3429 routing numbers fixed to use a real Federal Reserve lookup table; Ruby #1123 wrong pt-BR zip ranges.
+- **No locales at all** in gofakeit (#352, open) and Chance.
+- **"Column-by-column" generation, no relationships between columns/tables** as a general critique of Faker-style tools (https://securityboulevard.com/2026/04/best-synthetic-data-generation-tools-and-platforms-compared-for-2026/).
+- **Uniqueness**: faker-js removed `faker.unique` and tells users to roll their own (https://fakerjs.dev/guide/unique.html); Python `fake.unique` raises `UniquenessException` after retries.
+
+## 4. Notable and unusual features
+
+- **Datafaker** — expression language `#{Name.first_name}`, `#{regexify '[a-z]{4,10}'}`, `#{numerify '##'}`, `#{bothify}`, `#{letterify}`, `#{templatify}`, `#{examplify}`, `#{options.option 'A','B'}`, `#{csv …}`, `#{json …}`, method calls with args (`#{date.birthday 'yy DDD'}`); nested resolution (https://www.datafaker.net/documentation/expressions/). `Schema.of(field(...), compositeField(...))` with transformers to CSV, JSON, YAML, XML, TOML, SQL (batch inserts, Postgres/Oracle/Spark dialects, ARRAY/MULTISET/ROW), `@FakeForSchema` POJO population (https://www.datafaker.net/documentation/schemas/). `faker.collection(...).len(3,5)`, infinite `stream()`, `faker.unique()`, sequences, custom yml via `addPath`. 263 providers, the largest pop-culture catalogue (≈110 franchises/games).
+- **gofakeit** — Go `text/template` based `Template()`, `Markdown()`, `EmailText()`, `FixedWidth()`; `Generate("{firstname} {regex:[a-z]{5}} {randomstring:[a,b]}")` mini-language; `Struct(&v)` fill via `fake:"{city}"`, `fakesize`, `format` tags; `Slice`, `Map`, `Regex`; `CSV/JSON/XML/SQL` output; `AddFuncLookup` registers custom functions with parameter metadata (used by its CLI and HTTP server); `Weighted()`; word families split by grammatical role (46 Noun/Verb/Adjective/Pronoun/Connective sub-kinds); `Error*` generators; Minecraft.
+- **Mimesis** — `Field`/`Fieldset`/`Schema` builder (`Field(locale)("person.full_name")`, keyed lookups with `key=` post-processors like `maybe`, `romanize`), `Schema(schema=lambda: {...}, iterations=n)` exporting to JSON/CSV/pickle, relational data with `SchemaRef` foreign keys, custom field handlers, factory-boy integration, fastest Python generator; locale = six JSON files; ISO country codes in A2/A3/numeric; DMS coordinates.
+- **Bogus** — `Faker<T>().RuleFor(x => x.Prop, f => …)` fluent rules with `StrictMode`, `CustomInstantiator`, `FinishWith`, `Rules()`, `Ignore`, rule sets; local vs global seeding, `UseDateTimeReference`; `Bogus.Distributions.Gaussian`; country ID extension packages; premium Locations/Healthcare/Hollywood/Text packages; Roslyn analyzer; `AutoBogus` auto-fills any class.
+- **faker-js** — locale fallback stack with explicit `null` = "not applicable"; `mergeLocales`; `helpers.fake('{{person.firstName}}')` template; `helpers.fromRegExp`, `weightedArrayElement`, `multiple`, `uniqueArray`, `mustache`; pluggable `Randomizer` (32/53-bit Mersenne), `Distributors` (uniform, exponential); `date.birthdate({mode:'age'|'year'})`; `location.nearbyGPSCoordinate`; `git` module; `person.bio`; typed per-locale definition files generated by script.
+- **Python Faker** — `use_weighting` (frequency-weighted names/formats from census data); `OrderedDict` weighted `random_element`; `fake.unique`, `fake.optional`; multi-locale instance with weights; `geo.local_latlng` real geonames records; `misc` file/archive/CSV/JSON/image generators; `pystr_format`; pytest fixture; CLI; 24 community providers (healthcare, market data, observability, airtravel, education, security, pyspark …).
+- **Chance** — `weighted`, `normal` (Gaussian), `unique`, `n`, `mixin`, `set` (override data), `pick/pickone/pickset`; mobile IDs (android_id, apple_token, bb_pin, wp7/8 anid); geohash/altitude/depth.
+- **Ruby faker** — `full_address_as_hash`, `Faker::Config.locale`, `Faker::UniqueGenerator`, `Faker::Config.random`; `Types` generator for random Ruby types; company registration numbers for ~12 countries; deep pop-culture catalogue (~85 themed generators).

docs/research/research-sources-se.md

@@ -0,0 +1,146 @@
+# Swedish (sv_SE) fake-data sources — research 2026-09-17
+
+Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
+"Verified" = fetched and read for this research.
+
+## Licence summary (embed in MIT repo?)
+
+| Source | Licence (exact) | Embed in MIT? | Attribution |
+|---|---|---|---|
+| SCB Statistikdatabasen + geodata | CC0 1.0 Universal | Yes | None required; SCB asks "Källa: SCB" for statistics taken from scb.se |
+| SCB files on scb.se (xlsx: names, SNI, SSYK, SUN) | Not stated per file; scb.se terms: cite "Källa: SCB" | Yes (flag: CC0 statement names only statistikdatabasen/geodata) | "Källa: SCB" |
+| Skatteverket testpersonnummer / testsamordningsnummer | CC0 1.0 (in DCAT) | Yes | None |
+| Skatteverket namn på nyfödda, efternamn lists | No licence in DCAT/page; "fri att använda, inga avtal eller avgifter" (PSI law 2022:818) | Yes, flag | Cite Skatteverket |
+| Bolagsverket statistics CSVs | CC BY 2.5 SE | Yes | Attribution required |
+| Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
+| Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
+| Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
+| PTS numbering plan PDF, pts.se number pages | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
+| Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
+| SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
+| Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
+| Folkets lexikon | CC BY-SA 2.5 Generic | No (share-alike) | — |
+| SAOL (svenska.se) | Copyright, written permission required | No | — |
+| Hunspell sv_SE (DSSO) | MPL 1.1 / GPL 2 / LGPL 2.1 (tri); yeager/hunspell-sv LGPL-3.0 | Avoid | — |
+| Mobility Sweden registrations | "ange källa" (no open licence) | Top-N facts only | Cite Mobility Sweden |
+| Wikipedia (sv/en) | CC BY-SA 4.0 | No copying; facts only | — |
+| CLDR (via ICU/Intl) | Unicode licence | Yes | Unicode notice |
+
+## 1. Person names
+
+### SCB (historic, frozen)
+- SCB stopped producing name statistics from 2024 and refers to Skatteverket (Namnsök page, verified). Statistikdatabasen folder BE0001 now holds only "Äldre tabeller som inte längre uppdateras" (BE0001D newborns, BE0001G whole population).
+- Whole-population xlsx (verified, downloaded): https://www.scb.se/contentassets/9fe7dbb460994c72b835163dbc491ef9/namn-med-minst-tva-barare-31-december-2022.xlsx — 13.1 MB, last-modified 2024-11-21, reference date 2022-12-31, names with >= 2 bearers, names in UPPERCASE.
+
+| Sheet | Rows (incl. 4 header rows) | Columns |
+|---|---|---|
+| Efternamn | 411,802 | Efternamn, Antal bärare |
+| Förnamn kvinnor | 91,247 | Förnamn, Antal bärare |
+| Förnamn män | 79,128 | Förnamn, Antal bärare |
+| Tilltalsnamn kvinnor | 57,785 | Tilltalsnamn, Antal bärare, Medelålder (only where >= 10 bearers, else "-") |
+| Tilltalsnamn män | 49,404 | same |
+
+- Gender split: a name appearing in both kvinnor/män sheets gives P(female) = count_k / (count_k + count_m).
+- Middle names: "Förnamn" = every given name a person carries; "Tilltalsnamn" = the one used. Förnamn-minus-tilltalsnamn frequency approximates middle-name usage. Legal "mellannamn" (a second surname) is a different concept, no longer grantable under namnlagen 2016:1013 (not verified this session).
+- Junk rows to filter: single letters ("A"), initials ("A-C", "K C", "JR"), "A:SON".
+- Legacy PxWeb tables (folder BE0001G, verified via web UI): BE0001T06AR, BE0001TNamn10 (tilltalsnamn >= 10 bearers 1999–2020), BE0001T100, BE0001T08AR, BE0001T07AR, BE0001FNamn10 (förnamn >= 10 bearers), BE0001F100, BE0001T03Ar (efternamn top), BE0001ENamn10 (efternamn >= 10 bearers 1999–2020). PxWebApi v1 (`api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0001/BE0001G/...`) returns HTTP 400 and PxWebApi 2 (`/v2beta/api/v2/tables/BE0001T06AR`) returns "Non-existent table" — these tables are web-UI only now (verified). Use the xlsx.
+- Licence: SCB terms page (verified): "Statistik och geodata som SCB tillgängliggör som öppna data i statistikdatabasen och i vår geodataplattform har licensen Creative commons 0 1.0 Universal, CC0" and "Om du använder eller sprider statistik från scb.se ska dock som källa alltid SCB anges". The xlsx sits on scb.se, not in statistikdatabasen — flag; treat as CC0 + "Källa: SCB".
+
+### Skatteverket (current)
+- Namn på nyfödda (2021+), annual: DCAT https://skatteverket.entryscape.net/store/26/metadata/416; CSV `fbf_namn_nyfodda.csv` https://skatteverket.entryscape.net/store/26/resource/417; JSON API https://skatteverket.entryscape.net/rowstore/dataset/da2556d0-c717-45e8-a1d8-3320161d3a7d/json?_limit=100&_offset=0 (47,529 rows, verified). Columns: rangordning, kön (Kvinna/Man), fodelsear, uppdateringsdatum, namn, gruppering (Kommun/…), grupperingsvärde (e.g. ALE), antal. Names in Title case. No licence field in DCAT — flag.
+- Most common surnames (> 2,000 bearers, ~500 names), 2026 text file (verified, 517 lines, format `Andersson 211808`): https://www.skatteverket.se/download/18.70685bee19c85dd5dd03b59/1775049117448/Fria_efternamn_namn_antal_textfil_2026.txt (also two PDFs). Page: https://www.skatteverket.se/privat/folkbokforing/namn/bytaefternamn/sokblanddevanligasteefternamnen.4.515a6be615c637b9aa48e09.html. Licence not stated.
+- Skatteverket general open-data terms (verified): "Vår öppna data är fri att använda och kräver varken några avtal eller innebär några avgifter" under lag 2022:818; no CC licence named except on the test-number datasets (CC0).
+- Skatteverket publishes no whole-population first-name list as a file; its statistikportalen "Så många har ett visst namn" is an interactive search (page 404 on fetch; unverified).
+
+Recommendation: embed SCB 2022 xlsx-derived lists (top-N per sheet with counts) + Skatteverket 2026 surname counts for freshness.
+
+## 2. Identifiers
+
+### Personnummer (Folkbokföringslagen 1991:481 18 §, verified via lagen.nu)
+- 10 digits: YYMMDD + födelsenummer (3 digits) + kontrollsiffra. Födelsenummer odd = male, even = female. Separator "-" ; "+" once the person turns 100. 12-digit form YYYYMMDDNNNC (no separator).
+- Check digit: Luhn/mod 10 over the 9 digits YYMMDDNNN (weights 2,1,2,1,… from the left; sum digits of two-digit products; check = (10 − sum mod 10) mod 10).
+- Before 1990 the first two födelsenummer digits encoded county; now random (Wikipedia, unverified officially).
+
+### Samordningsnummer
+- Same layout; day + 60 (3rd → 63); individual number random 001–999, odd male / even female; same Luhn (Skatteverket rättslig vägledning, via search excerpt; page itself blocked). Now regulated by Lag (2022:1697) om samordningsnummer (18 a § FBL repealed) — flag.
+- Example from Skatteverket: man born 1970-10-03, no. 239 → 701063-2391.
+
+### Official test numbers (Skatteverket, CC0 1.0)
+- Testpersonnummer dataset: https://www.dataportal.se/sv/datasets/6_67959/testpersonnummer (DCAT: https://admin.dataportal.se/store/6/metadata/67959?recursive=dcat). "Testpersonnummer med sekelsiffror för åren 1890-2025. Nya testnummer för kommande år läggs ut i december varje år." Formats: Excel (one file), CSV (split into many period files), JSON API https://skatteverket.entryscape.net/rowstore/dataset/b4de7df7-63c0-4e7e-bb59-1f156a591763/json?_limit=100 — resultCount 43,895 (verified), one column `testpersonnummer`, 12 digits.
+- Observed pattern (verified on 1890s CSV file fully and API samples at offsets 0/10000/30000/43880): every day gets two numbers, födelsenummer 238 (female) and 239 (male), through 2026-12-31; 1890s use 980/981 (plus a few 982/983/936). All Luhn-valid. Skatteverket blocks these from ever being assigned.
+- Testsamordningsnummer: DCAT https://skatteverket.entryscape.net/store/9/metadata/153; years 1914–2025; CSV per year; sample file 2022 (1,498 rows): days 61–91, individual numbers 238/239, Luhn-valid.
+- Should a generator restrict to these? Skatteverket does not mandate it, but only these are guaranteed never to collide with a real person; any other Luhn-valid number may be real. Recommended: default to the official series (date + 238/239, or 980/981 for 1890s) and embed nothing but the rule (~530 kB if the full list were embedded). Flag: the 238/239 rule was sampled, not exhaustively verified for every year.
+- Common practice elsewhere (Faker etc.): random date + random 3 digits + Luhn — valid but collision-prone.
+
+### Organisationsnummer
+- 10 digits NNNNNN-NNNN, last = Luhn over first 9 (same algorithm as personnummer). Digits 3–4 ("month") are always >= 20, distinguishing from personnummer (Wikipedia; not found on an official page — flag).
+- First digit (gruppnummer), Bolagsverket page (verified): 5 = aktiebolag, filialer, banker, försäkringsbolag, europabolag; 9 = handelsbolag, kommanditbolag; 7 or 8 = bostadsrättsföreningar, ekonomiska föreningar, näringsdrivande ideella föreningar etc.; 2 or 8 = trossamfund; 20… = state agencies (Bolagsverket itself 202100-5489); 3 = foreign companies. Wikipedia adds: 1 = dödsbon, 2 = stat/region/kommun, 6 = samfällighetsföreningar, 8 = ideella föreningar/stiftelser. Enskild firma = owner's personnummer.
+- VAT number (Skatteverket, verified): "SE" + 12 digits = organisationsnummer (10) + "01" ("De två sista siffrorna är alltid 01"). Written SE556047352101.
+
+### Bankgiro / Plusgiro (Bankgirot "10-modul" PDF 2016-12-01, verified)
+- Bankgironummer: 7 or 8 digits, last digit mod-10 (Luhn) check; printed 991-2346 (7) / 5555-5551 (8), i.e. hyphen before the last four digits. 90-konton (charity) 900-000x…904-999x, always 7 digits (Wikipedia).
+- Plusgironummer: 2–8 digits, last digit Luhn (Wikipedia/samlogic; not from an official Nordea page — flag). Printed with hyphen before check digit, e.g. "12 34 56-7" (convention, unverified).
+
+### Clearing numbers / bank accounts
+- Bankinfrastruktur i Sverige AB CSV (verified, 57 rows, 40 actors): https://www.bankinfrastruktur.se/media/1melztro/tabell-over-clearingnummer-250305.csv (also `/media/sqfjsvkp/clearingnummertabell-for-nedladdning.csv`, 63 rows, newer, includes Zimpler 2130-2139). Columns: `Clearingnummer;Aktör;BIC;IBAN ID;Konto-typ;Metod IBAN konvertering`. Semicolon, UTF-8 BOM. Page: https://www.bankinfrastruktur.se/framtidens-betalningsinfrastruktur/iban-och-svenskt-nationellt-kontonummer. No licence; disclaimer "garanterar inte att publicerade uppgifter är korrekta".
+- Key ranges: Nordea 1100–1199, 1400–2099, 3000–3399, 3410–3999, 4000–4999 (3300 and 3782 = personkonto); Danske 1200–1399, 2400–2499; SEB 5000–5999, 9120–9124, 9130–9149; Handelsbanken 6000–6999; Swedbank 7000–7999 (4-digit) and 8000–8999 (5-digit clearing with own check digit); Länsförsäkringar 3400–3409, 9020–9029, 9060–9069; Skandiabanken 9150–9169; ICA 9270–9279; Avanza 9550–9569; Nordnet 9100–9109; Klarna 9780–9789.
+- Bankgirot "Bankernas kontonummer" 2024-02-22 (verified text): Typ 1 = 4-digit clearing + 7-digit account (11 digits) with mod-11 check, weights 1,10,9,…,1; comment 1 = weigh clearing minus first digit + 7 digits; comment 2 = whole clearing + 7 digits. Typ 2 = clearing not part of the account: Handelsbanken 9-digit account mod-11 (comment 2); Swedbank 8000–8999 up to 10 digits mod-10 (comment 3); Danske 9180–9189, Nordea personkonto 3300/3782, Sparbanken Syd 9570–9579 10 digits mod-10 (comment 1).
+- IBAN SE (Bankinfrastruktur page, verified): 24 chars = "SE" + 2 check digits + 3-digit IBAN ID (from CSV column, e.g. Nordea 300, SEB 500, Handelsbanken 600, Swedbank 800, Danske 120, LF 902) + 17-digit account (zero-padded; clearing included/excluded per "Metod IBAN konvertering" 1–3).
+
+## 3. Phone
+
+- Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
+- Reserved for fiction, PTS "Telefonnummer till böcker och filmer" (verified 2026-09-21; curl and Playwright are Radware-blocked, a rendering fetcher gets through): https://www.pts.se/internet-och-telefoni/telefonnummer-och-adressering/telefonnummer-till-bocker-och-filmer/ — mobile 070-1740605–1740699, geographic 08-46500400–46500499 (Stockholm), 031-3900600–3900699 (Göteborg), 040-6280400–6280499 (Malmö), 0980-319200–319299 (Kiruna). 495 numbers PTS holds back so they reach nobody, which is goal 15's never-reaching path for `sv_SE.phone`. Same licence status as the numbering plan: none stated, facts only, cite PTS. The mobile block fits the shipped `{a} {b} {c}` grouping (`174 06 05`–`99`), as do 031 and 040; Stockholm's 8 subscriber digits and Kiruna's 6 need a format of their own, and 0980 is not among the 20 shipped area codes.
+- 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
+- Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
+- Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).
+- Wikipedia "Lista över riktnummer i Sverige" cites the same PTS document (CC BY-SA — do not copy).
+
+## 4. Company
+
+- Legal forms + suffixes (Bolagsverket, lag 2018:1653 om företagsnamn): aktiebolag must contain "aktiebolag" or "AB" (verified); public companies add "(publ)"; handelsbolag "HB" / "handelsbolag", kommanditbolag "KB" / "kommanditbolag", ekonomisk förening "ekonomisk förening" / "ek. för.", enskild firma no suffix — statutory wording for HB/KB/ek.för. not verified on the fetched page (flag).
+- Legal-form codes (SCB "juridisk form", from Skatteverket organisationsnummer page and SCB variabelbeskrivning): 10 fysiska personer, 21 enkla bolag, 22 partrederier, 23 värdepappersfonder, 31 handelsbolag/kommanditbolag, 32 gruvbolag, 41 bankaktiebolag, 42 försäkringsaktiebolag, 43 europabolag, 49 övriga aktiebolag, 51 ekonomiska föreningar, 53 bostadsrättsföreningar, 54 kooperativ hyresrättsförening, 61 ideella föreningar, 62 samfälligheter, 63 registrerade trossamfund, 71 familjestiftelser, 72 övriga stiftelser, 81 statliga enheter, 82 kommuner, 84 regioner, 87 offentliga korporationer, 88 hypoteksföreningar, 91 oskiftade dödsbon, 92 ömsesidiga försäkringsbolag, 93 sparbanker, 94 understödsföreningar, 95 arbetslöshetskassor, 96 övriga svenska juridiska former, 98 utländska juridiska personer — codes beyond 49 quoted from memory of the SCB list, verify against https://www.scb.se/contentassets/8a8eb5c3d45f461ea93482f8e8d4de4f/variabelbeskrivning-api.pdf (flag).
+- Distribution (Bolagsverket ftgstat_oppna.csv, CC BY 2.5 SE, 404,779 rows, latin-1, verified): https://www.bolagsverket.se/statistik/ftgstat_oppna.csv. Columns: ar, manad, handelse (1 = nyregistrerade, 2 = alla registrerade, 3 = avslutade), regfam, län/kommun, then counts per form: AB, BAB (bankaktiebolag), BF (bostadsförening), BRF, EK (ekonomisk förening), E (enskild näringsidkare registered at Bolagsverket only), SE (europabolag), FL (filial), FAB (försäkrings-AB), HB, I (näringsdrivande ideell förening), KB, KHF, MB, SF, SB (sparbank), TSF (trossamfund), BFL, OFB, SCE, S, EGTS, FOF, TPAB, OTPB, TPF. New registrations 2025 (computed): AB 50,859; E 8,468; HB 1,557; BRF 398; EK 319; FL 269; KB 176; I 59. Column description xlsx: https://www.bolagsverket.se/download/18.46f4138717c599ee403aac05/1638951733112/beskrivning-av-csv-fil.xlsx.
+- Company-name register: Bolagsverket "värdefulla datamängder" bulk file https://vardefulla-datamangder.bolagsverket.se/bolagsverket/bolagsverket_bulkfil.zip (250 MB, updated 2026-09-14; org.nr, names, legal form, business description, address) and SCB bulk https://vardefulla-datamangder.bolagsverket.se/scb/scb_bulkfil.zip (71 MB; adds SNI up to 5 codes, status). Free, no agreement; licence not stated on the page (statistics CSVs are CC BY 2.5 SE) — flag. Use offline to derive name-token frequencies; do not embed rows. API (org.nr lookup only; name search "planned, no date"): https://bolagsverket.se/apierochoppnadata/hamtaforetagsinformation/apiforatthamtaforetagsinformation.3988.html.
+- Naming rules (Bolagsverket "Välja företagsnamn", verified): must be distinctive — not only a generic activity ("Bilverkstad AB") or a bare surname; no confusion with existing names/trademarks; no professional titles without credentials; no apparent internet address; "aktiebolag/AB/handelsbolag" ignored when comparing names. Typical patterns (unverified heuristic, derive from bulk file): `<Surname> <Trade> AB`, `<Surname>s <Trade>`, `<Place> <Trade> AB`, `<Fantasy word> AB`, `<Initials> <Trade> HB`, suffix words Bygg, Konsult, Holding, Invest, Fastigheter, Förvaltning, Entreprenad, Teknik, Design.
+- SNI 2007 (SCB, verified xlsx, 94 kB): https://www.scb.se/contentassets/d43b798da37140999abf883e206d0545/sni2007.xlsx — sheets Detaljgrupp (5-digit, 821 codes), Undergrupp (615), Grupp (272), Huvudgrupp (88), Avdelning (21 letters). Code format `01.110` + Swedish label. SNI 2025 replaces it from 2025-12-08 (835/651/287/87/22): https://www.scb.se/globalassets/sni-2025.xlsx (93 kB). Skatteverket/Bolagsverket transition status unverified. Licence: no statement on the classification pages; SCB terms apply ("Källa: SCB") — flag.
+
+## 5. Licence plates and cars
+
+- Format: `ABC 123` (since 1973) and `ABC 12A` (since 2019-01-16; last character a letter, never O). Letters never used: I, Q, V, Å, Ä, Ö. Digits 001–999 (000 not issued). Personal plates 2–7 characters, may use any letters, must not mimic the standard pattern. Source: sv.wikipedia Registreringsskyltar i Sverige (facts; not confirmed on a Transportstyrelsen page — flag).
+- Blocked letter combinations, Transportstyrelsen (verified, page updated 2026-03-31, 99 codes): https://www.transportstyrelsen.se/sv/vagtrafik/fordon/aga-kopa-eller-salja-fordon/registreringsskyltar/byte-av-registreringsnummer/Sparrade-bokstavskombinationer/
+  APA ARG ASS BAJ BSS CUC CUK CUM DUM ETA ETT FAG FAN FEG FEL FEM FES FET FNL FUC FUK FUL GAM GAY GEJ GEY GHB GUD GYN HAT HBT HKH HOR HOT KGB KKK KUC KUF KUG KUK KYK LAM LAT LEM LOJ LSD LUS MAD MAO MEN MES MLB MUS NAZ NRP NSF NYP OND OOO ORM PAJ PKK PLO PMS PUB RAP RAS ROM RPS RUS SEG SEX SJU SOS SPY SUG SUP SUR TBC TOA TOK TRE TYP UFO USA WAM WAR WWW XTC XTZ XUK XXL XXX ZEX ZOG ZPY ZUG ZUP ZOO
+- Car makes, Mobility Sweden 2025 full-year new passenger cars (272,987 total; press release 2026-01-02 "Svagt fordonsår 2025", via search excerpt; "ange källa" required): Volvo 48,961; Volkswagen 38,677; Toyota 22,189; Kia 19,922; Mercedes-Benz 17,372; Skoda 16,664; BMW 15,001; Audi 14,887; Peugeot 9,059; Polestar 7,601. Top models 2025: Volvo XC60 17,933, Volvo EX40, VW ID.7, Tesla Model Y 5,822 (press excerpts). Monthly xlsx: https://mobilitysweden.se/statistik/Nyregistreringar_per_manad_1. Not an open licence — embed as a curated list with attribution.
+- Transportstyrelsen fordonsstatistik (open data, by vehicle type only, not make): https://www.transportstyrelsen.se/sv/om-oss/statistik-och-analys/statistik-inom-vagtrafik/fordonsstatistik/. Trafa (https://www.trafa.se/vagtrafik/fordon/) publishes by owner/technical/region, no make/model list found; licence not stated.
+
+## 6. Words
+
+- SALDO morphology (recommended): https://sprakbanken.se/en/resources/saldom — CC BY 4.0, 128,036 entries, LMF XML `saldom.xml` 254,107,385 bytes (https://svn.spraakbanken.gu.se/sb-arkiv/pub/lmf/saldom/saldom.xml, 2017-09-19). Fields: lemma, POS, inflection paradigm, full word forms with MSD. Derive a small `{lemma, pos}` list (nouns/adjectives/verbs) offline; embed with the citation "Språkbanken (2017). SALDO's morphology. https://doi.org/10.23695/agcm-ny22".
+- Frequencies: Språkbanken word statistics https://sprakbanken.se/resurser/ordstatistik — `stats_all.txt.zip` 763.87 MB, CC BY 4.0, 2025-04-22, aggregated over modern corpora (Korp). Column layout not confirmed (FAQ entry not found; per-corpus files are tab-separated, 6 columns incl. word form, POS, lemgram, SALDO sense, compound flag, frequency) — flag; inspect a per-corpus stats file first (smaller).
+- Folkets lexikon (KTH): CC BY-SA 2.5 — share-alike, not suitable for an MIT-embedded derivative. SAOL: svenska.se "får inte användas … utan skriftligt tillstånd" — unusable. Hunspell sv_SE/DSSO: MPL/GPL/LGPL — avoid.
+
+## 7. Dates, money, email, domains
+
+- CLDR sv-SE (verified with Node 24.18.1 ICU): short date `2026-09-17`; long `17 september 2026`; full `torsdag 17 september 2026 kl. 16:30`; months januari…december (lowercase), short `jan. feb. mars apr. maj juni juli aug. sep. okt. nov. dec.`; weekdays måndag…söndag, short `mån tis ons tors fre lör sön`; currency `1 234,56 kr`; number `1 234 567,891`. ICU uses U+00A0 (NBSP) as group separator; typographic guidance says hard space, decimal comma.
+- Språkrådet (search excerpts): ISO 8601 `ÅÅÅÅ-MM-DD` in formal/technical text, `6 juli 2022` in running text; clock `14.30` (period; colon accepted); amounts `1 234,50 kr`.
+- Public holidays, Lag (1989:253) om allmänna helgdagar (verified): nyårsdagen 1/1, trettondedag jul 6/1, långfredagen, påskdagen, annandag påsk, Kristi himmelsfärdsdag (6th Thursday after Easter), pingstdagen, nationaldagen 6/6, midsommardagen (Saturday 20–26 June), alla helgons dag (Saturday 31 Oct–6 Nov), juldagen 25/12, annandag jul 26/12, plus Sundays. Semesterlagen 3 a § (verified): "Med söndag jämställs allmän helgdag samt midsommarafton, julafton och nyårsafton". Easter needs computing (Gregorian).
+- Email domains: no official statistic found (Internetstiftelsen "Svenskarna och internet" does not report providers). Common providers by general knowledge only (unverified): gmail.com, hotmail.com, outlook.com, icloud.com, yahoo.com/yahoo.se, live.se, telia.com (Telia is phasing out its mail service), bredband.net, comhem.se, spray.se, me.com. Flag as curated.
+- TLDs (Internetstiftelsen registrar stats JSON, verified): .se active domains 1,475,495 (2026-07); .nu 203,940 (2026-07). https://internetstiftelsen.se/registrar-stats/se/activedomainsmonthend.json, `/nu/…`. Licence not stated.
+
+## 8. Other
+
+- Occupations, SSYK 2012 (SCB, verified xlsx): https://www.scb.se/contentassets/0c0089cc085a45d49c1dc83923ad933a/ssyk-2012-koder.xlsx (50 kB; sheets 1/2/3/4-siffer; 429 four-digit codes) and job-title index https://www.scb.se/contentassets/0c0089cc085a45d49c1dc83923ad933a/ssyk_index_webb_2026.xlsx (213 kB, 2026-03-18, thousands of yrkesbenämningar → code). Licence: SCB terms ("Källa: SCB") — flag as with SNI.
+- Education, SUN 2020 (SCB): https://www.scb.se/contentassets/aeeedec0e28c465aa524429407dcd5ba/sun-2020_niva_inriktning2.xlsx (304 kB; levels + orientations). Population 25–64 (SCB "Befolkningens utbildning 2024", via search excerpt): ~10 % förgymnasial, ~40 % gymnasial, ~31 % eftergymnasial >= 3 years (remainder shorter post-secondary). Approximate — flag.
+- Blood groups (geblod.nu 2007 via sv.wikipedia Blodgruppsfördelning; geblod.nu today only says RhD+ ≈ 85 %): A+ 37, O+ 32, B+ 10, AB+ 5, A− 7, O− 6, B− 2, AB− 1 (%).
+- Regional/kommun data: out of scope (geography).
+
+## Unverified / flagged items
+1. CC0 applies by SCB's statement to statistikdatabasen/geodata; xlsx files on scb.se (names, SNI, SSYK, SUN) carry only "Källa: SCB" terms.
+2. Skatteverket name datasets and surname file: no licence field; only "fri att använda" under PSI law.
+3. Testpersonnummer 238/239 rule: sampled (1890s file fully, offsets 0/10k/30k/43.9k), not exhaustively checked; per-period CSV list has many files.
+4. Organisationsnummer "digits 3–4 >= 20" and group digits 1/6 — Wikipedia only.
+5. Plusgiro length/format — Wikipedia/blog only.
+6. PTS: no licence; 264 vs 263 area codes counted in the PDF text; Swedish 2026 PDF and nummer.pts.se blocked (Radware).
+7. Plate format details (000 excluded, O excluded as last letter, 2019-01-16) — Wikipedia only.
+8. Legal-form codes above 49 — from memory; verify against the SCB variabelbeskrivning PDF.
+9. Word-statistics column layout; email-provider list; company-name patterns; education shares — heuristic/approximate.

docs/research/research-sources-world.md

@@ -0,0 +1,616 @@
+# Open data sources and format rules — US/locale-neutral fake-data categories
+
+Researched 2026-09-17 for embedding as data files in an MIT-licensed library. Geography/addresses and Swedish data are out of scope (covered elsewhere). Each item that was not read first-hand from the cited page/file is marked **UNVERIFIED**.
+
+## Licence key (what each licence means inside an MIT repo)
+
+| Licence | Embed? | Obligation |
+|---|---|---|
+| Public domain (US federal work, 17 USC §105), CC0 1.0, PDDL, Unlicense | Yes | None; cite the source in the data file header as courtesy |
+| MIT, BSD-2, ISC | Yes | Keep the copyright + permission notice next to the copied data |
+| Unicode License v3 (OSI-approved 2023-11-17, verified) | Yes | Notice in LICENSE/NOTICE or docs |
+| WordNet License (verified at wordnet.princeton.edu) | Yes | Notice + disclaimer on all copies; no Princeton name in advertising |
+| W3C Document License 2023 | Yes (data tables) | Notice "includes material copied from or derived from <title> <URI>" |
+| CC BY 3.0 / 4.0 | Yes | Creator, licence link, indicate changes (verified deed) |
+| Apache-2.0 (code), LGPL-2.1/3.0 (data) | Awkward | LGPL data files stay LGPL; keep them out of an MIT repo, use as reference for facts only |
+| CC BY-SA, ODbL, CC BY-NC-SA, GFDL | No | Share-alike / non-commercial — incompatible |
+| No licence stated, "all rights reserved", proprietary directories (SWIFT BIC, Fed RTN, CUSIP, ICAO 8643, GS1 page, ISO OBP, SAE WMI, whatismybrowser, useragents.me, MySQL manual, Pantone) | No | Generate synthetically from the format rules instead |
+
+## Embed verdict at a glance
+
+| Category | Use (clean) | Use with notice | Avoid |
+|---|---|---|---|
+| 1 Names/IDs | SSA baby names (CC0), Census 2010 surnames (PD), IRS EIN prefixes / ITIN ranges, SSA SSN rules | — | — (prefix/suffix lists: hand-curate) |
+| 2 Finance | ISO 4217 via datasets/currency-codes (PDDL) or SIX list-one.xml; algorithms (ABA, Luhn, ISIN, CUSIP, IBAN mod 97, EIP-55, Bech32) | npm currency-codes (MIT), GLEIF BIC-LEI (custom notice), CLDR currency symbols (Unicode-3.0) | Fed routing directory, SwiftRef BIC directory, real CUSIP lists, php-iban (LGPL) verbatim |
+| 3 Company | NAICS 2022 (PD), SEC company_tickers.json (free reuse), GLEIF ELF list (CC0) | — | — |
+| 4 Internet | All IANA registries (CC0): media types, HTTP status, ports, TLDs, IPv4/IPv6 special-purpose; RFC 5737/3849/9637 ranges; locally-administered MAC | mime-db (MIT), Linguist languages.yml (MIT), top-user-agents (MIT) / user-agents (BSD-2), IEEE OUI (IEEE "does not assert copyright" — second-hand) | useragents.me, whatismybrowser, MySQL manual tables |
+| 5 Vehicle | NHTSA vPIC makes/models/WMIs/value lists (PD); VIN check-digit rule | — | SAE WMI DB, openalpr plate patterns (AGPL), Wikipedia lists verbatim |
+| 6 Airline | OurAirports (PD/Unlicense), Wikidata airlines + aircraft types (CC0) | OPTD airlines (CC BY 4.0) | OpenFlights + npm mirrors (ODbL), ICAO Doc 8643 + scraped mirrors, OpenSky aircraft DB |
+| 7 Science | PubChem periodic table (PD), NASA planetary fact sheet (PD), Wikidata animals/plants/moons (CC0), Stanford/AABB blood types (facts) | BIPM SI brochure (CC BY 4.0), GBIF backbone (CC BY 4.0 + citation), faker-js lists (MIT) | Bowserinator periodic table (CC BY-SA 3.0) |
+| 8 Text | Moby POS (PD), 12dicts PD lists, XKCD colours (CC0), Lorem ipsum/Cicero, Bartlett via Gutenberg (PD) | WordNet 3.x (WordNet License), Open English WordNet (CC BY 4.0), SCOWL (MIT-like), CSS named colours (W3C notice), Google Books ngrams (CC BY 3.0) | wordfreq data (CC BY-SA), hermitdave (CC BY-SA), Norvig count_1w (LDC-derived), Wikiquote |
+| 9 Dates | IANA tzdb (PD) | CLDR cldr-dates-full (Unicode-3.0) | — |
+| 10 ISO | datasets/country-codes (PDDL), LoC ISO 639-2 (PD, UNVERIFIED layout), Census state.txt (PD) | annexare/Countries, i18n-iso-countries, langs, iso-639-1 (all MIT) | mledoze (ODbL), lukes + stefangabos (CC BY-SA), Debian iso-codes (LGPL), SIL 639-3 table (no-redistribution clause), olahol iso-3166-2 (unclear) |
+| 11 Commerce | Check-digit algorithms (EAN/UPC/GTIN, ISBN-10/13, ISSN, IMEI Luhn); GS1 demo prefix 952 | faker-js en commerce word lists (MIT) | GS1 prefix page + ISBN range file verbatim, Wikipedia tables verbatim |
+| 12 Books/music/food | Gutenberg pg_catalog.csv (CC0), MusicBrainz genre names (CC0), ID3v1 genres, Wikidata instruments/dishes/cuisines/cocktails (CC0), USDA FoodData Central (CC0) | Open Brewery DB (MIT) | MusicBrainz tag associations (CC BY-NC-SA), IBA site text |
+
+## Corrections to the brief's assumptions
+
+- Bowserinator/Periodic-Table-JSON is CC BY-SA 3.0 (not MIT) and has 119 rows; use PubChem CSV (118, PD).
+- wordfreq code is Apache-2.0 and its data CC BY-SA 4.0 — avoid the data.
+- CSS Color 4 has 148 named colours (rebeccapurple included).
+- Gutenberg #27889 Bartlett is the 9th edition (1903); the 1919 10th is not on Gutenberg (UNVERIFIED elsewhere).
+- CLDR file path is `cldr-dates-full/main/<locale>/ca-gregorian.json`; current package 48.2.0.
+- Diners Club US/Canada is a Mastercard co-brand on 55; replace the loose "Maestro 50/56–58/6xxx" rule with the explicit IINs in part 2.
+- Maestro/Discover/Diners/JCB/UnionPay lengths vary 12–19 — generate per the table, not a fixed 16.
+- The 987-65-4320..4329 SSN advertising range is repeated by third parties but not found on any ssa.gov/POMS page (UNVERIFIED).
+- DUNS dropped its mod-10 check digit in 2006 — no validation rule today.
+- IEEE's OUI "does not assert copyright" statement is only recorded second-hand (Debian/FSF); no current IEEE page restates it.
+- Sites blocking automation (403/Cloudflare): ssa.gov, www2.census.gov, swift.com, redcrossblood.org, loc.gov, gs1.org (WebFetch), standards-oui.ieee.org (needs browser UA). Download those manually with a browser UA.
+
+## Recommended default picks for an en_US library
+
+- Given names: SSA `names.zip` aggregated to top-N per sex (CC0). Surnames: Census `Names_2010Census.csv` top-N (PD).
+- Countries: `datasets/country-codes` CSV (PDDL); flag emoji derived from alpha-2. Currencies: same file's ISO 4217 columns or `datasets/currency-codes`; symbols from CLDR `en` (Unicode-3.0).
+- Internet: IANA registries + mime-db; UAs from templates (no data) or `top-user-agents` (MIT).
+- Vehicles: vPIC `GetMakesForVehicleType/car` (195) + models per make; VIN generated with check digit.
+- Airports: OurAirports rows with `iata_code` and type large/medium (4,569). Airlines: Wikidata (CC0).
+- Words: WordNet 3.0 index files (with LICENSE) or Moby POS (PD) for POS-tagged words; XKCD + CSS for colours.
+- Timezones: `zone1970.tab` names only; offsets via `Intl` at runtime.
+- Check digits: implement from the rules in parts 1, 2 and 4 — no data needed.
+
+---
+
+# Part 1 — Names, finance, company: sources and rules
+
+Researched 2026-09-17. "Embed" = ship as a data file in an MIT repo. Everything not marked UNVERIFIED was read from the cited page or file this session.
+
+## 1. US person names and identifiers
+
+### SSA baby names (given names by year and sex)
+- Page: https://www.ssa.gov/oact/babynames/limits.html — national zip https://www.ssa.gov/oact/babynames/names.zip (7 MB), state zip https://www.ssa.gov/oact/babynames/state/namesbystate.zip (23 MB), territories 228 kB.
+- Format (per SSA page + readme as mirrored by https://github.com/hackerb9/ssa-baby-names): one `yobYYYY.txt` per year 1880–2025, CSV `name,sex,count`, name 2–15 chars, sex `M`/`F`, sorted by sex then count desc. Names with < 5 occurrences in a year are excluded.
+- Counts: ~100,364 distinct names, ~2.02 M rows across all years (hackerb9 snapshot; year of snapshot UNVERIFIED). Top 1000 names cover 71.5 % of 2025 births (SSA page).
+- Licence: data.gov catalog entry https://catalog.data.gov/dataset/baby-names-from-social-security-card-applications-national-data lists licence **CC0 1.0** (`https://creativecommons.org/publicdomain/zero/1.0/`), publisher SSA. US federal work, no attribution required; embeddable.
+- Note: ssa.gov returns 403 to non-browser user agents (curl/WebFetch); download with a browser UA or via the data.gov link.
+
+### Census 2010 surnames
+- Page: https://www.census.gov/topics/population/genealogy/data/2010_surnames.html
+- Files: full set https://www2.census.gov/topics/genealogy/2010surnames/names.zip (CSV + XLSX, `Names_2010Census.csv`), top 1000 `Names_2010Census_Top1000.xlsx`, docs https://www2.census.gov/topics/genealogy/2010surnames/surnames.pdf.
+- Fields (from surnames.pdf): `name, rank, count, prop100k, cum_prop100k, pctwhite, pctblack, pctapi, pctaian, pct2prace, pcthispanic`. `(S)` = suppressed percentage. Names are UPPERCASE.
+- Count: 162,253 surnames occurring ≥ 100 times (covers 90.1 % of people with a recorded surname); the CSV also has a trailing `ALL OTHER NAMES` row (UNVERIFIED — www2.census.gov returned 403 to curl this session; download with a browser).
+- Licence: US Census Bureau work → US federal government work, public domain in the US (17 U.S.C. §105). Census only asks for citation (https://www.census.gov/about/policies/citation.html: "U.S. Census Bureau, [Table], [Product], [Vintage], [URL], accessed on [date]"). Embeddable; cite source in the data file header.
+
+### Name prefixes / suffixes
+- No authoritative enumerated list exists. USPS Publication 28 §33 (https://pe.usps.com/text/pub28/28c3_011.htm) defines the fields: Name Prefix, First Name, Middle Name or Initial, Surname, **Suffix Title** = "maturity (e.g., JR, SR) and professional (e.g., PHD, DDS) suffixes"; example "MR. WALTER W. WITHERSPOON JR.".
+- Recommendation: hand-curate. Prefixes `Mr., Mrs., Ms., Mx., Dr., Rev.`; suffixes `Jr., Sr., II, III, IV, PhD, MD, DDS, Esq.`. Rule of thumb: generational suffix (Jr/Sr/II–IV) only with a male name in traditional US usage; never combine Jr. with Sr.; at most one generational suffix.
+
+### SSN
+- Structure: `AAA-GG-SSSS` (area 3, group 2, serial 4). No check digit (Wikipedia).
+- Randomization since **2011-06-25** (https://www.ssa.gov/employer/randomization.html): "Previously unassigned area numbers were introduced for assignment excluding area numbers 000, 666 and 900-999." Area no longer geographic.
+- Invalid per SSA POMS RM 10201.035 (https://secure.ssa.gov/apps10/poms.nsf/lnx/0110201035): area `000`, `666`, `900–999`; group `00`; serial `0000`.
+- Generator rule: area ∈ [001,899] \ {666}; group ∈ [01,99]; serial ∈ [0001,9999].
+- Advertising SSNs (https://www.ssa.gov/history/ssn/misused.html): `078-05-1120` (Woolworth wallet card, 1938; > 40,000 people claimed it) and `219-09-9999` (1940 Social Security Board pamphlet). Both are structurally valid — exclude from output.
+- `987-65-4320`–`987-65-4329` "reserved for advertising": repeated by searchbug.com, ssn-check.org, a Missouri state standard (https://oa.mo.gov/sites/default/files/CC-SocialSecurityNumberNamingStandardV050305.pdf, which even misprints it as 987-54-4329). **UNVERIFIED** — not found on any ssa.gov page or POMS this session; Wikipedia no longer carries it. Safe to use these ten as the library's "obviously fake" examples, but do not cite SSA for it.
+
+### ITIN (IRS)
+- IRS Publication 4757 (Rev. 6-2026), p. 5 (https://www.irs.gov/pub/irs-pdf/p4757.pdf): "All valid ITINs are nine-digit numbers in the same format as the SSN (9XX-8X-XXXX), beginning with a "9" and the 4th and 5th digits ranging from 50 to 65, 70 to 88, 90 to 92, and 94 to 99."
+- Group `93` = ATIN (adoption TIN); `89` reserved (Intuit/TaxSlayer support pages, not IRS — UNVERIFIED).
+- Generator rule: `9` + 2 digits + group ∈ {50–65, 70–88, 90–92, 94–99} + 4 digits (serial `0000` not documented as invalid for ITIN; avoid it anyway).
+
+### EIN (IRS)
+- Format `NN-NNNNNNN`. Page: https://www.irs.gov/businesses/small-businesses-self-employed/how-eins-are-assigned-and-valid-ein-prefixes ("The first two digits of your EIN show the IRS campus that assigned it or if you applied online").
+- Valid prefixes (union of the table, 2026-09-17):
+  `01 02 03 04 05 06 10 11 12 13 14 15 16 20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 71 72 73 74 75 76 77 80 81 82 83 84 85 86 87 88 90 91 92 93 94 95 98 99`
+  By campus: Andover 10,12 · Atlanta 60,67 · Austin 50,53 · Brookhaven 01–06,11,13,14,16,21–23,25,34,51,52,54–59,65 · Cincinnati 30,32,35–38,61 · Fresno 15,24 · Kansas City 40,44 · Memphis 94,95 · Ogden 80,90 · Philadelphia 33,39,41–43,46,48,62–64,66,68,71–77,85–88,91–93,98,99 · Internet 20,26,27,33,39,41,42,45–47,81–88,92,93,99 · SBA 31.
+- Not valid: 00, 07, 08, 09, 17, 18, 19, 28, 29, 49, 69, 70, 78, 79, 89, 96, 97. No check digit.
+
+## 2. Finance
+
+### ABA routing number (RTN)
+- Wikipedia https://en.wikipedia.org/wiki/ABA_routing_transit_number: 9 digits; check: `(3(d1+d4+d7) + 7(d2+d5+d8) + (d3+d6+d9)) mod 10 = 0`.
+- First two digits: `00` US Government; `01–12` Federal Reserve districts (normal banks); `21–32` thrifts (historic, still valid); `61–72` non-bank payment processors/clearinghouses; `80` traveler's checks. Other prefixes unused (Wikipedia does not say so explicitly — inferred).
+- Generator rule: prefix ∈ {01–12, 21–32, 61–72}, 6 random digits, compute 9th as `(10 − (3d1+7d2+d3+3d4+7d5+d6+3d7+7d8) mod 10) mod 10`.
+- Open list of real RTNs: the Federal Reserve E-Payments Routing Directory (https://www.frbservices.org/resources/routing-number-directory/index.html) says "may not be sold, re-licensed, or otherwise used for commercial gain" → **not embeddable**. Generate synthetically.
+
+### Payment card numbers (Wikipedia "Payment card number", https://en.wikipedia.org/wiki/Payment_card_number; all Luhn-checked)
+| Network | IIN ranges | Length | CVV |
+|---|---|---|---|
+| Visa | 4 | 13, 16, 19 | 3 |
+| Mastercard | 51–55, 2221–2720 | 16 | 3 |
+| American Express | 34, 37 | 15 | 4 (front) |
+| Discover | 6011, 644–649, 65, 622126–622925 (UnionPay co-brand) | 16–19 | 3 |
+| JCB | 3528–3589 | 16–19 | 3 |
+| Diners Club International | 30 (covers 300–305, 3095), 36, 38, 39 | 14–19 | 3 |
+| Diners Club US & Canada | 55 (Mastercard co-brand) | 16 | 3 |
+| UnionPay | 62 | 16–19 | 3 |
+| Maestro | 5018, 5020, 5038, 5893, 6304, 6759, 6761, 6762, 6763 | 12–19 | 3 |
+| Maestro UK | 6759, 676770, 676774 | 12–19 | 3 |
+| Dankort 5019 · Mir 2200–2204 · RuPay 60,65,81,82,508 · Troy 65,9792 · UATP 1 (15) · Verve 506099–506198, 507865–507964, 650002–650027 (16/18/19) | | | |
+- CVV lengths: https://en.wikipedia.org/wiki/Card_security_code ("three-digit ... Visa, Mastercard, and Discover"; "American Express is a four-digit code on the front").
+- The requested "Maestro 50/56–58/6xxx" is a looser legacy rule; use Wikipedia's explicit IINs.
+- Wikipedia text is CC BY-SA 4.0 — copy the *facts* (not prose) into your table; facts are not copyrightable.
+
+### Published test cards
+- Stripe: https://docs.stripe.com/testing — Visa 4242424242424242, Mastercard 5555555555554444, Amex 378282246310005, Discover 6011111111111117, Diners 3056930009020004 (14), JCB 3566002020360505, UnionPay 6200000000000005; any future expiry, any CVC (4 for Amex).
+- PayPal: https://developer.paypal.com/tools/sandbox/card-testing/ — Visa 4005519200000004, 4012000033330026, 4012000077777777, 4012888888881881, 4217651111111119, 4500600000000061, 4772129056533503, 4915805038587737; Mastercard 2223000048400011; Amex 371449635398431, 376680816376961; Diners 36461510000039, 36461510000013; Maestro 6304000000000000, 5063516945005047; JCB 3636500000000260, 3636500000000989; CUP 6200680000000004, 6200680000000038.
+- Reuse: neither page carries an open licence (Stripe docs are under Stripe's site terms). The numbers themselves are Luhn-valid facts, widely reproduced; embedding the numbers with a "from Stripe/PayPal test docs" note is low-risk but **UNVERIFIED** as an explicit grant. Alternative: generate Luhn-valid numbers from the IIN table above — no dependency on either vendor.
+
+### BIC / SWIFT (ISO 9362)
+- https://en.wikipedia.org/wiki/ISO_9362: 4 letters bank code + 2 letters ISO 3166-1 country + 2 alphanumeric location + optional 3 alphanumeric branch (8 or 11 chars; `XXX` = primary office). Location second char `0` = test BIC, `1` = passive participant, `2` = reverse billing.
+- Generator rule: `[A-Z]{4}[A-Z]{2}[A-Z2-9][A-NP-Z0-9]([A-Z0-9]{3})?` — avoid `0`/`1` as the 8th char for a "live" BIC.
+- SWIFT's BIC Directory is proprietary (SwiftRef licence, redistribution needs a separate "SwiftRef Redistribution License": https://www.swift.com/myswift/ordering/order-products-services/swiftref-redistribution-license) → **not embeddable**.
+- Open alternative: GLEIF BIC-to-LEI mapping, https://www.gleif.org/en/lei-data/lei-mapping/download-bic-to-lei-relationship-files, files at https://mapping.gleif.org/api/v2/bic-lei/ (monthly zip; `LEI-BIC-20260828.zip` → `lei-bic-20260828T000000.csv`, columns `LEI,BIC`, 39,347 rows). Licence: "BIC/LEI Mapping Table License Agreement" (https://www.gleif.org/lei-data/lei-mapping/download-bic-to-lei-relationship-files/2017-12-21_annex-2_bic-to-lei-mapping-table-license-agreement_final.pdf) — a CC0-style grant "for any purpose whatsoever, including ... commercial" but **conditional on reproducing the notice** "SWIFT © and database rights [month year]. All rights reserved. This Mapping Table has been developed by SWIFT. Any use of the Mapping Table ... is subject to the BIC/LEI Mapping Table License Agreement ... For the latest BIC information and updates, always refer to www.swift.com/bic." Embeddable in an MIT repo with that notice in the data file; it is a list of real live BICs (bank names not included — join to GLEIF LEI data if names are wanted).
+
+### ISIN (https://en.wikipedia.org/wiki/International_Securities_Identification_Number)
+- 12 chars: 2-letter country + 9 alphanumeric NSIN + 1 check digit.
+- Check: convert letters A=10…Z=35 (ASCII − 55) to produce a digit string, then Luhn over that string (double every second digit from the right, sum digits, check makes total ≡ 0 mod 10). Pitfall: work on the *expanded* digit string; a transposed letter pair can pass.
+
+### CUSIP (https://en.wikipedia.org/wiki/CUSIP)
+- 9 chars: 6 issuer + 2 issue + 1 check. Values: digits as-is, A=10…Z=35, `*`=36, `@`=37, `#`=38.
+- Check: for i in 1..8, v = value; if i even, v ×= 2; sum += v div 10 + v mod 10; check = (10 − sum mod 10) mod 10.
+- Real CUSIPs are proprietary (CGS/FactSet); generate synthetic ones only.
+
+### IBAN (ISO 13616)
+- Validation (https://en.wikipedia.org/wiki/International_Bank_Account_Number): move first 4 chars to end, letters → 10–35, integer mod 97 must equal 1; max 34 chars; check digits 02–98.
+- SWIFT registry: https://www.swift.com/standards/data-standards/iban-international-bank-account-number, PDF release 101 https://www.swift.com/sites/default/files/files/iban-registry-v101.pdf (release 100 was Oct 2025); TXT also published (URL **UNVERIFIED** — swift.com returns 403/errors to automation this session). Terms: free of charge; no explicit licence text found (**UNVERIFIED**). Country count: 89 countries per Wikipedia (Dec 2024).
+- Embeddable per-country length tables already extracted from the registry:
+  - php-iban `registry.txt` (https://github.com/globalcitizen/php-iban, **LGPL-3.0**): pipe-separated, 121 rows (116 official + unofficial), columns `country_code|country_name|domestic_example|bban_example|bban_format_swift|bban_format_regex|bban_length|iban_example|iban_format_swift|iban_format_regex|iban_length|bban_bankid_start_offset|...|country_sepa|swift_official|...|currency_iso4217|central_bank_url|central_bank_name|membership`. LGPL data in an MIT repo is awkward; use it as a *reference* to build your own table (formats are facts) rather than copying the file.
+  - ibankit-js (https://github.com/koblas/ibankit-js, **Apache-2.0**, registry v95) — same caveat.
+  - Best route: own table `{country, length, bban_format}` derived from the SWIFT PDF (facts), cite SWIFT.
+
+### Currencies (ISO 4217)
+- ISO says use is free: https://www.iso.org/iso-4217-currency-codes.html — "ISO allows free-of-charge use of its country, currency and language codes from ISO 3166, ISO 4217 and ISO 639". Official list from SIX: https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml (+ `.xls`, `list-three` historic). Fields `CtryNm, CcyNm, Ccy, CcyNbr, CcyMnrUnts`; published 2026-01-01; 280 entity rows, 178 distinct codes. No symbols. No licence text inside the XML.
+- Alternatives:
+  - datasets/currency-codes (https://github.com/datasets/currency-codes): **PDDL** (public domain); `data/codes-all.csv` fields `Entity, Currency, AlphabeticCode, NumericCode, MinorUnit, WithdrawalDate`; built from SIX list one + three; no symbols. Embeddable.
+  - npm `currency-codes` v2.2.0 (https://github.com/freeall/currency-codes): **MIT**; fields `code, number, digits, currency, countries[]`; generated from the SIX XML; no symbols. Embeddable.
+  - umpirsky/currency-list (https://github.com/umpirsky/currency-list): **MIT**; code → localised name only (311 entries in `data/en_US/currency.json`, includes historic); no symbols, no numeric codes, no decimals.
+  - Debian iso-codes (https://salsa.debian.org/iso-codes-team/iso-codes): **LGPL-2.1+**; `data/iso_4217.json` has `alpha_3, name, numeric` only (179 entries); no symbols/minor units. LGPL → avoid embedding.
+  - Symbols: none of the above carry them. Unicode CLDR (`common/main/en.xml` currency symbols, Unicode License, MIT-compatible with notice) is the usual source — **UNVERIFIED this session**.
+
+### Crypto addresses
+- Bitcoin (https://en.bitcoin.it/wiki/List_of_address_prefixes): P2PKH version 0x00 → leading `1`; P2SH 0x05 → leading `3`; Base58Check 25–34 chars (alphabet excludes `0OIl`; last 4 bytes = first 4 of double-SHA256 of version+payload). Testnet `m`/`n`, `2`, `tb1`.
+- Bech32 (BIP-173, https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki, BSD-2-Clause): HRP `bc`, separator `1`, charset `qpzry9x8gf2tvdw0s3jn54khce6mua7l`, all-lowercase (or all-uppercase), max 90 chars; P2WPKH `bc1q…` = 42 chars, P2WSH `bc1q…` = 62 chars.
+- Bech32m (BIP-350, https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): witness v1+ (P2TR `bc1p…`, 62 chars), checksum constant `0x2bc830a3` instead of 1.
+- Ethereum EIP-55 (https://eips.ethereum.org/EIPS/eip-55, CC0): `0x` + 40 hex; keccak256 of the lowercase hex (no 0x, as ASCII); for each hex letter, uppercase iff the corresponding hash nibble ≥ 8 (bit 4·i set). Digits unchanged.
+
+## 3. Company
+
+### NAICS 2022 (US Census)
+- Page https://www.census.gov/naics/ ; files: 6-digit list https://www.census.gov/naics/2022NAICS/6-digit_2022_Codes.xlsx (82 kB; **1,012** six-digit codes, 111110…928120, verified by parsing), 2–6 digit https://www.census.gov/naics/2022NAICS/2-6%20digit_2022_Codes.xlsx, structure https://www.census.gov/naics/2022NAICS/2022_NAICS_Structure.xlsx, descriptions https://www.census.gov/naics/2022NAICS/2022_NAICS_Descriptions.xlsx, manual PDF https://www.census.gov/naics/reference_files_tools/2022_NAICS_Manual.pdf.
+- Format: xlsx, column A code (numeric), column B title (trailing spaces in some titles — trim).
+- Licence: no statement on the page or in the manual front matter; US federal government work → public domain in the US (17 U.S.C. §105). Embeddable; cite Census/OMB.
+
+### SEC EDGAR company names
+- `https://www.sec.gov/files/company_tickers.json`: object keyed `"0".."N"` of `{cik_str, ticker, title}`; **10,422** entries (2026-09-17). `company_tickers_exchange.json`: `{"fields":["cik","name","ticker","exchange"],"data":[...]}`.
+- `https://www.sec.gov/Archives/edgar/cik-lookup-data.txt`: 40 MB, **1,059,372** lines, `NAME:CIK:` (`COMPANY NAME:0001234567:`), all filers incl. individuals — filter to the ticker list for company names.
+- Access: declare `User-Agent: Company Name email@domain`, ≤ 10 req/s (https://www.sec.gov/os/webmaster-faq). Licence: "All Government-created content on sec.gov and EDGAR public filing content are free to access and reuse" (same FAQ). Embeddable; the ticker file (~800 kB) is the practical one.
+
+### DUNS
+- https://en.wikipedia.org/wiki/Data_Universal_Numbering_System: 9 digits, no significance; shown `NN-NNN-NNNN` or plain. Had a mod-10 check digit until ~Dec 2006; **dropped** since (expanded the pool by 800 M). So: any 9 digits are structurally valid today; no validation rule to enforce. Optional DUNS+4 suffix (4 alphanumerics) is user-assigned.
+
+### Legal entity suffixes per country
+- GLEIF ISO 20275 Entity Legal Forms code list v1.6 (2026-02-19): page https://www.gleif.org/en/lei-data/code-lists/iso-20275-entity-legal-forms-code-list ; CSV https://www.gleif.org/lei-data/code-lists/iso-20275-entity-legal-forms-code-list/2026-02-19-elf-code-list-v1.6.csv (758 kB; xlsx alongside).
+- Columns: `ELF Code, Country of formation, Country Code (ISO 3166-1), Jurisdiction of formation, Country sub-division code (ISO 3166-2), Entity Legal Form name Local name, Language, Language Code (ISO 639-1), Entity Legal Form name Transliterated name, Abbreviations Local language, Abbreviations transliterated, Date created, ELF Status ACTV/INAC, Modification, Modification date, Reason`.
+- Counts: 4,002 rows; 130 countries; 3,790 ACTV / 212 INAC; 1,518 rows have a local abbreviation (e.g. US `N.A.`, `FSA`; SE `AB`… multiple abbreviations separated by `;`). US has 737 rows (per-state forms). Abbreviations are the field you want for `Inc`, `LLC`, `GmbH`, `AB`, `SAS`, `Pty Ltd`; many rows have none, so filter on non-empty abbreviation and status ACTV.
+- Licence: GLEIF Open Data page (https://www.gleif.org/en/about/open-data): "The data on GLEIF's website is provided under a Creative Commons (CC0) license"; LEI Data Terms of Use: "provided under the CC0 licence, see CC0 1.0 Universal". The ELF page itself names no licence — that CC0 explicitly covers this code list is **UNVERIFIED**, but GLEIF treats all its published data this way. Embeddable.
+
+## Quick embed verdict
+| Item | Embed? | Licence / attribution |
+|---|---|---|
+| SSA baby names | yes | CC0 (data.gov) |
+| Census 2010 surnames | yes | US gov PD; cite Census |
+| EIN prefixes, ITIN/SSN rules | yes (rules) | facts from IRS/SSA |
+| NAICS 2022 | yes | US gov PD |
+| SEC company_tickers.json | yes | free to reuse (SEC) |
+| GLEIF ELF code list | yes | CC0 (GLEIF) |
+| GLEIF BIC-LEI (real BICs) | yes, with mandatory SWIFT notice | custom permissive licence |
+| ISO 4217 (SIX XML / datasets PDDL / npm MIT) | yes | free use (ISO), PDDL, MIT |
+| Fed routing directory, SwiftRef BIC directory, CUSIP lists | no | restricted/proprietary — generate synthetically |
+| php-iban / ibankit registries | copy facts only | LGPL-3.0 / Apache-2.0 |
+| Stripe / PayPal test cards | numbers only, note source | no explicit grant (UNVERIFIED) |
+# Part 2 — Internet, vehicle, airline
+
+Verdict key: **EMBED** = safe to vendor into an MIT repo; **EMBED+ATTR** = ok with a notice; **AVOID** = licence incompatible or unclear.
+
+## 4. Internet
+
+### IANA registries (all of them)
+- Licence: IANA/IETF statement at https://www.iana.org/help/licensing-terms — "IANA and IETF intend that the Protocol Registries may be freely used by any party for any purpose", data dedicated under **CC0 1.0**. No attribution required. **EMBED**.
+- All registries below share this licence; ship one `SOURCES`/NOTICE line pointing at that URL.
+
+| Registry | CSV URL | Rows (2026-09-16) | Notes |
+|---|---|---|---|
+| Media types | `https://www.iana.org/assignments/media-types/<type>.csv`, type ∈ application, audio, font, haptics, image, message, model, multipart, text, video (also `example`) | application 1800, audio 165, font 6, haptics 3, image 88, message 27, model 42, multipart 17, text 105, video 97 (header excluded) ≈ **2350** | Fields: `Name,Template,Reference`. Template is the full `type/subtype`; some rows are DEPRECATED/OBSOLETED in Name (filter). Format: RFC 6838 §4.2 — `type "/" subtype`, restricted chars, max 127 chars each, case-insensitive. |
+| HTTP status codes | https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv | 75 rows, **64 assigned** (306 and 418 are "(Unused)"; 104 is TEMPORARY) → 62 usable | Fields: `Value,Description,Reference`. Value is 3 digits 1xx–5xx. |
+| IPv4 special-purpose | https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry-1.csv | 26 blocks | Fields: `Address Block,Name,RFC,Allocation Date,Termination Date,Source,Destination,Forwardable,Globally Reachable,Reserved-by-Protocol`. |
+| IPv6 special-purpose | https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry-1.csv | 27 blocks | Same fields. Contains `2001:db8::/32` and `3fff::/20`. |
+| Service names / ports | https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv | 14,534 rows; 11,731 with both name and a single port; **6,102 distinct named ports**; 6,008 tcp-named; 707 tcp names < 1024 | Fields: `Service Name,Port Number,Transport Protocol,Description,Assignee,Contact,Registration Date,Modification Date,Reference,Service Code,Unauthorized Use Reported,Assignment Notes`. Port Number may be empty, single, or a range `a-b`. Well-known = 0–1023, registered 1024–49151, dynamic 49152–65535 (RFC 6335). ~1.1 MB; embed only the named-port subset. |
+| TLDs | https://data.iana.org/TLD/tlds-alpha-by-domain.txt | **1,401** TLDs (first line is a `# Version …` comment) | Upper-case ASCII; IDNs as `XN--…` punycode. Same IANA CC0 terms (UNVERIFIED that data.iana.org is explicitly covered by the licensing page; it is the same registry data). Lower-case on use. |
+
+### MIME extension mapping: jshttp/mime-db
+- https://github.com/jshttp/mime-db — **MIT** (GitHub API spdx MIT; npm `license: MIT`), version 1.54.0 on npm.
+- `db.json` (CDN: https://cdn.jsdelivr.net/npm/mime-db/db.json): object keyed by lower-case type → `{source, extensions[], compressible, charset}`.
+- Counts (1.54.0): **2,522 types**, **1,015 with extensions**; source = iana 2,136, apache 275, nginx 13, custom 98.
+- Sources: IANA registry (CC0), Apache httpd `mime.types` (Apache-2.0), nginx `mime.types` (BSD-2). **EMBED+ATTR** (keep the MIT copyright notice). Data updates are not semver-breaking per README.
+
+### IPv4 / IPv6 generation rules (RFCs, no data file needed)
+- RFC 5737 documentation: `192.0.2.0/24` (TEST-NET-1), `198.51.100.0/24` (TEST-NET-2), `203.0.113.0/24` (TEST-NET-3). https://www.rfc-editor.org/rfc/rfc5737.html
+- RFC 1918 private: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`. Also useful: `100.64.0.0/10` CGNAT (RFC 6598), `198.18.0.0/15` benchmarking (RFC 2544), `169.254.0.0/16` link-local.
+- IPv6 documentation: `2001:db8::/32` (RFC 3849) and `3fff::/20` (RFC 9637, "expands on the existing 2001:db8::/32 … with the reservation of an additional, larger prefix"). ULA `fc00::/7` (use `fd00::/8`), link-local `fe80::/10`.
+- Validation: generate inside those blocks; avoid network/broadcast (`.0`/`.255`) if the consumer expects host addresses. Format IPv6 per RFC 5952 (lower-case hex, `::` once, longest zero run).
+
+### MAC OUI (IEEE MA-L)
+- CSV: https://standards-oui.ieee.org/oui/oui.csv (3.8 MB, **40,161 rows**, fields `Registry,Assignment,Organization Name,Organization Address`; Assignment is 6 hex chars). Server returns HTTP 418 to non-browser user agents; fetch with a browser UA. Also MA-M `oui28.csv`, MA-S `oui36.csv`, CID `cid.csv`.
+- Licence: no licence on the IEEE pages themselves (site footer "© IEEE – All rights reserved" is generic). Debian's `ieee-data` copyright file (https://metadata.ftp-master.debian.org/changelogs/main/i/ieee-data/unstable_copyright) records IEEE's 2014 statement: *"IEEE does not assert any copyright in the OUI Public Listing or attempt to restrict distribution of the listing in any way. The IEEE Registration Authority does, however, strongly encourage those who use the list to obtain it directly from IEEE…"* — same text in the FSF directory. Debian ships it in `main` on that basis. **EMBED+ATTR** (cite the statement); the org-address column is not needed — keep `Assignment,Organization Name` only (~1 MB → few hundred KB). UNVERIFIED: no current IEEE page restates it (regauth FAQ and IPR pages do not mention it).
+- Alternative needing no data: locally administered unicast MAC — first octet's bit 1 (U/L) = 1, bit 0 (I/G) = 0 → second hex digit ∈ {2, 6, A, E}, e.g. `x2:xx:xx:xx:xx:xx`. Guaranteed never to collide with a vendor OUI. Notations: `01:23:45:67:89:ab`, `01-23-45-67-89-AB`, Cisco `0123.4567.89ab`.
+
+### User-agent strings
+| Source | Licence | Verdict |
+|---|---|---|
+| https://www.useragents.me/ | No licence stated anywhere on the page ("© 2022–2025 Useragents.me"); `/api` returns 404 now; JSON blobs embedded in page, weekly updates | **AVOID** (unlicensed = all rights reserved) |
+| https://explore.whatismybrowser.com/useragents/explore/ + API legal https://developers.whatismybrowser.com/api/about/legal/ | Proprietary: "You may NOT share the unique database download URL, or the downloaded file itself … EXCEPT for making it publicly available on the internet" | **AVOID** |
+| npm `user-agents` (intoli) https://github.com/intoli/user-agents | **BSD-2-Clause**, v2.1.185, daily-rebuilt dataset from Intoli traffic, weighted by frequency; fields `userAgent, platform, vendor, appName, deviceCategory, screenWidth/Height, viewportWidth/Height, connection{…}, oscpu, cpuClass, pluginsLength` | **EMBED+ATTR** (keep BSD notice). Count not stated on the page (UNVERIFIED; dataset is a few thousand rows). |
+| npm `top-user-agents` (microlinkhq) https://github.com/microlinkhq/top-user-agents | **MIT** (GitHub API). Top 100 UA strings from microlink.io traffic, weekly; JSON: https://cdn.jsdelivr.net/gh/microlinkhq/top-user-agents@master/src/index.json (+ desktop.json, mobile.json) | **EMBED+ATTR** — smallest clean option |
+- None of the above is CC0. Alternative: generate UAs from templates (Chrome/Firefox/Safari/Edge strings are fixed-form; only version numbers vary), no data needed.
+
+### Programming languages: GitHub Linguist
+- https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml — **MIT** (repo LICENSE, "Copyright (c) 2017 GitHub, Inc."). **EMBED+ATTR**.
+- **835** top-level entries; **562** with `type: programming` (others: data, markup, prose). Per-entry fields: `type, color, extensions, aliases, tm_scope, ace_mode, language_id, group, interpreters, …`.
+
+### Operating systems
+- No open dataset found beyond awesome-lists. OS names/versions are facts — hand-author a small table (Windows 10/11, Windows Server, macOS versions+names, Ubuntu/Debian/Fedora/Arch/Alpine, iOS, Android, FreeBSD…). `user-agents` (BSD-2) carries `platform`/`oscpu` values if a data-driven list is wanted.
+
+### Database engines / collations
+- Engine names: hand-author (facts).
+- MySQL manual (https://dev.mysql.com/doc/refman/8.4/en/preface.html): Oracle terms — "you may not … copy, reproduce … distribute … any part" except distributing the docs with the software. **AVOID copying tables from MySQL docs.** Instead dump `SHOW COLLATION` from a `mysql:8.4.x` container (server output is factual, not the manual) — ~286 collations / 41 charsets in 8.x (UNVERIFIED count). PostgreSQL docs are under the PostgreSQL licence (permissive, keep notice) — `pg_collation` dump or docs both fine. MariaDB KB is CC BY-SA — avoid.
+
+## 5. Vehicle
+
+### NHTSA vPIC
+- API base `https://vpic.nhtsa.dot.gov/api/vehicles/`, `?format=json|xml|csv`. Endpoints verified: `GetAllMakes` (**12,363 makes**, fields `Make_ID,Make_Name`; 612 KB JSON, mostly small trailer/custom builders — filter with `GetMakesForVehicleType/car` → 195 makes, `/truck` → 207), `GetModelsForMake/honda` (362 models, `Make_ID,Make_Name,Model_ID,Model_Name`), `DecodeWMI/1HG`, `GetWMIsForManufacturer/honda` (45 WMIs, fields `WMI,Name,Country,VehicleType,Id,…`), `GetVehicleVariableList`, `GetVehicleVariableValuesList/<display name>` (must be the display name, URL-encoded, e.g. `Fuel%20Type%20-%20Primary`; the camel-case name returns 0 rows).
+- Value lists (counts): **Fuel Type - Primary 14**, **Body Class 71**, **Transmission Style 12**, **Drive Type 23**, **Vehicle Type 9**. Fields `Id,Name`.
+- Bulk: https://vpic.nhtsa.dot.gov/downloads/ — `vPICList_lite_2026_08.bak.zip` (MS SQL Server 2019+ backup, 190 MB, updated 2026-08-14), also `.plain.zip` / `.custom.zip`. Page says standalone DB "limited to VIN decoding"; makes/models/variables still via API. Contains `Wmi`, `Make`, `Model`, `WMIYearValidChars` tables (UNVERIFIED table names).
+- Licence: FAQ https://vpic.nhtsa.dot.gov/api/home/index/faq — "No, NHTSA is a government agency and the services provided on the API are free for use by the public as an offering as a part of our Open Data initiatives"; no rate-limit number stated, batch jobs asked to run at night. Work of the US federal government → public domain under 17 USC §105; data.gov entry lists licence as "unknown". **EMBED** (note source; no attribution legally required).
+
+### VIN (ISO 3779 / 49 CFR 565, North America)
+- 17 chars from `0-9 A-H J-N P R-Z` (**I, O, Q never**). Positions: 1–3 WMI, 4–8 VDS, 9 check digit, 10 model year, 11 plant, 12–17 serial (last 4 must be digits for US, positions 12–14 alphanumeric — 49 CFR 565.15).
+- Check digit: transliterate `A=1 B=2 C=3 D=4 E=5 F=6 G=7 H=8 J=1 K=2 L=3 M=4 N=5 P=7 R=9 S=2 T=3 U=4 V=5 W=6 X=7 Y=8 Z=9`, digits as-is; weights `8 7 6 5 4 3 2 10 0 9 8 7 6 5 4 3 2`; sum mod 11 → digit, remainder 10 → `X`.
+- Model year (pos 10): excludes `I O Q U Z 0`; `A=2010 … H=2017 J=2018 K=2019 L=2020 M=2021 N=2022 P=2023 R=2024 S=2025 T=2026 V=2027 W=2028 X=2029 Y=2030`, `1=2031…9=2039` (30-year cycle; `1–9` also = 2001–2009). Source: https://en.wikipedia.org/wiki/Vehicle_identification_number (rules are facts; do not copy prose).
+
+### WMI list
+- SAE J1044 / SAE WMI database is paid (https://www.sae.org/standards/j1044_202501-world-manufacturer-identifier) — **AVOID**.
+- Wikipedia "List of WMIs" — CC BY-SA — **AVOID**.
+- Use vPIC: `GetWMIsForManufacturer/<name or id>` per manufacturer, or the `Wmi` table in the standalone `.bak`. Public domain. Country prefix rule (first char): `1,4,5` USA, `2` Canada, `3` Mexico, `J` Japan, `K` Korea, `S` UK, `W` Germany, `Y` Sweden/Finland, `Z` Italy, `L` China (ISO 3780 facts).
+
+### US licence plate formats
+| Source | Licence | Verdict |
+|---|---|---|
+| openalpr `runtime_data/postprocess/us.patterns` (https://github.com/openalpr/openalpr) | **AGPL-3.0** | **AVOID** (291 lines, `@`=letter `#`=digit — useful only as a reference to check hand-authored patterns) |
+| Wikipedia "United States license plate designs and serial formats" | CC BY-SA | **AVOID copying**; formats themselves are facts |
+| jonnii/platekit | MIT, but SVG rendering, no serial patterns | not useful |
+- No MIT/CC0 per-state pattern list found. Recommendation: hand-author a per-state table (facts); current standard passenger formats as listed by Wikipedia (verify against DMV pages before use): AL `0AXXXXX`/`00AXXXX`, AK `ABC 123`, AZ `XXX 1XX`, AR `ABC 12D`, CA `1ABC123` (digit-3 letters-3 digits), CO `ABC-D12`, CT `AB·12345`, DE `123456`, DC `AB-1234`, FL `ABC D12`, GA `ABC1234`, HI `ABC 123`, ID `A 1234U`, IL `AB 12345`, IN `123ABC`, IA `ABC 123`, KS `1234ABC`, KY `ABC123`, LA `123 ABC`, ME `123·ABC`, MD `1AB2345`, MA `1ABC 23`, MI `ABC 1234`, MN `ABC-123`, MS `ABC 123`, MO `AB1 C2D`, MT `0-AB1234`, NE `ABC 123`, NV `123·A45`, NH `123 4567`, NJ `D12-ABC`, NM `123-ABC`, NY `ABC-1234`, NC `ABC-1234`, ND `123 ABC`, OH `ABC 1234`, OK `ABC-123`, OR `123 ABC`, PA `ABC1234`, RI `1AB 234`, SC `123ABC`, SD `0A1 234`, TN `ABC 1234`, TX `ABC-1234`, UT `A12 3BC`, VT `ABC 123`, VA `ABC-1234`, WA `ABC1234`, WV `X1A 2345`, WI `ABC-1234`, WY `1A-123A`. Most states omit I/O/Q from serials (state-specific, UNVERIFIED per state).
+
+## 6. Airline
+
+### OurAirports
+- https://ourairports.com/data/ — "All data is released to the Public Domain"; mirror repo https://github.com/davidmegginson/ourairports-data is **Unlicense** (GitHub API). Nightly updates. **EMBED**.
+- Raw URLs: `https://davidmegginson.github.io/ourairports-data/{airports,airport-frequencies,runways,navaids,countries,regions}.csv`.
+
+| File | Rows | Fields |
+|---|---|---|
+| airports.csv (12.7 MB) | **86,089**; by type: small_airport 42,734, heliport 23,216, closed 13,524, medium 4,106, seaplane_base 1,273, large 1,174, balloonport 62 | `id,ident,type,name,latitude_deg,longitude_deg,elevation_ft,continent,iso_country,iso_region,municipality,scheduled_service,icao_code,iata_code,gps_code,local_code,home_link,wikipedia_link,keywords` |
+| — with `iata_code` | **9,055** (large 1,171, medium 3,398, small 4,233, seaplane 153, heliport 100); `scheduled_service=yes` 4,335; US with IATA 2,036 (873 large/medium) | Suggested subset: `iata_code != '' AND type IN (large_airport, medium_airport)` → 4,569 rows |
+| countries.csv | **249** | `id,code,name,continent,wikipedia_link,keywords` |
+| regions.csv | **3,987** | `id,code,local_code,name,continent,iso_country,wikipedia_link,keywords` (code = ISO 3166-2, e.g. `US-KS`) |
+- No airline file. Derived: datasets/airport-codes on datahub (PDDL) — no need, use upstream.
+
+### OpenFlights
+- https://openflights.org/data.php — airports/airlines/routes/planes under **ODbL 1.0** (+ DbCL); airline/plane data partly from Wikipedia (GFDL/CC BY-SA). 5,888 airlines (2012). **AVOID** (share-alike). npm `airline-codes` (npow, ISC) is a straight OpenFlights sync → inherits ODbL → **AVOID**.
+
+### Airline IATA/ICAO codes — open options
+| Source | Licence | Count | Verdict |
+|---|---|---|---|
+| Wikidata SPARQL (https://query.wikidata.org/sparql) | **CC0** (https://www.wikidata.org/wiki/Wikidata:Licensing) | **2,928** items with IATA code (P229), **3,644** with ICAO code (P230) | **EMBED**. Query: `SELECT ?item ?itemLabel ?iata ?icao ?callsign WHERE { ?item wdt:P229 ?iata . OPTIONAL { ?item wdt:P230 ?icao } OPTIONAL { ?item wdt:P432 ?callsign } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }`. Includes defunct airlines — filter `MINUS { ?item wdt:P576 ?dissolved }` and/or P31 = Q46970 (airline). |
+| OpenTravelData `optd_airlines.csv` (https://github.com/opentraveldata/opentraveldata) | **CC BY 4.0** | 1,620 rows; `^`-separated; fields `pk^env_id^validity_from^validity_to^3char_code^2char_code^num_code^name^name2^alliance_code^alliance_status^type^wiki_link^flt_freq^alt_names^bases^key^version^parent_pk_list^successor_pk_list` | **EMBED+ATTR** (attribution notice required) |
+- Format rules: IATA designator = 2 alphanumeric chars (`[A-Z0-9]{2}`, at least one letter in practice; a third optional char exists in the standard but is unused); "controlled duplicates" share a code between non-overlapping regional carriers. ICAO designator = 3 letters `[A-Z]{3}`, unique. Accounting/prefix code = 3 digits (ticket number prefix, e.g. `016` United).
+
+### Aircraft types
+| Source | Licence | Verdict |
+|---|---|---|
+| ICAO Doc 8643 (https://www.icao.int/operational-safety/doc-8643-aircraft-type-designators) | ICAO copyright notice: "None of the materials … may be used, reproduced or transmitted … without permission in writing from ICAO"; API Data Service is paid (25 free trial calls) | **AVOID** — including GitHub mirrors such as ColtJD45/icao-aircraft-designator-list (MIT-labelled, 7,388 rows, fields `manufacturer,model,type_designator,description,engine_type,engine_count,wtc`) because the underlying data is scraped ICAO content (UNVERIFIED provenance) |
+| OpenSky aircraft database (https://opensky-network.org/data/aircraft; CSV https://s3.opensky-network.org/data-samples/metadata/aircraftDatabase.csv) | "The aircraft database is unlicensed and does not fall under our terms of use … offered as is"; built from registries, openflights.org and ICAO Doc 8643; citation requested; page says it is no longer up to date | **AVOID** ("unlicensed" ≠ open; upstream includes ODbL/ICAO). Fields for reference: `icao24,registration,manufacturericao,manufacturername,model,typecode,serialnumber,linenumber,icaoaircrafttype,operator,operatorcallsign,operatoricao,operatoriata,owner,…,engines,…,categoryDescription` |
+| Wikidata | **CC0** | **550** items with ICAO aircraft type designator (P8305); also IATA aircraft code P9040 (UNVERIFIED property id), manufacturer P176 | **EMBED**. Query: `SELECT ?item ?itemLabel ?icao ?mfrLabel WHERE { ?item wdt:P8305 ?icao . OPTIONAL { ?item wdt:P176 ?mfr } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }` |
+- Format: ICAO type designator 2–4 alphanumerics (`[A-Z0-9]{2,4}`, e.g. `B738`, `A20N`, `E190`); IATA aircraft code 3 alphanumerics (`738`, `32N`).
+
+### Airport / flight / seat format rules
+- IATA airport code `[A-Z]{3}`; ICAO location indicator `[A-Z0-9]{4}` (US `K…`, Canada `C…`, UK `EG…`, Sweden `ES…`). Use OurAirports rows so codes are real.
+- Flight number: 2-char IATA designator + 1–4 digits, no leading zeros in display (`QF9`, `AA1234`); systems cap at 0001–9999. Conventions (not rules): 1–999 mainline, 3000–5999 regional affiliates, ≥6000 codeshares, 8xxx charters, ≥9000 ferry/positioning. ICAO form: 3-letter designator + 1–4 alphanumerics (`AFR1`). Source: https://en.wikipedia.org/wiki/Flight_number.
+- Seat: `row + letter`. Rows typically 1–~60 (A380 up to ~90); many airlines skip 13 (and 14/17 on some carriers). Letters `A–K` **skipping I** (looks like 1/l); narrow-body 3-3 = `ABC DEF` (A/F windows, C/D aisles); wide-body 3-4-3 = `ABC DEFG HJK`, 2-4-2 = `AB DEFG JK`. Regex: `^[1-9][0-9]?[A-HJK]$`.
+# Part 3 — Science/nature, Text, Dates: open data sources for an MIT fake-data library
+
+Researched 2026-09-17. "OK to embed" = redistributable inside an MIT repo with the stated notice. Anything not fetched first-hand is marked **UNVERIFIED**.
+
+## 7. Science / nature
+
+### Periodic table
+
+| Source | Licence | Embed in MIT repo | Fields | Rows | Format |
+|---|---|---|---|---|---|
+| PubChem periodic table — CSV https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/CSV, JSON https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/JSON (page: https://pubchem.ncbi.nlm.nih.gov/periodic-table/) | US-government work, public domain. NLM policy (https://www.ncbi.nlm.nih.gov/home/about/policies/): "Information that is created by or for the US government on this site is within the public domain … may be freely distributed and copied. However, it is requested that in any subsequent use of this work, NLM be given appropriate acknowledgment." | Yes; add an acknowledgment line ("Element data: PubChem/NLM"). | `AtomicNumber, Symbol, Name, AtomicMass, CPKHexColor, ElectronConfiguration, Electronegativity, AtomicRadius, IonizationEnergy, ElectronAffinity, OxidationStates, StandardState, MeltingPoint, BoilingPoint, Density, GroupBlock, YearDiscovered` (17) | 118 (Z=1..118, verified) | CSV; JSON is `{Table:{Columns:{Column:[...]},Row:[{Cell:[...]}]}}` |
+| NIST periodic table https://www.nist.gov/pml/periodic-table-elements | Public domain (17 USC 105); NIST asks for citation (https://www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications) | Yes, but | PDF only (no CSV/JSON) — not usable as a data source | 118 | PDF |
+| Bowserinator/Periodic-Table-JSON https://github.com/Bowserinator/Periodic-Table-JSON | **CC BY-SA 3.0** (LICENSE.md, verified; data derived from Wikipedia) | **No** — ShareAlike is incompatible with MIT redistribution | 33 keys incl. `name, symbol, number, atomic_mass, category, phase, period, group, block, shells, electron_configuration, summary, cpk-hex, …` | **119** (includes hypothetical Ununennium) | JSON, CSV |
+| faker-js `src/locales/en/science/chemical_element.ts` https://github.com/faker-js/faker (branch `next`) | MIT (LICENSE verified) | Yes, keep MIT notice | `{symbol, name, atomicNumber}` | 118 | TS |
+
+Validation rules: `AtomicNumber` 1..118 unique; `Symbol` 1–2 letters, first uppercase, unique; `Name` unique. PubChem leaves numeric fields empty for Z≥104 where unknown; `ElectronConfiguration` for Og is `"[Rn]7s2 7p6 5f14 6d10 (predicted)"` and `StandardState` is `"Expected to be a Gas"` — strip "(predicted)"/"Expected to be a" if you want an enum. `AtomicMass` for unstable elements is the mass number of the most stable isotope as a bare decimal (e.g. `295.216`), no brackets. `CPKHexColor` is 6 hex digits without `#`, empty for some.
+
+### SI units
+
+- Source: BIPM SI Brochure, 9th ed. (2019; current revision v4.01, 2026) https://www.bipm.org/en/publications/si-brochure ; PDF https://www.bipm.org/documents/20126/41483022/SI-Brochure-9-EN.pdf
+- Licence: **CC BY 4.0** — stated on the brochure page and on BIPM's copyright page (https://www.bipm.org/en/copyright): content may be adapted, copied, used in commercial products with "appropriate acknowledgement of the BIPM and its source"; BIPM name/logo must not imply endorsement. Verified via the HTML pages; the PDF's own front-matter licence line is **UNVERIFIED** (could not text-extract the PDF here).
+- Embed: Yes — the unit *names/symbols* are facts (not copyrightable); attribute "SI Brochure, BIPM, CC BY 4.0" anyway.
+- Data (cross-checked against https://en.wikipedia.org/wiki/International_System_of_Units):
+  - 7 base units: second s (time), metre m (length), kilogram kg (mass), ampere A (electric current), kelvin K (thermodynamic temperature), mole mol (amount of substance), candela cd (luminous intensity).
+  - 22 coherent derived units with special names: radian rad, steradian sr, hertz Hz, newton N, pascal Pa, joule J, watt W, coulomb C, volt V, farad F, ohm Ω, siemens S, weber Wb, tesla T, henry H, degree Celsius °C, lumen lm, lux lx, becquerel Bq, gray Gy, sievert Sv, katal kat.
+  - 24 prefixes: quetta Q 10^30, ronna R 10^27, yotta Y 10^24, zetta Z 10^21, exa E 10^18, peta P 10^15, tera T 10^12, giga G 10^9, mega M 10^6, kilo k 10^3, hecto h 10^2, deca da 10^1, deci d 10^-1, centi c 10^-2, milli m 10^-3, micro µ 10^-6, nano n 10^-9, pico p 10^-12, femto f 10^-15, atto a 10^-18, zepto z 10^-21, yocto y 10^-24, ronto r 10^-27, quecto q 10^-30.
+- Format rules for valid output: unit symbols are case-sensitive (`s` vs `S`, `m` vs `M`); prefix and unit symbol are joined without a space (`kJ`, `µs`); a space separates number and unit (`12 kg`), except the degree/minute/second of plane angle; `kg` already carries a prefix — never `µkg`; micro is Greek mu U+03BC (U+00B5 micro sign is the legacy compatibility character); ohm is U+03A9 (U+2126 OHM SIGN is deprecated); `°C` not `° C`.
+- MIT alternative: faker-js `en/science/unit.ts` — 29 `{name, symbol}` objects, MIT.
+
+### Animals by class
+
+- **Wikidata** (https://www.wikidata.org/wiki/Wikidata:Licensing): all structured data **CC0** — no attribution required. Embed: Yes.
+  - Feasibility, live SPARQL (https://query.wikidata.org/sparql), pattern `?t wdt:P31 wd:Q16521; wdt:P105 wd:Q7432; wdt:P171* wd:<class>; wdt:P1843 ?cn FILTER(LANG(?cn)='en')` (taxon, rank species, parent-taxon chain, English common name):
+    - Mammalia Q7377: **5,759** species with ≥1 English common name.
+    - Aves Q5113: **12,432**.
+    - Reptilia Q10811: **16,369** — suspiciously large; Wikidata's parent chain likely pulls birds/Sauropsida in. Use Squamata Q122422 + Testudines Q223044 + Crocodilia Q1387 (or check the chain) instead.
+    - Insecta Q1390 and Plantae Q756: **timed out (502/504) every attempt** — trees too big for the 60 s public endpoint. Options: paginate by order (`P171` one level at a time), use the weekly JSON dump, or take GBIF instead.
+  - Sample rows (mammals): `Notoryctes typhlops` → "Central Desert Marsupial Mole", "Itjaritjari", "Marsupial Mole", "Southern marsupial mole", "Southern Marsupial Mole" — i.e. **several P1843 values per taxon, inconsistent casing, and indigenous-language names tagged `en`**. Validation: pick one name per taxon (prefer the one matching the item's English `rdfs:label`), normalise case, drop names that equal the scientific name.
+- **GBIF Backbone Taxonomy** https://www.gbif.org/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c — licence **CC BY 4.0** (verified via https://api.gbif.org/v1/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c, `license: http://creativecommons.org/licenses/by/4.0/legalcode`); required citation text: "GBIF Secretariat (2023). GBIF Backbone Taxonomy. Checklist dataset https://doi.org/10.15468/39omei". Embed: Yes with that citation. Vernacular names come via `/v1/species/{key}/vernacularNames`, each row from a *different* source checklist with its own licence (some CC BY-NC) — record and check per row. GBIF's site terms page (https://www.gbif.org/terms) was unreachable (timeout) — **UNVERIFIED** beyond the API field.
+- **faker-js** `src/locales/en/animal/*` (MIT; counts = quoted string lines on branch `next`): bird 821, snake 533, dog 493, cow 467, horse 342, rodent 161, insect 129, fish 95, cat 55, cetacean 52, rabbit 49, type 44 (generic: "bat, bear, bee, …, zebra"), pet_name 42, crocodilia 24, bear 8, lion 7. These are breeds/species mixed with common names, no class field; fine as MIT drop-ins. (A WebFetch summariser claimed 1,036 dogs; the regex count is 493 — treat the exact count as UNVERIFIED.)
+
+### Plants
+
+- Same Wikidata approach (CC0) — but the Plantae query timed out; paginate by order or family (`?t wdt:P171 ?family . ?family wdt:P105 wd:Q35409`) or use the dump.
+- GBIF Plantae (kingdomKey 6) vernacular names, CC BY 4.0, same per-row-licence caveat.
+- faker-js has **no** plant list (en locale dirs: airline, animal, app, book, color, commerce, company, database, date, finance, food, hacker, internet, location, lorem, medical, music, person, phone_number, science, team, vehicle, word).
+- USDA PLANTS database (public domain, has common names) — **UNVERIFIED**, not fetched.
+
+### Planets / moons
+
+- NASA NSSDCA Planetary Fact Sheet https://nssdc.gsfc.nasa.gov/planetary/factsheet/ — 10 columns (Mercury, Venus, Earth, Moon, Mars, Jupiter, Saturn, Uranus, Neptune, Pluto) × 20 rows (mass 10^24 kg, diameter km, density, gravity, escape velocity, rotation period, day length, distance from Sun, perihelion, aphelion, orbital period, orbital velocity, inclination, eccentricity, obliquity, mean temp °C, surface pressure, number of moons, rings Y/N, magnetic field Y/N). HTML table only. NASA content "generally not subject to copyright in the United States" (https://www.nasa.gov/nasa-brand-center/images-and-media/); NASA asks to be acknowledged; the NASA insignia must not be used. Embed: Yes with "Source: NASA NSSDCA".
+- Wikidata (CC0): planets = `?p wdt:P31/wdt:P279* wd:Q634; wdt:P397 wd:Q525` → Mercury Q308, Venus Q313, Earth Q2, Mars Q111, Jupiter Q319, Saturn Q193, Uranus Q324, Neptune Q332 (plus hypothetical Theia Q1053432 — filter out). Moons = `?m wdt:P397 <planet>; wdt:P31/wdt:P279* wd:Q2537` (natural satellite; note **Q2199 is "dwarf planet"**, not moon): Saturn 165, Jupiter 85, Uranus 29, Neptune 16, Earth 5, Mars 2. Caveats: Wikidata lags IAU counts (Saturn has 274 recognised moons as of 2025 — UNVERIFIED figure); Earth's 5 includes quasi-satellites; provisional designations (`S/2004 S 3`) are labels too — filter `^S/\d{4}` if you want proper names only.
+
+### Blood type distribution
+
+- **US** (Stanford Blood Center https://stanfordbloodcenter.org/donate-blood/blood-donation-facts/blood-types/, verified; cites AABB Technical Manual 18th ed.): O+ 37.4 %, A+ 35.7 %, B+ 8.5 %, AB+ 3.4 %, O− 6.6 %, A− 6.3 %, B− 1.5 %, AB− 0.6 % (sums to 100.0). Facts — freely embeddable; cite Stanford/AABB.
+- American Red Cross https://www.redcrossblood.org/donate-blood/blood-types.html — **UNVERIFIED**: site returns HTTP 403 (Akamai) to both WebFetch and the headless browser. Search snippets say the Red Cross gives type O by ethnicity (Latino 57 %, African American 51 %, Caucasian 45 %) rather than an 8-way national table.
+- **Global**: no authoritative figure found. Wikipedia "Blood type distribution by country" gives a population-weighted row (O+ 38.4, A+ 27.3, B+ 8.1, AB+ 2.0, O− 13.1, A− 8.1, B− 2.0, AB− 0.01) that its own editors flag "unreliable source"; WorldAtlas gives O+ 42, A+ 31, B+ 15, AB+ 5, O− 3, A− 2.5, B− 1, AB− 0.5 with no source. Recommend shipping only the US table, or per-country tables from Wikipedia (facts; Wikipedia prose is CC BY-SA but the numbers are cited to national blood services — cite those).
+- Validation: ABO ∈ {A, B, AB, O}; Rh ∈ {+, −}; weights must sum to 100 ± rounding.
+
+## 8. Text
+
+### English word lists with part of speech
+
+| Source | Licence | Embed | Counts | Format |
+|---|---|---|---|---|
+| Princeton WordNet 3.0 / 3.1 https://wordnet.princeton.edu/license-and-commercial-use ; download https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz | "WordNet License" (SPDX `WordNet`), BSD-style: permission to use/copy/modify/distribute "for any purpose and without fee or royalty … provided that … the following copyright notice and statements, including the disclaimer … appear on ALL copies … including modifications". Notice text: "WordNet 3.0 Copyright 2006 by Princeton University. All rights reserved." + AS-IS disclaimer; no use of Princeton's name in advertising. | Yes — ship the LICENSE text alongside the data | WordNet 3.0 (wnstats(7WN), verified): unique strings noun 117,798 / verb 11,529 / adj 21,479 / adv 4,481 (total 155,287); synsets 82,115 / 13,767 / 18,156 / 3,621 | WNDB: `index.noun|verb|adj|adv` (one lemma per line: `lemma pos synset_cnt p_cnt [ptr…] sense_cnt tagsense_cnt synset_offset…`), `data.<pos>` (synsets with glosses). Multi-word lemmas use `_` for spaces; lower-case; entries can contain digits/apostrophes/hyphens. |
+| Open English WordNet 2025 https://en-word.net/ ; https://github.com/globalwordnet/english-wordnet | **CC BY 4.0**; cite "Open English WordNet … derived from Princeton WordNet" | Yes, with attribution | 135,969 words, 107,519 synsets (README, verified) | `english-wordnet-2025.zip` (WNDB, 9.2 MB), `english-wordnet-2025.xml.gz` (WN-LMF, 10.8 MB), `english-wordnet-2025-json.zip` (9.5 MB), `.ttl.gz` (16.9 MB) |
+| SCOWL v2 / ESDB https://github.com/en-wl/wordlist (branch v2) ; classic http://wordlist.aspell.net/ | Custom MIT-like (Copyright 2000-2026 Kevin Atkinson: use/copy/modify/distribute/sell "provided that the above copyright notice appears in all copies and that both the above copyright notice and this notice appear in supporting documentation"). Sources 12dicts + ENABLE2K are public domain; lists above size 80 add the UKACD notice; **POS assignments partly from WordNet, so the WordNet notice "MIGHT apply"** (their words). Australian-English parts carry an extra notice. | Yes, ship the Copyright file | Sizes 35 (small), 50 (medium), 60 (spell-check default), 70 (large), 80 (incl. game words), 85 (archaic). No per-size counts published in the v2 README. Classic SCOWL sizes 10–95 — counts **UNVERIFIED** (README not reachable). | v2: `scowl.db` (SQLite), `scowl.txt`, Python `libscowl`. ESDB rows carry POS, spelling (A/B/Z/C/D) and region codes. |
+| 12dicts http://wordlist.aspell.net/12dicts-readme/ | Public domain ("I explicitly release them to the public domain, but request acknowledgment"), **except** `2of12inf` and the `2+2+3*` lists, which depend on AGID and inherit its terms | Yes (PD lists); acknowledge Alan Beale | 3esl ≈22k, 6of12 ≈32k, 2of12 ≈41k, 2of12inf ≈82k, 3of6game ≈65k, 5d+2a ≈68k, 3of6all ≈83k, 2+2+3lem ≈84k, 2of5core ≈4.7k, 6phrase ≈22k, neol2016 ≈600 | Plain text, one word per line; marker suffixes `+ ! ^ &` (not POS). **No POS tags** in any current 12dicts list. |
+| Moby Part-of-Speech II https://www.gutenberg.org/ebooks/3203 ; file https://www.gutenberg.org/files/3203/files/mobypos.txt | Public domain ("Public Domain material by grant from the author, January, 2001", in the package README) | Yes, no notice needed | **233,356** lines (verified) | One entry per line: `word\CODES`, CRLF line ends (e.g. `A-line\NA`, `a tempo\Avh`). Codes in priority order: `N` noun, `p` plural, `h` noun phrase, `V` verb (usu. participle), `t` transitive verb, `i` intransitive verb, `A` adjective, `v` adverb, `C` conjunction, `P` preposition, `!` interjection, `r` pronoun, `D` definite article, `I` indefinite article, `o` nominative. Many entries are phrases, proper nouns, or 1990s-era; non-ASCII entries use a legacy 8-bit encoding (e.g. `a bon march\v` lost its `é`) — restrict to ASCII `[a-z]+` for a clean list. |
+| faker-js `en/word/*` (MIT) | MIT | Yes | adjective 1000, noun 1000, verb 1000, adverb 325, preposition 109, conjunction 51, interjection 46 | TS string arrays |
+
+### Word frequency lists
+
+| Source | Licence | Verdict |
+|---|---|---|
+| Google Books Ngram data v2/v3 https://storage.googleapis.com/books/ngrams/books/datasetsv3.html | **CC BY 3.0** ("This compilation is licensed under a Creative Commons Attribution 3.0 Unported License") | Embeddable with attribution. Format `ngram TAB year TAB match_count TAB volume_count`; 1-gram files split by leading letter, GBs each — aggregate offline to a top-N list. |
+| wordfreq https://github.com/rspeer/wordfreq | Code **Apache 2.0** (not MIT). Data: "may be redistributed under a Creative Commons Attribution-ShareAlike 4.0 license" (mix of Google Books, Wikipedia, OpenSubtitles, SUBTLEX, Leeds, ParaCrawl, Twitter); maintainer notes the CSV export "does not follow the CC-By-SA license". Project in sunset mode (data snapshot ≈2021). | **Avoid** embedding the data (ShareAlike). |
+| Peter Norvig count_1w.txt https://norvig.com/ngrams/ | Page says "Code … under the MIT license"; the **data** comes from the Google Web 1T corpus distributed by LDC under LDC terms — no data licence stated. | **UNVERIFIED / avoid**. 333,333 words, `word TAB count`, lowercase. |
+| hermitdave/FrequencyWords https://github.com/hermitdave/FrequencyWords | "MIT License for code. CC-by-sa-4.0 for content." | **Avoid** (ShareAlike). |
+
+### Lorem ipsum
+
+- Canonical paragraph (Letraset, 1966; public domain — garbled Cicero): "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+- Source: Cicero, *De finibus bonorum et malorum* 1.10.32–33 (45 BC), public domain; the "dolorem ipsum" fragment begins "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit…". Reference: https://en.wikipedia.org/wiki/Lorem_ipsum
+- MIT word pool alternative: faker-js `en/lorem/word.ts` — 999 words.
+
+### Quote collections
+
+- Project Gutenberg #27889 https://www.gutenberg.org/ebooks/27889 — Bartlett, *Familiar Quotations*, **9th edition** (title page "NINTH EDITION", copyright 1875/1882/1891/1903) — not the 1919 10th. "Public domain in the USA." Plain text 3.1 MB, HTML zip 50 MB, EPUB.
+- Project Gutenberg #16732 https://www.gutenberg.org/ebooks/16732 — an early (Hurst & Co. reprint) edition, no edition number given. PD in USA.
+- 10th edition (1914/1919, ed. Nathan Haskell Dole) exists on archive.org (https://archive.org/details/bartlettsfamilia0000john_o4b4) — not found on Gutenberg; **UNVERIFIED** availability as clean text.
+- Either Bartlett text needs parsing (quote / author / source headings); no structured file. Wikiquote is CC BY-SA — avoid.
+- Validation: strip Gutenberg header/footer (`*** START OF THE PROJECT GUTENBERG EBOOK` … `*** END …`) before extraction; Gutenberg's trademark terms only bind if you keep the "Project Gutenberg" name on the output — strip it.
+
+### Colour names
+
+| Source | Licence | Embed | Count / format |
+|---|---|---|---|
+| CSS Color Module Level 4 §named colors https://www.w3.org/TR/css-color-4/#named-colors | W3C Document License (2023) https://www.w3.org/copyright/document-license-2023/ — copy/distribute allowed for any purpose; derivatives allowed only "to facilitate implementation of the technical specifications"; notice required: "Copyright © 2023 W3C®. This software or document includes material copied from or derived from [title and URI]." | Yes — the name→hex table is factual data and is copied into every browser; include the W3C notice line with the spec title + URI. | **148** names (verified: aliceblue … yellowgreen, incl. `rebeccapurple #663399`); all names lowercase ASCII, hex 6 lowercase digits. |
+| XKCD colour survey https://xkcd.com/color/rgb.txt | **CC0** (file header: `# License: https://creativecommons.org/publicdomain/zero/1.0/`) | Yes, no notice needed | **949** lines (verified). Format `name TAB #rrggbb TAB` (note the trailing tab). Max name length 26; all hex valid lowercase; no duplicate names. Contains crude names ("poo", "baby poop green", "diarrhea", "vomit") — apply a blocklist. |
+| faker-js `en/color/human.ts` | MIT | Yes | 31 names |
+| Pantone | Proprietary | **Avoid** | — |
+
+## 9. Dates
+
+### IANA tz database
+
+- Source https://www.iana.org/time-zones ; files https://data.iana.org/time-zones/tzdb/ (current version **2026d**, verified). LICENSE: "Unless specified below, all files in the tz code and data (including this LICENSE file) are in the public domain" (only `date.c`, `newstrftime.3`, `strftime.c` are BSD-3). Embed: Yes, no notice needed.
+- `zone1970.tab` — **312** data rows; tab-separated; UTF-8; `#` comments. Columns: (1) comma-separated ISO 3166-1 alpha-2 codes of countries overlapping the zone, most-populous first; (2) coordinates in ISO 6709 `±DDMM±DDDMM` or `±DDMMSS±DDDMMSS` (lat then lon, no separator); (3) TZ name (`Europe/Zurich`); (4) comment, present iff the country has several zones. One row per timezone where civil time has agreed since 1970.
+- `zone.tab` — **418** rows, older one-country-per-row table (column 1 is a single country code); kept for backward compatibility. Prefer `zone1970.tab`.
+- `backward` — **252** `Link TARGET LINK-NAME` lines mapping old/merged names (`US/Eastern`, `Asia/Calcutta`, `Europe/Kiev`) to current ones; also a few `Zone` entries. Use it to (a) accept aliases as input and (b) never emit them as output.
+- UTC offsets: do **not** embed — offsets depend on DST rules that change several times a year. Derive at runtime: `new Intl.DateTimeFormat('en-US', { timeZone, timeZoneName: 'longOffset' }).formatToParts(date)` → part `timeZoneName` = `GMT-05:00` (or `GMT` for zero) — strip `GMT`, treat bare `GMT` as `+00:00`. Validation of a generated zone name: `Intl.supportedValuesOf('timeZone')` (canonical names only; whether links like `US/Eastern` appear depends on the ICU build) or `try { new Intl.DateTimeFormat(undefined, { timeZone }) } catch { invalid }`, which accepts links too.
+
+### ISO 8601 layouts (https://en.wikipedia.org/wiki/ISO_8601)
+
+| Layout | Extended | Basic | Example |
+|---|---|---|---|
+| Calendar date | `YYYY-MM-DD` | `YYYYMMDD` | 2009-01-06 |
+| Ordinal date | `YYYY-DDD` | `YYYYDDD` | 1981-095 |
+| Week date | `YYYY-Www-D` | `YYYYWwwD` | 2009-W01-1 (weeks start Monday, week 1 contains 4 Jan) |
+| Time | `hh:mm:ss[.fff]` | `hhmmss` | 13:47:30 (24 h; `T` prefix optional standalone) |
+| UTC / offset | `Z` or `±hh:mm` | `±hhmm`, `±hh` | 2007-04-05T14:30Z, 2024-06-01T09:00-05:00 |
+| Date-time | date `T` time offset | | 2007-04-05T14:30:00+02:00 |
+| Duration | `PnYnMnDTnHnMnS` / `PnW` | | P3Y6M4DT12H30M5S |
+| Interval | start`/`end, start`/`duration, duration`/`end | | 2007-03-01T13:00:00Z/2008-05-11T15:30:00Z |
+| Recurring | `Rn/`interval | | R5/2008-03-01T13:00:00Z/P1Y2M10DT2H30M |
+
+RFC 3339 profile (what JSON/APIs usually mean): extended forms only, `T`/`Z` may be lowercase, space allowed instead of `T`, `-00:00` = unknown offset, no durations/intervals/week/ordinal dates. Generate RFC 3339 by default.
+
+### Unicode CLDR (locale month/weekday names, date patterns)
+
+- Licence: **Unicode License v3** (SPDX `Unicode-3.0`), OSI-approved 2023-11-17 (https://opensource.org/license/unicode-3-0). Text https://www.unicode.org/license.txt — MIT-style: use/copy/modify/sell "provided that either (a) this copyright and permission notice appear with all copies of the Data Files or Software, or (b) this copyright and permission notice appear in associated Documentation"; no use of the Unicode name in advertising. Embed: Yes; put the notice in LICENSE/NOTICE.
+- npm: `cldr-core`, `cldr-dates-full`, `cldr-numbers-full` (peer of dates), etc.; current **48.2.0** (CLDR 48, verified via npm registry, `license: Unicode-3.0`); repo https://github.com/unicode-org/cldr-json .
+- Path is **`cldr-dates-full/main/<locale>/ca-gregorian.json`** (not `dates/gregorian.json`), rooted at `main.<locale>.dates.calendars.gregorian`. Keys (verified for `en`): `months`, `days`, `quarters`, `dayPeriods`, `eras`, `dateFormats`, `dateSkeletons`, `timeFormats`, `timeSkeletons`, `dateTimeFormats`, `dateTimeFormats-atTime`, `dateTimeFormats-relative`.
+  - `months.{format|stand-alone}.{abbreviated|narrow|wide}` keyed `"1".."12"`; `days.{format|stand-alone}.{abbreviated|narrow|short|wide}` keyed `sun..sat`.
+  - `dateFormats`: en = full `EEEE, MMMM d, y`, long `MMMM d, y`, medium `MMM d, y`, short `M/d/yy`.
+  - `timeFormats`: en = full `h:mm:ss a zzzz`, long `h:mm:ss a z`, medium `h:mm:ss a`, short `h:mm a` — **the `a` is preceded by U+202F NARROW NO-BREAK SPACE**; `*-alt-ascii` variants use a plain space. Pick one spelling (ASCII) and ignore the other.
+  - `dateTimeFormats.{full|long|medium|short}` = glue like `{1}, {0}` ({1} date, {0} time); `availableFormats` = skeleton→pattern map (`yMMMd` → `MMM d, y`), plus `intervalFormats`, `appendItems`.
+  - `dayPeriods.format.abbreviated`: `am: AM`, `pm: PM`, plus `midnight`, `noon`, `morning1`… variants.
+- Pattern letters follow LDML (https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table): `y` year, `M`/`L` month (format/stand-alone), `d` day, `E`/`c` weekday, `h` 1–12, `H` 0–23, `a` AM/PM, `z`/`zzzz` zone name, `x`/`X` ISO offset; literal text in single quotes. Same patterns drive `Intl.DateTimeFormat` internally, so runtime `Intl` can replace embedding for month/weekday names if bundle size matters.
+
+## Summary of licence verdicts
+
+- Embed freely (PD/CC0): PubChem periodic table, NIST, NASA fact sheet, Wikidata, Moby POS, 12dicts (PD lists), Bartlett via Gutenberg, XKCD colours, IANA tz, Lorem ipsum/Cicero.
+- Embed with notice: WordNet 3.x (WordNet License), Open English WordNet (CC BY 4.0), SCOWL/ESDB (custom MIT-like), GBIF backbone (CC BY 4.0 + citation), BIPM SI (CC BY 4.0), CSS named colours (W3C notice), CLDR (Unicode-3.0), Google Books ngrams (CC BY 3.0), faker-js lists (MIT).
+- Avoid: Bowserinator periodic table (CC BY-SA 3.0), wordfreq data (CC BY-SA 4.0), hermitdave (CC BY-SA 4.0), Norvig count_1w (LDC-derived, unstated), Wikiquote (CC BY-SA), Pantone.
+- UNVERIFIED: American Red Cross figures (site blocks fetches), any global blood-type table, BIPM PDF front-matter licence line, classic SCOWL per-size counts, USDA PLANTS, IAU moon counts, exact faker dog count, Bartlett 10th-edition text availability.
+# Part 4 — ISO lists, commerce, books/music/food
+
+Researched 2026-09-17. "Embed?" = can the data ship inside an MIT-licensed repo. Anything marked UNVERIFIED was not confirmed against the primary source.
+
+## 10. ISO lists
+
+### ISO 3166-1 countries
+
+| Source | Licence | Embed in MIT repo? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [datasets/country-codes](https://github.com/datasets/country-codes) | PDDL (README: "Public Domain Dedication and License"; GitHub API detects no licence file) | Yes, no attribution required. README caveat: ISO itself says its list is "for internal use and non-commercial purposes free of charge" — the repo argues no rights subsist in a list of facts | 249 | ISO3166-1-Alpha-2/-Alpha-3/-numeric, official_name_en, UNTERM names (ar/zh/en/fr/ru/es short+formal), Dial, TLD, Capital, Continent, Languages, ISO4217 currency code/name/numeric/minor_unit, Region/Sub-region/Intermediate names+codes (M49), FIPS, IOC, FIFA, MARC, ITU, WMO, GAUL, EDGAR, Geoname ID, wikidata_id, CLDR display name, is_independent, LDC/LLDC/SIDS flags | CSV `data/country-codes.csv` |
+| [annexare/Countries](https://github.com/annexare/Countries) (npm `countries-list` 3.4.1, 2026-07-14) | MIT | Yes; keep MIT notice | 252 countries, 185 languages (incl. non-ISO entries like AC, XK) | per country: name, native, phone[] (calling codes), continent, capital, currency[], languages[], optional alias[], partOf; separate ISO 4217 currency table (name, native, symbol, numeric, decimals); languages: name, native | TS source, exported JSON/CSV/SQL |
+| [Debian iso-codes](https://salsa.debian.org/iso-codes-team/iso-codes) | LGPL-2.1-or-later (Debian copyright file) | Not safely as embedded data in an MIT repo — LGPL applies to the data files; you would have to ship it as a separately licensed component with LGPL notice. Avoid unless accepting that | 3166-1: 249 | alpha_2, alpha_3, numeric, name, official_name, common_name, flag | JSON `data/iso_3166-1.json` (raw URL needs `?inline=false`) |
+| [mledoze/countries](https://github.com/mledoze/countries) | ODbL-1.0 | No — share-alike + attribution; derived databases must be ODbL. Avoid | ~250 | common/official names, cca2/cca3/ccn3/cioc, tld, currencies, idd (calling codes), capital, region/subregion, languages, translations, latlng, borders, area, demonyms, flag emoji, UN member | JSON/CSV/XML/YAML |
+| [restcountries](https://gitlab.com/restcountries/restcountries) | MPL-2.0 (source repo; hosted API v5 is proprietary/API-key) | File-level copyleft: a copied data file stays MPL and must carry the notice; OK to ship next to MIT code but not to relicense. Prefer PDDL/MIT sources | 250+ | name, tld, cca2, ccn3, cca3, cioc, fifa, independent, status, unMember, currencies, idd, capital, capitalInfo, altSpellings, region, subregion, continents, languages, translations, latlng, landlocked, borders, area, flag (emoji), demonyms, flags, coatOfArms, population, maps, gini, car, postalCode, startOfWeek, timezones | JSON `src/main/resources/countriesV3.1.json` |
+| [lukes/ISO-3166-Countries-with-Regional-Codes](https://github.com/lukes/ISO-3166-Countries-with-Regional-Codes) | CC BY-SA 4.0 (LICENSE.md) | No — share-alike. Avoid | 249 | name, alpha-2, alpha-3, country-code, iso_3166-2, region, sub-region, intermediate-region + codes | CSV/JSON/XML (all, slim-2, slim-3) |
+| [stefangabos/world_countries](https://github.com/stefangabos/world_countries) | Data: CC BY-SA 4.0 (README, GitHub licence detection); npm `world_countries_lists` package.json says LGPL-3.0-or-later — contradictory | No — share-alike either way. Avoid | 249 world / 193 UN; subdivisions 5046 | id (numeric), alpha2, alpha3, name (37 languages); subdivisions: country, code, name, name_en, type, parent | JSON/CSV/PHP/SQL/XML |
+| [i18n-iso-countries](https://www.npmjs.com/package/i18n-iso-countries) 7.14.0 | MIT | Yes | 78 language files | alpha2, alpha3, numeric, localized names | JSON per language |
+| Wikipedia ISO 3166-1 | CC BY-SA 4.0 | No (share-alike) | — | — | — |
+| [ISO OBP](https://www.iso.org/obp) | Proprietary; ISO grants free use only for "internal use and non-commercial purposes" (quoted in datasets/country-codes README) | No | — | — | — |
+
+- Flag emoji: derive, no data needed — for alpha-2 `XY`, emit U+1F1E6 + (X − 'A') and U+1F1E6 + (Y − 'A') (regional indicator symbols).
+- Recommendation: `datasets/country-codes` (PDDL) covers every requested field (name, alpha2, alpha3, numeric, calling code, TLD, capital, currency, languages) in one CSV; flag emoji derived. `annexare` (MIT) is the fallback for native names/calling codes if PDDL's ISO caveat worries you; it lacks TLD.
+- Validation rules: alpha-2 `^[A-Z]{2}$`, alpha-3 `^[A-Z]{3}$`, numeric `^\d{3}$` (zero-padded, keep as string); calling code 1–3 digits, NANP countries share `1`; TLD `^\.[a-z]{2}$` (ccTLD = lowercase alpha-2, exceptions: GB uses `.uk`).
+
+### ISO 639 languages
+
+| Source | Licence | Embed? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [langs](https://www.npmjs.com/package/langs) 2.0.0 (2017, unmaintained) | MIT | Yes | 184 | name, local, 1 (639-1), 2, 2T, 2B, 3 | `data.js` array |
+| [iso-639-1](https://www.npmjs.com/package/iso-639-1) 3.1.6 (2026-07-02) | MIT | Yes | 183 | code, name, nativeName | `src/data.js` |
+| annexare/Countries languages | MIT | Yes | 185 | code (639-1), name, native | TS/JSON |
+| Debian iso-codes 639-2 / 639-3 / 639-5 | LGPL-2.1+ | See above (LGPL) | 639-2: 487; 639-3: 7923 | 639-2: alpha_2, alpha_3, bibliographic, name, common_name; 639-3: + inverted_name, scope, type | JSON |
+| [SIL ISO 639-3 tables](https://iso639-3.sil.org/code_tables/download_tables) | SIL terms: may incorporate into software (commercial or not) with attribution to iso639-3.sil.org, must not modify identifiers, and product "does not provide a means to redistribute the code set" | Risky — a git repo with the table IS a means to redistribute; avoid embedding the full table. Codes themselves are facts | 7927 | Id, Part2b, Part2t, Part1, Scope, Language_Type, Ref_Name, Comment | tab-delimited `iso-639-3.tab` |
+| [Library of Congress ISO 639-2](https://www.loc.gov/standards/iso639-2/ascii_8bits.html) | US federal government work → public domain in the US | Yes | ~487 (matches Debian 639-2 count) | pipe-delimited: alpha3-B, alpha3-T, alpha2, English name, French name — UNVERIFIED: loc.gov is behind a Cloudflare bot check (WebFetch, curl and Playwright all blocked); field layout from memory | `ISO-639-2_utf-8.txt` |
+
+- Recommendation: `langs` or `iso-639-1` (MIT) for 639-1 names; both are small and derived from public code tables. Anything 639-3-sized is LGPL or SIL-restricted.
+- Rules: 639-1 `^[a-z]{2}$`, 639-2/3 `^[a-z]{3}$`; 639-2 has B/T pairs (e.g. `fre`/`fra`, `ger`/`deu`) — pick one consistently (T codes = 639-3).
+
+### ISO 3166-2 subdivisions
+
+| Source | Licence | Embed? | Rows | Fields |
+|---|---|---|---|---|
+| Debian iso-codes 3166-2 | LGPL-2.1+ | LGPL caveat | 5046 | code, name, type, parent |
+| [olahol/iso-3166-2.json](https://github.com/olahol/iso-3166-2.json) | package.json says ISC; no LICENSE file; GitHub detects none; data scraped from eQuest xls (provenance unclear); last push 2021 | No — unclear licence and provenance | 237 countries / 3807 divisions | `{alpha2: {name, divisions: {code: name}}}` |
+| [esosedi/3166](https://github.com/esosedi/3166) (npm `iso3166-2-db` 2.3.11) | MIT | Yes; data mixes GeoNames (CC BY 4.0), Wikipedia (CC BY-SA), OSM (ODbL) — the MIT label may not survive scrutiny for the Wikipedia/OSM-derived parts. UNVERIFIED how much is derived from each | "all countries and regions" (count not stated) | ISO 3166-2 + FIPS codes, names in 12 languages, admin level, GeoNames/OSM/Wikipedia/WOF ids |
+| stefangabos subdivisions | CC BY-SA 4.0 | No | 5046 | country, code, name, name_en, type, parent |
+| [US Census state.txt](https://www2.census.gov/geo/docs/reference/state.txt) | US government work → public domain | Yes | 57 (50 states + DC + 6 territories) | STATE (FIPS), STUSAB, STATE_NAME, STATENS |
+
+- Recommendation: for an en_US library, ship only the US list (Census/USPS, public domain: 50 states + DC, optionally territories) — ISO 3166-2 code = `US-` + USPS abbreviation. Skip the ~5000-row world list; every open compilation is LGPL, CC BY-SA or unclear.
+- Rule: `^[A-Z]{2}-[A-Z0-9]{1,3}$`.
+
+## 11. Commerce
+
+### Word lists (product name / adjective / material)
+
+- [@faker-js/faker](https://github.com/faker-js/faker) 10.6.0 — MIT (LICENSE covers code and locale data; copyright Faker contributors 2022-2025 and Marak Squires 2011-2020; acknowledges Ruby Faker and Perl Data::Faker, both MIT). No separate data licence.
+- Provenance policy in [CONTRIBUTING.md](https://github.com/faker-js/faker/blob/next/CONTRIBUTING.md): "Faker must not contain copyrighted materials"; facts (finite known lists) are OK; compilations must not be copied from a single source (Wikipedia, "most popular" articles, government sites) because "a compilation of facts can be copyrighted" — contributors are told to draw on multiple sources and their own judgement.
+- Verdict: reusing faker's `en` commerce lists (`product_name.adjective/material/product`, `department`) is fine under MIT — keep the MIT notice with the copied data. The lists are short curated words (no external dataset). Nothing in the repo or docs restricts locale-data reuse beyond MIT.
+
+### Garment and shoe sizes
+
+- Letter sizes XS–XXL: EN 13402-3 letter codes with chest/bust ranges (Wikipedia [EN 13402](https://en.wikipedia.org/wiki/EN_13402), CC BY-SA — but the table is a handful of facts; the standard itself is paywalled). Men chest / women bust cm: XXS 70–78 / 66–74, XS 78–86 / 74–82, S 86–94 / 82–90, M 94–102 / 90–98, L 102–110 / 98–107, XL 110–118 / 107–119, XXL 118–129 / 119–131, 3XL 129–141 / 131–143.
+- US numeric women's sizes (0–20, even) come from ASTM D5585 (paywalled); no open dataset. Just generate the even-number series.
+- Shoe sizes: no open dataset found (search hits are ML demos and retailer charts). Use the formulas from Wikipedia [Shoe size](https://en.wikipedia.org/wiki/Shoe_size) (facts, not copyrightable): EU (Paris point) ≈ 1.5 × foot cm + 2 (i.e. 3/20 × mm + 2); UK adult ≈ 3 × foot in − 23; US men ≈ 3 × foot in − 22 = UK + 1; US women (common) = US men + 1.5 (FIA scale: 3 × foot in − 21); Mondopoint = foot length mm. ISO/TS 19407:2023 has the official conversion tables (paywalled). Generate from a foot length (men 24–31 cm, women 21–27 cm) in half-size steps so all systems agree.
+
+### Check-digit rules (all verified against Wikipedia articles; algorithms are facts)
+
+| Code | Rule |
+|---|---|
+| EAN-13 / UPC-A / GTIN-8/12/13/14 | Weights 3,1,3,1… starting from the rightmost data digit (the one next to the check digit); check = (10 − sum mod 10) mod 10. UPC-A = EAN-13 with a leading 0 (GS1 prefix of a 12-digit GTIN is `0` + first two digits). GTIN-14 first digit is an indicator 1–8 (packaging level) or 9 (variable measure); 0 is not valid there. |
+| UPC-A number system (first digit) | 0,1,6,7,8,9 regular; 2 variable-weight; 3 drugs (NDC); 4 in-store/local; 5 coupons. |
+| ISBN-10 | Σ(weights 10..2 × first 9 digits) + check ≡ 0 mod 11; check = (11 − sum mod 11) mod 11, 10 → `X`. |
+| ISBN-13 | EAN-13 with prefix 978 or 979. 979 groups in use: 979-8 USA, 979-10 France, 979-11 Korea, 979-12 Italy. 978 groups 0 and 1 = English-language. For an en_US generator: `978-0…`, `978-1…`, `979-8…`. The official range file (`https://www.isbn-international.org/export_rangemessage.xml`) is XML only and the site says "You must not republish material from our site … without our written permission" — do not embed it; group/registrant length only matters for hyphenation, not validity. |
+| ISSN | Format `NNNN-NNNC`; weights 8..2 over first 7 digits, sum mod 11; check = 0 if remainder 0 else 11 − remainder; 10 → `X`. EAN-13 form: `977` + 7 ISSN digits (no check) + 2 variant digits + EAN check. |
+| IMEI | 15 digits: TAC 8 (RBI 2 + 6) + serial 6 + Luhn check (ISO/IEC 7812; from the right, double every second digit, sum digits, total ≡ 0 mod 10). SVN never enters the check. IMEISV = 14 digits + 2-digit SVN, no check. RBI `00` = test IMEI (Wikipedia [Reporting Body Identifier](https://en.wikipedia.org/wiki/Reporting_Body_Identifier)); live RBIs: 01 CTIA/PTCRB (US), 35 TÜV SÜD/BABT (UK), 86 TAF (China), 99 GHA. For fake data use TAC `00xxxxxx` so it can never match a real handset — UNVERIFIED: third-party pages claim the GSMA TS.06 test-TAC shape is `00 44` + 4 digits; the TS.06 PDF could not be text-extracted here. |
+| ASIN | 10 chars `[A-Z0-9]{10}`; non-books start `B0` + 8 alphanumerics (`^B0[A-Z0-9]{8}$`); books reuse their ISBN-10 verbatim. No check digit (Wikipedia). |
+| SKU | Not standardised (Wikipedia). Convention only: 8–12 uppercase alphanumerics, e.g. `[A-Z]{3}-\d{4}-[A-Z]{2}`; avoid leading 0 and O/I if you want scanner-safe. |
+
+### GS1 prefixes
+
+- Official table: [gs1.org/standards/id-keys/company-prefix](https://www.gs1.org/standards/id-keys/company-prefix) (rendered via Playwright; WebFetch gets 403). Copyright notice ([gs1.org/terms-use](https://www.gs1.org/terms-use)): reproduction allowed only "in unaltered form … for your personal, non-commercial use or use within your organisation"; "You are not permitted to re-transmit, distribute or commercialise the information or material without seeking prior written approval from GS1." → Do not copy the GS1 page. The prefix-to-country mapping is factual and also on Wikipedia ([List of GS1 country codes](https://en.wikipedia.org/wiki/List_of_GS1_country_codes), CC BY-SA 4.0 — avoid verbatim copy). Safe path: embed only the US/special ranges you need, written from the facts below.
+- Ranges relevant to an en_US generator (from the GS1 page): 001–019, 030–039, 060–139 GS1 US (UPC-A compatible); 020–029 restricted circulation within a region; 040–049 restricted within a company; 050–059 GS1 US reserved; 200–299 restricted circulation (region); 952 "used for demonstrations and examples of the GS1 system"; 977 ISSN; 978–979 ISBN; 980 refund receipts; 981–983 coupons (common currency); 990–999 coupons. Prefixes do not identify country of origin. Selected others: 300–379 France, 400–440 Germany, 450–459 & 490–499 Japan, 500–509 UK, 690–699 China, 730–739 Sweden, 750 Mexico, 754–755 Canada, 760–769 Switzerland, 800–839 Italy, 840–849 Spain, 870–879 Netherlands, 880–881 South Korea, 890 India, 930–939 Australia, 940–949 New Zealand.
+- Generator rule: for fake UPC/EAN use prefix `952` (GS1 demo) or a `2xx` restricted-circulation prefix — never a real company prefix; for US-looking UPC-A use number system 0–1/6–8 with a random 5-digit manufacturer code and accept that it may collide with real products.
+
+## 12. Books, music, food
+
+### Project Gutenberg catalog
+
+- URL: `https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv` (20 MB) / `pg_catalog.csv.gz` (5.3 MB), regenerated weekly; also RDF (`rdf-files.tar.bz2`, 121 MB) and MARC.
+- Fields: `Text#, Type, Issued, Title, Language, Authors, Subjects, LoCC, Bookshelves` (comma CSV, quoted; Authors as `Last, First, birth-death`, `;`-separated).
+- Count (2026-09-17): 79 381 rows; Type: Text 78 130, Sound 1 114, Dataset 89, Image 33, other 12; Language: en 62 860, fr 4 190, fi 3 684, de 2 425, it 1 110.
+- Licence: each RDF record carries `<cc:Work><cc:license rdf:resource="https://creativecommons.org/publicdomain/zero/1.0/"/>` → catalog metadata is CC0 1.0 (verified on `cache/epub/1/pg1.rdf`). The CSV itself has no header notice; treat as CC0 by the same source. Terms of use: do not hammer their servers (bulk feeds are the sanctioned path), "Project Gutenberg" is a trademark — royalties for commercial use of the *name*; do not brand the data file with it beyond a source citation.
+- Embed: yes (CC0). For a fake-data library ship a filtered subset (Type=Text, Language=en, title + first author, maybe 5–10k rows) rather than 20 MB.
+
+### Music genres and instruments
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| [MusicBrainz genre list](https://musicbrainz.org/genres) | Genre is a core entity (schema doc lists 13 core entities incl. Genre) → core data dump `mbdump.tar.bz2` is CC0. The genre→entity *associations* come via user tags (CC BY-NC-SA 3.0) — do not use those | Yes, names only | 2 202 (`/ws/2/genre/all?fmt=json`, `genre-count`) | WS API needs a descriptive User-Agent; list is a JSON array of `{id, name, disambiguation}` |
+| [Discogs data dumps](https://data.discogs.com/) | CC0 | Yes | genre/style vocab is embedded in release XML (monthly dumps, GBs) — no standalone style list; UNVERIFIED count (~15 genres, ~600 styles from memory) | Impractical to extract; prefer MusicBrainz |
+| ID3v1 genres | Names 0–79 from the 1999 ID3v1 spec, 80–125 Winamp, 126–191 Winamp 5.6 (2010) — a de-facto spec table, treated as public domain facts | Yes | 192 (0–191) | Wikipedia [List of ID3v1 genres](https://en.wikipedia.org/wiki/List_of_ID3v1_genres) (CC BY-SA page; the list itself is a spec enumeration). id3.org returned HTTP 500 during research — UNVERIFIED against the original spec page |
+| Wikidata instruments | CC0 | Yes | 9 194 items `P31/P279* Q34379` with English labels; 2 372 with a Hornbostel–Sachs number (P1762) | Filter by P1762 for a clean orchestral/folk instrument list |
+| Wikidata music genres | CC0 | Yes | 6 619 (`Q188451`) | Noisier than MusicBrainz |
+
+### USDA FoodData Central
+
+- Downloads: [fdc.nal.usda.gov/download-datasets](https://fdc.nal.usda.gov/download-datasets/) (April 2026 release):
+  - Foundation Foods: JSON 459 KB zip / 6.5 MB; CSV 3.7 MB zip / 32 MB — 394 foods (API `totalHits`, dataType=Foundation).
+  - SR Legacy (final, April 2018): JSON 12.3 MB zip / 205 MB; CSV 6.7 MB zip / 54 MB — 7 793 foods.
+  - FNDDS 2021-2023: CSV 200 MB zip / 1.6 GB — 5 432 foods.
+  - Branded: CSV 428 MB zip / 2.9 GB — 433 403 foods.
+  - Full: CSV 460 MB zip / 3.1 GB.
+- Licence ([API guide](https://fdc.nal.usda.gov/api-guide/)): "USDA FoodData Central data are in the public domain and they are not copyrighted. They are published under CC0 1.0 Universal (CC0 1.0)". No permission needed; USDA *requests* listing FoodData Central as source and notifying them. Suggested citation: "U.S. Department of Agriculture, Agricultural Research Service. FoodData Central, 2019. fdc.nal.usda.gov."
+- Embed: yes (CC0). Ship a derived list (SR Legacy `description` + `food_category`) not the raw dump.
+- CSV structure (UNVERIFIED — the field-description PDF could not be text-extracted; from memory): `food.csv` (fdc_id, data_type, description, food_category_id, publication_date), `food_category.csv` (id, code, description), `nutrient.csv` (id, name, unit_name, nutrient_nbr, rank), `food_nutrient.csv` (id, fdc_id, nutrient_id, amount, …), `food_portion.csv`, `measure_unit.csv`, `sr_legacy_food.csv` (fdc_id, NDB_number), `foundation_food.csv`.
+
+### Dishes, cuisines, drinks
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| Wikidata dishes (`Q746549`) | CC0 | Yes | 7 480 with English label | SPARQL `?i wdt:P31/wdt:P279* wd:Q746549` |
+| Wikidata cuisines (`Q1968435`) | CC0 | Yes | 219 | |
+| Wikidata cocktails (`Q134768`) | CC0 | Yes | 305 | |
+| Wikidata beer (`Q44` subclass tree) / wine (`Q282`) | CC0 | Yes | 420 / 3 025 | Wine tree is mostly appellations/brands; filter by P279 depth |
+| IBA official cocktails | IBA site "© IBA 2026 – All rights reserved"; Wikipedia list CC BY-SA 4.0 | Names only are facts (102 cocktails: 34 Unforgettables, 34 Contemporary Classics, 34 New Era, 2024 revision); do not copy recipes/descriptions | 102 | Safest: take the names from Wikidata (`P31 Q134768`) rather than the IBA page |
+| [Open Brewery DB](https://github.com/openbrewerydb/openbrewerydb) | MIT (LICENSE, © 2025 Open Brewery DB) | Yes | 11 931 breweries (US 8 308, DE 1 445, AU 514, BE 478, CA 283, NZ 243) | `breweries.csv`: id (UUID), name, brewery_type (micro 5 906, brewpub 3 947, closed 644, planning 635, regional 240, contract 208, large 137, proprietor, taproom, bar, nano, cidery, beergarden), address_1..3, city, state_province, postal_code, country, phone, website_url, longitude, latitude. Real businesses — fine for names, but generating "fake" data with real addresses/phones may be undesirable |
+| Wikipedia lists (cuisines, dishes, IBA) | CC BY-SA 4.0 | No verbatim copies | | |
+
+## Summary of safe picks
+
+- Countries: `datasets/country-codes` (PDDL) or `annexare/Countries` (MIT); flag emoji derived from alpha-2.
+- Languages: `langs` / `iso-639-1` (MIT, 639-1 only).
+- Subdivisions: US-only list from Census (public domain); skip world list.
+- Commerce words: faker `en` lists (MIT). Check digits: implement from the rules above; GS1 prefix table: hand-write the few ranges needed, use `952`/`2xx` for generated barcodes; ISBN range file: do not embed.
+- Books: Gutenberg `pg_catalog.csv` (CC0), filtered.
+- Music: MusicBrainz genre names (CC0, 2 202), ID3v1 list (192), Wikidata instruments (CC0).
+- Food: USDA FDC SR Legacy/Foundation descriptions (CC0, cite USDA), Wikidata dishes/cuisines (CC0), Open Brewery DB (MIT).
+- Avoid: mledoze (ODbL), lukes (CC BY-SA), stefangabos (CC BY-SA/LGPL), Debian iso-codes (LGPL), SIL 639-3 table (no-redistribution clause), GS1/ISBN-International pages (all rights reserved), Wikipedia tables verbatim (CC BY-SA), MusicBrainz tag associations (CC BY-NC-SA).

draw.go

@@ -1,461 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// drawSet is one render's reference draws: the unnamed draw group's, and each named one's.
-type drawSet struct {
-	unnamed draws
-	named   map[string]*draws
-}
-
-// drawScope is where a render reads its reference paths: a draw set, in the draw group of the
-// template rendering, and the row a table's format is rendering, which its columns read.
-type drawScope struct {
-	set   *drawSet
-	group string
-	t     *table
-	row   int
-}
-
-// newDrawSet makes the unnamed group's maps where the set is declared, keeping them on that frame's
-// stack for a render that reads through them; a zero drawSet makes them on its first read instead.
-func newDrawSet(s *session) drawSet {
-	return drawSet{unnamed: draws{variant: map[string]node{}, value: map[string]draw{}, s: s}}
-}
-
-// renderOnce renders n as one render, over draws of its own.
-func renderOnce(s *session, n node) string {
-	set := drawSet{unnamed: draws{s: s}}
-	return render(s, n, drawScope{set: &set})
-}
-
-// in is the scope t renders in: its draw group where it names one, else its caller's.
-func (sc drawScope) in(t *template) drawScope {
-	if t.drawGroupKey != "" {
-		sc.group = t.drawGroupKey
-	}
-	return sc
-}
-
-// draws is the set's draws for sc's draw group. The session is passed in rather than
-// read out of the set: copying it from there would leak the set's maps to the heap.
-func (sc drawScope) draws(s *session) *draws {
-	if sc.group == "" {
-		return &sc.set.unnamed
-	}
-	d, drew := sc.set.named[sc.group]
-	if !drew {
-		if sc.set.named == nil {
-			sc.set.named = map[string]*draws{}
-		}
-		d = &draws{variant: map[string]node{}, value: map[string]draw{}, s: s}
-		sc.set.named[strings.Clone(sc.group)] = d // a key from sc would leak sc, and with it every render's draw set, to the heap
-	}
-	return d
-}
-
-// drawGroupOf reads a template's "drawGroup" (default ""), which a repeat cannot carry: each
-// iteration renders in no draw group.
-func drawGroupOf(m map[string]any, repeat int) (string, error) {
-	v, ok := m["drawGroup"]
-	if !ok {
-		return "", nil
-	}
-	name, ok := v.(string)
-	switch {
-	case !ok:
-		return "", fmt.Errorf("drawGroup must be a string, got %T", v)
-	case name == "":
-		return "", fmt.Errorf(`drawGroup "" is the default, so it has no effect; drop it`)
-	case repeat > 1:
-		return "", fmt.Errorf("drawGroup %q on a repeat names nothing, since each iteration is a render of its own; drop it", name)
-	}
-	return name, nil
-}
-
-// keyDrawGroup keys t's draw group by the category t sits in, "" for an inline template, so a name
-// is local to its category.
-func (t *template) keyDrawGroup(category string) {
-	if t.drawGroup != "" {
-		t.drawGroupKey = category + "/" + t.drawGroup
-	}
-}
-
-// checkNestedDrawGroup refuses a template beneath one drawing in group that names group again,
-// short of a repeat or another draw group.
-func checkNestedDrawGroup(fields map[string]node, group string) error {
-	if group == "" {
-		return nil
-	}
-	var walk func(path string, n node) error
-	walk = func(path string, n node) error {
-		t, isTemplate := n.(*template)
-		switch {
-		case isTemplate && t.drawGroup == group:
-			return fmt.Errorf("%q names drawGroup %q, the draw group this template draws in already; drop it", path, group)
-		case isTemplate && (t.drawGroup != "" || t.repeat > 1):
-			return nil
-		}
-		for _, c := range contained(n) {
-			if err := walk(join(path, c.name), c.node); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	for _, name := range sortedNames(fields) {
-		if err := walk(name, fields[name]); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// drawCheck fences each template of a scope as a render of its own, remembering which nodes read a
-// reference path.
-type drawCheck struct {
-	memo map[readsMemo]bool
-}
-
-type readsMemo struct {
-	n           node
-	stopAtGroup bool
-}
-
-// checkDrawGroup refuses a draw group that splits nothing: one whose render reads every reference
-// path inside a repeat or a nested draw group, which draw apart from it whatever it names.
-func (c *drawCheck) checkDrawGroup(path string, n node) error {
-	if t, ok := n.(*template); ok && t.drawGroup != "" && !c.splitsDraws(t) {
-		return fmt.Errorf("%s: drawGroup %q splits nothing, since nothing it renders reads a reference path outside a repeat or a nested drawGroup; drop it", path, t.drawGroup)
-	}
-	return nil
-}
-
-func (c *drawCheck) checkDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok {
-		return nil
-	}
-	if err := checkOwnFamily(t); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	if !c.readsPath(t) {
-		return nil
-	}
-	w := newDrawWalk()
-	for _, e := range renderEdges(t) {
-		w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
-	}
-	if err := w.check(); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// checkOwnFamily refuses a table's format or cell that reads, however many templates
-// away and through a repeat or a draw group too, a table of its own family: a row
-// rendered whole draws its row without pinning it, so the family would draw apart
-// from the row being rendered, whichever draws the reaching template holds.
-func checkOwnFamily(t *template) error {
-	own := t.table
-	if own == nil {
-		own = t.cellOf
-	}
-	if own == nil {
-		return nil
-	}
-	seen := map[node]bool{}
-	var find func(n node) (renderEdge, *table, bool)
-	find = func(n node) (renderEdge, *table, bool) {
-		if seen[n] {
-			return renderEdge{}, nil, false
-		}
-		seen[n] = true
-		for _, e := range renderEdges(n) {
-			if a, isRef := refRead(n, e.label); isRef {
-				if head, isTable := n.(*template).fields[a.key].(*table); isTable && head.family() == own.family() {
-					return e, head, true
-				}
-			}
-			if e, head, found := find(e.to); found {
-				return e, head, true
-			}
-		}
-		return renderEdge{}, nil, false
-	}
-	if e, head, found := find(t); found {
-		return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", e.reached(), head.category, own.category)
-	}
-	return nil
-}
-
-// checkRecordDraws fences the columns of a record — a template compiled at the top without a
-// repeat — so a load proves the record view of it as well as the string view.
-func (c *drawCheck) checkRecordDraws(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || !t.record {
-		return nil
-	}
-	// A record-only template's format renders nothing, so weigh the columns, not the format.
-	columns := recordColumns(t)
-	reads := false
-	for _, name := range columns {
-		reads = reads || c.readsPath(t.fields[name])
-	}
-	if !reads {
-		return nil
-	}
-	if err := checkColumnDraws(t, columns); err != nil {
-		return fmt.Errorf("%s: %w", path, err)
-	}
-	return nil
-}
-
-// readsPath reports whether rendering n reads a reference path, short of a repeat, which renders
-// over draws of its own.
-func (c *drawCheck) readsPath(n node) bool { return c.reads(n, false) }
-
-// splitsDraws reports whether rendering n reads a reference path that n's own draw group answers
-// for: one outside a repeat and outside a nested draw group, which hold their own draws.
-func (c *drawCheck) splitsDraws(n node) bool { return c.reads(n, true) }
-
-// reads walks what rendering n renders for a reference path, stopping at a repeat — and at a nested
-// draw group when stopAtGroup — since each holds draws of its own.
-func (c *drawCheck) reads(n node, stopAtGroup bool) bool {
-	k := readsMemo{n, stopAtGroup}
-	if r, done := c.memo[k]; done {
-		return r
-	}
-	r := false
-	for _, e := range renderEdges(n) {
-		if a, isRef := refRead(n, e.label); (isRef && (len(a.tail) > 0 || readsTable(n, a))) || (!repeats(e.to) && !(stopAtGroup && grouped(e.to)) && c.reads(e.to, stopAtGroup)) {
-			r = true
-			break
-		}
-	}
-	if c.memo == nil {
-		c.memo = map[readsMemo]bool{}
-	}
-	c.memo[k] = r
-	return r
-}
-
-func repeats(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.repeat > 1
-}
-
-func grouped(n node) bool {
-	t, isTemplate := n.(*template)
-	return isTemplate && t.drawGroupKey != ""
-}
-
-// readsTable reports whether a reference of n names a table, whose draw the render's
-// group answers for even when read whole.
-func readsTable(n node, a arm) bool {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return false
-	}
-	_, isTable := t.fields[a.key].(*table)
-	return isTable
-}
-
-// refRead is the reference an edge of n reads; false when the edge reads none.
-func refRead(n node, label string) (arm, bool) {
-	t, isTemplate := n.(*template)
-	if !isTemplate {
-		return arm{}, false
-	}
-	a := splitArm(label, t.refs)
-	return a, isRef(a.key)
-}
-
-// checkColumnDraws fences a record's columns as one render.
-func checkColumnDraws(t *template, columns []string) error {
-	w := newDrawWalk()
-	for _, name := range columns {
-		w.walk(t.fields[name], drawAt{group: t.drawGroupKey, route: drawRoute{spelling: fmt.Sprintf("column %q", name)}})
-	}
-	return w.check()
-}
-
-// drawWalk gathers the references one render reads, by draw group, for check to compare.
-type drawWalk struct {
-	reads []pathRead
-	read  map[drawKey]bool
-	seen  map[drawVisit]bool
-}
-
-// drawAt is where a walk stands: the draw group it draws in, how the render's root reached it, and
-// the table rows it entered.
-type drawAt struct {
-	group string
-	route drawRoute
-	alt   rowSet
-}
-
-// rowSet is the rows a walk entered, one per table: only one row of a table renders, so reads in
-// two rows of one table never meet, while reads in one row, across its columns and whatever they
-// reach, do.
-type rowSet []tablePin
-
-func (s rowSet) rowOf(t *table) (int, bool) {
-	for _, p := range s {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	return 0, false
-}
-
-// enter is s with row r of t, where t is not in it yet; the set is copied, since walks branch.
-func (s rowSet) enter(t *table, r int) rowSet {
-	if _, in := s.rowOf(t); in {
-		return s
-	}
-	out := append(append(make(rowSet, 0, len(s)+1), s...), tablePin{t, r})
-	sort.Slice(out, func(i, j int) bool { return out[i].t.category < out[j].t.category })
-	return out
-}
-
-// key spells the set for a map, by the tables' identities.
-func (s rowSet) key() string {
-	var b strings.Builder
-	for _, p := range s {
-		fmt.Fprintf(&b, "%p[%d]", p.t, p.row)
-	}
-	return b.String()
-}
-
-// alternatives reports whether two reads sit in different rows of one table.
-func alternatives(a, b drawAt) bool {
-	for _, p := range a.alt {
-		if r, in := b.alt.rowOf(p.t); in && r != p.row {
-			return true
-		}
-	}
-	return false
-}
-
-// drawRoute is how a render reaches a draw: as its author spells it, and the root edge's label.
-type drawRoute struct{ spelling, label string }
-
-// pathRead is one reference a render reads: a path, or a bare reference with no tail.
-type pathRead struct {
-	at drawAt
-	a  arm
-	tr *tableRead // set where the reference names a table
-}
-
-type drawVisit struct {
-	n     node
-	group string
-	alt   string
-}
-
-type drawKey struct {
-	group string
-	path  string
-	alt   string
-}
-
-func newDrawWalk() *drawWalk {
-	return &drawWalk{read: map[drawKey]bool{}, seen: map[drawVisit]bool{}}
-}
-
-// walk follows what rendering n renders. A repeat renders over draws of its own, so the walk stops
-// there.
-func (w *drawWalk) walk(n node, at drawAt) {
-	v := drawVisit{n, at.group, at.alt.key()}
-	if w.seen[v] {
-		return
-	}
-	w.seen[v] = true
-	if repeats(n) {
-		return
-	}
-	if t, isTemplate := n.(*template); isTemplate && t.drawGroupKey != "" {
-		at.group = t.drawGroupKey
-	}
-	for _, e := range renderEdges(n) {
-		if cell, isCell := e.to.(*template); isCell && cell.cellOf != nil {
-			if at.alt.excludes(cell) {
-				continue
-			}
-			w.edge(n, e, drawAt{at.group, at.route, at.alt.enter(cell.cellOf, cell.cellRow)})
-			continue
-		}
-		w.edge(n, e, at)
-	}
-}
-
-// excludes reports whether a cell's row cannot render with the rows entered: another
-// row of its table, or a row outside an entered ancestor's.
-func (s rowSet) excludes(cell *template) bool {
-	for _, p := range s {
-		if p.t == cell.cellOf {
-			return p.row != cell.cellRow
-		}
-		if cell.cellOf.descends(p.t) && !cell.cellOf.under(cell.cellRow, p.t, p.row) {
-			return true
-		}
-	}
-	return false
-}
-
-// edge records the reference an edge reads, then walks on with every row the read
-// pins entered, so a selected row renders only its own cells.
-func (w *drawWalk) edge(from node, e renderEdge, at drawAt) {
-	if a, reads := refRead(from, e.label); reads {
-		tr := tableReadOf(from.(*template).fields[a.key], a, e.to)
-		if k := (drawKey{at.group, a.path, at.alt.key()}); !w.read[k] {
-			w.read[k] = true
-			w.reads = append(w.reads, pathRead{at, a, tr})
-		}
-		if tr != nil {
-			tr.pins.each(func(t *table, r int) { at.alt = at.alt.enter(t, r) })
-		}
-	}
-	w.walk(e.to, at)
-}
-
-// check refuses what one draw per reference path cannot answer for: a read of a level beside a path
-// another read takes into it, and two reads of one table family that select different rows. Reads
-// are compared in path order, so which pair is reported does not vary.
-func (w *drawWalk) check() error {
-	sort.SliceStable(w.reads, func(i, j int) bool {
-		if w.reads[i].at.group != w.reads[j].at.group {
-			return w.reads[i].at.group < w.reads[j].at.group
-		}
-		return w.reads[i].a.path < w.reads[j].a.path
-	})
-	for i, level := range w.reads {
-		for _, into := range w.reads[i+1:] {
-			if into.at.group != level.at.group || alternatives(level.at, into.at) {
-				continue
-			}
-			if strings.HasPrefix(into.a.path, level.a.path+".") && !(level.tr != nil && level.tr.whole) {
-				return overlapError(level.at.route, level.a.name, into)
-			}
-		}
-	}
-	return checkFamilies(w.reads)
-}
-
-func overlapError(route drawRoute, ref string, into pathRead) error {
-	return fmt.Errorf("%s renders a level that %s reads a path into; name the fields you want instead, or draw them apart with a drawGroup", route.spelled(ref), into.at.route.spelled(into.a.name))
-}
-
-// spelled names the route, and the reference it reaches a draw by where its root edge is not that
-// reference.
-func (r drawRoute) spelled(ref string) string {
-	if ref == "" || ref == r.label {
-		return r.spelling
-	}
-	return fmt.Sprintf("%s with {%s}", r.spelling, ref)
-}

env.go

@@ -0,0 +1,113 @@
+package fejkdata
+
+import (
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// readValue is what one read drew: its text, and whether it landed on a null.
+type readValue struct {
+	text string
+	null bool
+}
+
+// frameStack is the frames of the name scopes rendering, innermost last. A render's env points
+// at it, and a repeat iteration's shares it: holding the frames in a renderEnv would move every
+// render's env to the heap.
+type frameStack struct {
+	frames []*pickFrame
+}
+
+func (st *frameStack) push(f *pickFrame) int {
+	mark := len(st.frames)
+	st.frames = append(st.frames, f)
+	return mark
+}
+
+func (st *frameStack) pop(mark int) { st.frames = st.frames[:mark] }
+
+// renderEnv is where a render reads: the frames of the name scopes rendering; and row, the row
+// of the table rendering, which its columns read. It passes by value, and what is read from it
+// reaches a map key, an interface or a func value only as a copy; else env, and with it every
+// render's frames, moves to the heap.
+type renderEnv struct {
+	frames *frameStack
+	row    renderedRow
+	// base is the depth of the frame stack when the read rendering the category entered it,
+	// below which no frame is the category's.
+	base int
+	// pick is the named pick being rendered, nil outside one; pickAt is the key in it of what renders.
+	pick   *namedPick
+	pickAt pickKey
+}
+
+type renderedRow struct {
+	t     *table
+	index int
+}
+
+// at returns the env n renders in, n being the leaf of a path whose rows sit in pins; where n
+// is a table's row or column, the env carries the row pins holds for that table.
+func (env renderEnv) at(n node, pins *pinSet) renderEnv {
+	switch n := n.(type) {
+	case *tableRow:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	case *tableColumn:
+		env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
+	}
+	return env
+}
+
+func (env renderEnv) rowOf(t *table) int {
+	if env.row.t != t {
+		panic(invariant.Broken("a column of %s renders in an env holding no row of it", t.rows.Segment()))
+	}
+	return env.row.index
+}
+
+// enter starts a read landing on n: the read sees no frame opened before it, and gets a frame for
+// each name scope around n, from memo where there is one, so reads sharing memo read one pick of
+// each name. It returns the mark that closes those frames.
+func (env renderEnv) enter(n node, memo *drawMemo) (renderEnv, int) {
+	env = env.entering()
+	for scope := scopeAround(n); scope != nil; scope = scope.up {
+		if len(scope.order) > 0 {
+			env.frames.push(memo.enteredFrame(scope))
+		}
+	}
+	return env, env.base
+}
+
+// entering is env as a read entering a category sees it: no frame opened before it, and no pick.
+func (env renderEnv) entering() renderEnv {
+	env.base, env.pick = len(env.frames.frames), nil
+	return env
+}
+
+// renderFrame opens a fresh frame of t's scope, where t binds names and no read entering the
+// category opened one, returning the mark that closes it, or -1.
+func (env renderEnv) renderFrame(t *template) int {
+	own := t.ownScope()
+	if own == nil || env.frameOf(own) != nil {
+		return -1
+	}
+	return env.frames.push(newPickFrame(own))
+}
+
+// frameOf is the frame of scope rendering since the read entering the category, nil where none is.
+func (env renderEnv) frameOf(scope *nameScope) *pickFrame {
+	for i := len(env.frames.frames) - 1; i >= env.base; i-- {
+		if f := env.frames.frames[i]; f.scope == scope {
+			return f
+		}
+	}
+	return nil
+}
+
+// drawRowOf draws the row a render of t reads: inside the pick's rows where t renders as part of one.
+func (env renderEnv) drawRowOf(s *drawstate.State, t *table) int {
+	if env.pick != nil {
+		return t.rows.DrawIn(s, &env.pick.pins)
+	}
+	return t.rows.Draw(s)
+}

family.go

@@ -1,319 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// tablePin is one table's pinned row.
-type tablePin struct {
-	t   *table
-	row int
-}
-
-// pinned is the row the render pinned for t, if any.
-func (d *draws) pinned(t *table) (int, bool) {
-	for _, p := range d.pins[:d.npins] {
-		if p.t == t {
-			return p.row, true
-		}
-	}
-	r, ok := d.more[t]
-	return r, ok
-}
-
-// mustRow is the row pinned for t, which the walk reaching a column pinned.
-func (d *draws) mustRow(t *table) int {
-	r, ok := d.pinned(t)
-	if !ok {
-		panic(fmt.Sprintf("fejkdata: a column of %s is rendered with no row pinned", t.category))
-	}
-	return r
-}
-
-// pin pins row r of t, and the rows of t's ancestors it links to.
-func (d *draws) pin(t *table, r int) {
-	for {
-		if _, done := d.pinned(t); done {
-			return
-		}
-		if d.npins < len(d.pins) {
-			d.pins[d.npins] = tablePin{t, r}
-			d.npins++
-		} else {
-			if d.more == nil {
-				d.more = map[*table]int{}
-			}
-			d.more[t] = r
-		}
-		if t.parentT == nil {
-			return
-		}
-		t, r = t.parentT, t.parentRow(r)
-	}
-}
-
-// each calls fn for every pinned row, in pin order, the spilled ones by table name.
-func (d *draws) each(fn func(t *table, r int)) {
-	for _, p := range d.pins[:d.npins] {
-		fn(p.t, p.row)
-	}
-	spilled := make([]*table, 0, len(d.more))
-	for t := range d.more {
-		spilled = append(spilled, t)
-	}
-	sort.Slice(spilled, func(i, j int) bool { return spilled[i].category < spilled[j].category })
-	for _, t := range spilled {
-		fn(t, d.more[t])
-	}
-}
-
-// rowOf is the render's row of t: the one pinned, else one drawn inside the
-// nearest pinned ancestor — its parent drawn inside that first where the ancestor
-// is further up — or over the whole table, and pinned with its ancestors.
-func (d *draws) rowOf(t *table) int {
-	if r, ok := d.pinned(t); ok {
-		return r
-	}
-	r := -1
-	for a := t.parentT; a != nil && r < 0; a = a.parentT {
-		if _, ok := d.pinned(a); !ok {
-			continue
-		}
-		if t.parentT != a {
-			d.rowOf(t.parentT)
-		}
-		pr, _ := d.pinned(t.parentT)
-		r = t.drawUnder(d.s, pr)
-	}
-	if r < 0 {
-		r = t.draw(d.s)
-	}
-	d.pin(t, r)
-	return r
-}
-
-// pinRow pins row r of t where it agrees with the rows pinned before it.
-func (d *draws) pinRow(t *table, r int) error {
-	if pr, ok := d.pinned(t); ok && pr != r {
-		return fmt.Errorf("%s and %s are two rows of %s", t.selectorSpelling(pr), t.selectorSpelling(r), t.path)
-	}
-	for a := t.parentT; a != nil; a = a.parentT {
-		if pa, ok := d.pinned(a); ok && !t.under(r, a, pa) {
-			return fmt.Errorf("%s is not inside %s", t.selectorSpelling(r), a.selectorSpelling(pa))
-		}
-	}
-	d.pin(t, r)
-	return nil
-}
-
-// selectRow pins the row a selector names.
-func (d *draws) selectRow(t *table, sel string) error {
-	r, err := t.find(sel, d)
-	if err != nil {
-		return err
-	}
-	return d.pinRow(t, r)
-}
-
-// inside keeps the rows of t that sit inside every pinned ancestor.
-func (d *draws) inside(t *table, rows []int) []int {
-	for a := t.parentT; a != nil; a = a.parentT {
-		pa, ok := d.pinned(a)
-		if !ok {
-			continue
-		}
-		var kept []int
-		for _, r := range rows {
-			if t.under(r, a, pa) {
-				kept = append(kept, r)
-			}
-		}
-		rows = kept
-	}
-	return rows
-}
-
-// tableRead is what a reference path reads of a table family: the table its head
-// names, the rows its selectors pin, the tables it draws — those it walks with no
-// row pinned, and their unpinned ancestors — each selector's spelling, and whether
-// it lands on a row rendered whole.
-type tableRead struct {
-	head  *table
-	pins  draws
-	drawn map[*table]bool
-	sels  []tableSel
-	whole bool
-}
-
-// tableSel is one selector on the way: the table it selects a row of, and the path
-// as written up to and including it.
-type tableSel struct {
-	t        *table
-	spelling string
-}
-
-// tableReadOf replays a reference's selectors through the walk a render uses, so
-// two paths pinning one row by different routes compare equal. checkPath proved
-// each selector names a row.
-func tableReadOf(head node, a arm, leaf node) *tableRead {
-	t, isTable := head.(*table)
-	if !isTable {
-		return nil
-	}
-	tr := &tableRead{head: t, drawn: map[*table]bool{}}
-	_, _ = walkPath(t, a.tail, pathWalk{pins: &tr.pins})
-	written := a.name[:len(a.name)-len(joinSegments(a.tail))]
-	cur := t
-	tr.draws(cur)
-	for i, seg := range a.tail {
-		switch d := cur.descendant(seg); {
-		case isSelector(seg):
-			tr.sels = append(tr.sels, tableSel{cur, written + joinSegments(a.tail[:i+1])})
-		case d != nil:
-			cur = d
-			tr.draws(cur)
-		}
-	}
-	_, tr.whole = leaf.(*row)
-	return tr
-}
-
-// draws marks t and its ancestors drawn, up to the first the read pins.
-func (r *tableRead) draws(t *table) {
-	for ; t != nil; t = t.parentT {
-		if _, pinned := r.pins.pinned(t); pinned {
-			return
-		}
-		r.drawn[t] = true
-	}
-}
-
-// joinSegments spells segments as a path: a selector attaches to the name before it.
-func joinSegments(segs []string) string {
-	var b strings.Builder
-	for _, s := range segs {
-		if b.Len() > 0 && !isSelector(s) {
-			b.WriteByte('.')
-		}
-		b.WriteString(s)
-	}
-	return b.String()
-}
-
-// family is the table a chain of parents ends at.
-func (t *table) family() *table {
-	for t.parentT != nil {
-		t = t.parentT
-	}
-	return t
-}
-
-// selected is the selector in r on t, or on the nearest ancestor of t it selects.
-func (r *tableRead) selected(t *table) (tableSel, bool) {
-	for ; t != nil; t = t.parentT {
-		for _, s := range r.sels {
-			if s.t == t {
-				return s, true
-			}
-		}
-	}
-	return tableSel{}, false
-}
-
-// checkFamilies refuses reads of one table family in one draw group that cannot
-// read one consistent draw: a table rendered whole beside a path into the family,
-// a table one read draws that another pins, and two reads pinning different rows.
-// The reads come sorted by group and path, so which pair is reported does not vary.
-func checkFamilies(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[:i] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(o.at, r.at) || o.tr.head.family() != r.tr.head.family() {
-				continue
-			}
-			if err := checkFamilyPair(o, r); err != nil {
-				return err
-			}
-		}
-	}
-	return replayPairs(reads)
-}
-
-// replayPairs replays every two reads of one draw group that can render together into
-// one draws, the earlier read first. A pin conflicts with one earlier pin, never with a
-// combination, so pairs find every conflict a full replay would.
-func replayPairs(reads []pathRead) error {
-	for i, r := range reads {
-		if r.tr == nil {
-			continue
-		}
-		for _, o := range reads[i+1:] {
-			if o.tr == nil || o.at.group != r.at.group || alternatives(r.at, o.at) {
-				continue
-			}
-			var d draws
-			if err := r.tr.replay(&d); err != nil {
-				return conflict(r, err)
-			}
-			if err := o.tr.replay(&d); err != nil {
-				return conflict(o, err)
-			}
-		}
-	}
-	return nil
-}
-
-func conflict(r pathRead, err error) error {
-	return fmt.Errorf("%s: %w; select the same rows in every path into the family, or draw them apart with a drawGroup", r.at.route.spelled(r.a.name), err)
-}
-
-// replay pins the read's rows into d, where they agree with the rows pinned before.
-func (r *tableRead) replay(d *draws) error {
-	var err error
-	r.pins.each(func(t *table, row int) {
-		if err == nil {
-			err = d.pinRow(t, row)
-		}
-	})
-	return err
-}
-
-// checkFamilyPair refuses a table read whole beside a path into its family, and a
-// table one read draws that the other pins, since which token renders first would
-// then decide the row.
-func checkFamilyPair(a, b pathRead) error {
-	for _, pair := range [][2]pathRead{{a, b}, {b, a}} {
-		x, y := pair[0], pair[1]
-		if len(x.a.tail) == 0 && len(y.a.tail) > 0 {
-			return overlapError(x.at.route, x.a.name, y)
-		}
-		if drawn := x.tr.drawnOf(&y.tr.pins); drawn != nil {
-			if s, ok := y.tr.selected(x.tr.head); ok && len(x.tr.sels) == 0 {
-				tail := x.a.tail
-				if s.t != x.tr.head {
-					tail = append([]string{x.tr.head.category}, tail...)
-				}
-				return fmt.Errorf("%s draws %s, which %s selects a row of; write {%s.%s}, or draw them apart with a drawGroup",
-					x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name), s.spelling, joinSegments(tail))
-			}
-			return fmt.Errorf("%s draws %s, which %s selects a row of; select that row in both, or draw them apart with a drawGroup",
-				x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name))
-		}
-	}
-	return nil
-}
-
-// drawnOf is a table the read draws that pins holds a row of, if any.
-func (r *tableRead) drawnOf(pins *draws) *table {
-	var found *table
-	pins.each(func(t *table, _ int) {
-		if found == nil && r.drawn[t] {
-			found = t
-		}
-	})
-	return found
-}

fejkdata.go

@@ -1,81 +1,51 @@
-// Package fejkdata generates fake data from recursive JSON templates.
-//
-// Data lives in JSON, not in Go. A generator starts from the shipped data set and
-// layers any directories you add; folders and files become a dot-path namespace,
-// then generate values by path:
-//
-//	f, _ := fejkdata.New(fejkdata.WithSeed(42))
-//	f.Fake("sv_SE.address")          // "Storgatan 12\n234 56 Göteborg"
-//	f.Fake("sv_SE.address.locality") // "Göteborg"
-//
-// Several sources merge in order, the last winning a name clash, so custom data
-// layers over the built-ins. The JSON template format is documented in the README.
 package fejkdata
 
 import (
-	crand "crypto/rand"
-	"embed"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io/fs"
-	"math/rand/v2"
-	"os"
 	"reflect"
 	"sort"
 	"sync"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
+	"github.com/larvit/fejkdata/internal/drawstate"
 )
 
-//go:embed data
+// MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
-var shippedFS embed.FS
+// any one path; the CLI's --repeat shares it.
-
-// MaxRepeat caps a repeat, and the renders nested repeats multiply to along any
-// path; the CLI's --repeat shares it.
 const MaxRepeat = 1 << 20
 
-var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions
+// Fails to compile where uint is narrower than 64 bits: docs/decisions.md#64-bit-targets-only
+var _ [^uint(0)>>63 - 1]struct{}
 
-// ErrNoData is returned by New when no source is loaded at all.
+// Generator generates fake data from a namespace tree. Create one with [New].
-var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS")
+// It is safe for concurrent use; a sequence is reproducible only when drawn
-
+// from one goroutine.
-// Generator generates fake data from a loaded namespace tree. Create one with [New].
-// It is safe for concurrent use; a seeded sequence is reproducible only when drawn
-// from one goroutine. The compiled tree is immutable after [New], and Fake,
-// NewTemplate and List read it concurrently without a lock.
 type Generator struct {
+	// mu guards drawState, structs and root, which gains an indexed category on the
+	// first call reaching it.
 	mu        sync.Mutex
-	rand       *session
+	drawState *drawstate.State
-	categories map[string]node
 	root      folder // the categories as the node a path walks from, owned here so a walk allocates none
-	set        drawSet // one Fake's draws, owned here so a walk pinning rows keeps them off the heap
-	records    map[node]recordShape
 	structs   map[reflect.Type]structResult
 }
 
-// session is one generator's mutable render state: the seeded rng plus the {seq()}
-// counters.
-type session struct {
-	*rand.Rand
-	counters map[string]uint64
-}
-
-func (s *session) next(key string) uint64 {
-	s.counters[key]++
-	return s.counters[key]
-}
-
 type config struct {
 	seed       uint64
 	seeded     bool
-	shipped bool
+	sources    []datafiles.Source
-	sources []dataSource
+	nilModules error
+	// dataGiven is whether a data option was passed, even WithDataFS(), which loads none.
+	dataGiven bool
 }
 
 // Option configures a [Generator].
 type Option func(*config)
 
-// WithSeed makes output reproducible: two generators with the same seed and data
+// WithSeed seeds the generator, as [New] requires: two generators with the same seed,
-// emit identical sequences.
+// version and data emit identical sequences. For a new seed each run, set
+// seed := rand.Uint64() (math/rand/v2), and log it to replay the run.
 func WithSeed(seed uint64) Option {
 	return func(c *config) { c.seed, c.seeded = seed, true }
 }
@@ -84,175 +54,85 @@ func WithSeed(seed uint64) Option {
 // layer several; the last wins a name clash.
 func WithDataPath(dir string) Option {
 	return func(c *config) {
-		c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir})
+		c.sources, c.dataGiven = append(c.sources, datafiles.Dir(dir)), true
 	}
 }
 
-// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own.
+// WithDataFS layers modules, each a data tree held in an [fs.FS], over what is loaded
-func WithDataFS(fsys fs.FS) Option {
+// before them, in order. A module may be a data package's FS, such as one of
-	return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) }
+// github.com/larvit/fejkdata/data's Modules, or an embed.FS of your own. WithDataFS()
+// loads no data.
+func WithDataFS(modules ...fs.FS) Option {
+	return func(c *config) {
+		for i, fsys := range modules {
+			if fsys == nil {
+				c.nilModules = errors.Join(c.nilModules, fmt.Errorf("fejkdata: WithDataFS module %d is nil", i+1))
+				continue
+			}
+			c.sources = append(c.sources, datafiles.FS(fsys))
+		}
+		c.dataGiven = true
+	}
 }
 
-// WithoutShippedData leaves the shipped data set out, so only the sources given
+var errNoSeed = errors.New(`fejkdata: no seed: add fejkdata.WithSeed(n), where the same n repeats a run's output; for a new seed each run, set n := rand.Uint64() (import "math/rand/v2") and log n to replay the run`)
-// with [WithDataPath] and [WithDataFS] load.
-func WithoutShippedData() Option {
-	return func(c *config) { c.shipped = false }
-}
 
-// New builds a generator from the shipped data set and the options' sources, merged
+var errNoData = errors.New(`fejkdata: no data: add fejkdata.WithDataFS(data.Modules()...) (import "github.com/larvit/fejkdata/data") for all shipped data, WithDataFS/WithDataPath for your own, or WithDataFS() for none`)
-// in order with the last winning a name clash. Each JSON file becomes a category
+
-// named after the file (address.json -> "address") and each subdirectory a
+// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
-// namespace segment. It errors on a missing directory, invalid JSON, invalid data,
+// of an indexed source.
-// or no data at all.
+var ErrLoad = errors.New("data fails to load")
+
+// loadError is a load's error, unchanged in its text, matching ErrLoad.
+type loadError struct{ error }
+
+func (e loadError) Unwrap() error      { return e.error }
+func (loadError) Is(target error) bool { return target == ErrLoad }
+
+// New builds a generator from the options' sources, merged in order with the last
+// winning a name clash; it fails where no [WithSeed] gives the seed, or no [WithDataFS]
+// or [WithDataPath] names the data to load. Each JSON file becomes a category named
+// after the file (address.json -> "address") and each subdirectory a namespace segment.
+// It errors on a missing directory, invalid JSON or invalid data. Given only
+// WithDataFS() with no modules, New loads no data, and the generator renders templates
+// that read none. A source whose .fejkdata.json carries an index, as each shipped
+// module's does, loads each category on the first call reaching it. Data failing to
+// load, in New or on that first call, gives an error matching [ErrLoad].
 func New(opts ...Option) (*Generator, error) {
-	c := config{shipped: true}
+	var c config
 	for _, opt := range opts {
 		opt(&c)
 	}
-	var sources []dataSource
+	if err := c.misuse(); err != nil {
-	if c.shipped {
+		return nil, err
-		sources = append(sources, dataSource{fsys: shippedFS, root: "data"})
 	}
-	sources = append(sources, c.sources...)
+	root, err := loadSources(c.sources)
-	if len(sources) == 0 {
-		return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
-	}
-	cats, err := loadData(sources)
 	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
+		return nil, fmt.Errorf("fejkdata: %w", loadError{err})
 	}
-	rng, err := newRand(c.seed, c.seeded)
+	return &Generator{drawState: drawstate.New(c.seed), root: root}, nil
-	if err != nil {
-		return nil, fmt.Errorf("fejkdata: %w", err)
-	}
-	return &Generator{rand: rng, categories: cats}, nil
 }
 
-// List returns the sorted dotted paths Fake can render: every category, the dotted
+// List returns the sorted dotted paths Fake renders: each category and every field,
-// fields within a template, and folder segments. A choice consumes no segment, so a
+// column and linked table below it, a direct descent at a time, listing a level
-// path continues through one only where every variant carries it, which is the
+// carrying a repeat but nothing below it. A choice consumes no segment, so a path
-// rule Fake applies too: List is the set of paths Fake accepts.
+// continues through one only where every variant carries it.
 func (f *Generator) List() []string {
-	var out []string
+	f.mu.Lock()
-	for _, name := range sortedNames(f.categories) {
+	defer f.mu.Unlock()
-		for _, p := range paths(f.categories[name]) {
+	out := paths(&f.root, false)
-			out = append(out, join(name, p))
-		}
-	}
 	sort.Strings(out)
 	return out
 }
 
-// paths lists the dot paths addressable from n, relative to it, where "" is n
+// misuse names every mistake in the options, before any data loads.
-// itself. A folder has no value of its own, so it contributes only its children's.
+func (c config) misuse() error {
-func paths(n node) []string {
+	var err error
-	switch n := n.(type) {
+	if !c.dataGiven {
-	case *folder:
+		err = errNoData
-		var out []string
-		for _, name := range sortedNames(n.children) {
-			for _, p := range paths(n.children[name]) {
-				out = append(out, join(name, p))
 	}
+	if !c.seeded {
+		err = errors.Join(err, errNoSeed)
 	}
-		return out
+	return errors.Join(err, c.nilModules)
-	case *template:
-		out := []string{""}
-		for _, name := range sortedNames(n.fields) {
-			if isRef(name) { // a binding, not a path segment
-				continue
-			}
-			for _, p := range paths(n.fields[name]) {
-				out = append(out, join(name, p))
-			}
-		}
-		return out
-	case *null:
-		return []string{""}
-	case *table:
-		return tablePaths(n)
-	case *column, *row:
-		return []string{""}
-	case *choice:
-		out := []string{""}
-		for p := range n.shared {
-			out = append(out, p)
-		}
-		return out
-	}
-	return nil
-}
-
-// tablePaths is a table's columns, then each table linked to it under its name: the
-// direct descents, a step at a time.
-func tablePaths(t *table) []string {
-	out := append([]string{""}, t.columns...)
-	sort.Strings(out[1:])
-	children := make([]string, 0, len(t.children))
-	for name := range t.children {
-		children = append(children, name)
-	}
-	sort.Strings(children)
-	for _, name := range children {
-		for _, p := range paths(t.children[name]) {
-			out = append(out, join(name, p))
-		}
-	}
-	return out
-}
-
-// sharedPaths is the sub-paths every item carries — the only ones a path may step
-// through a choice to reach. It intersects, bailing as soon as the set
-// is empty, which is immediate for a choice of plain strings.
-func sharedPaths(items []node) map[string]bool {
-	shared := subPaths(items[0])
-	for _, it := range items[1:] {
-		if len(shared) == 0 {
-			return nil
-		}
-		next := subPaths(it)
-		for p := range shared {
-			if !next[p] {
-				delete(shared, p)
-			}
-		}
-	}
-	return shared
-}
-
-// subPaths is paths(n) as a set, without the empty path that means n itself.
-func subPaths(n node) map[string]bool {
-	out := map[string]bool{}
-	for _, p := range paths(n) {
-		if p != "" {
-			out[p] = true
-		}
-	}
-	return out
-}
-
-func join(prefix, name string) string {
-	switch {
-	case prefix == "":
-		return name
-	case name == "":
-		return prefix
-	}
-	return prefix + "." + name
-}
-
-// randomBytes seeds an unseeded generator.
-var randomBytes = crand.Read
-
-func newRand(seed uint64, seeded bool) (*session, error) {
-	var r *rand.Rand
-	if seeded {
-		r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
-	} else {
-		var b [16]byte
-		if _, err := randomBytes(b[:]); err != nil {
-			return nil, fmt.Errorf("seeding from crypto/rand: %w", err)
-		}
-		r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:])))
-	}
-	return &session{Rand: r, counters: map[string]uint64{}}, nil
 }

fejkdata_test.go

@@ -3,11 +3,18 @@ package fejkdata
 import (
 	"errors"
 	"io/fs"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
-	"testing/fstest"
+
+	"github.com/larvit/fejkdata/data"
+	"github.com/larvit/fejkdata/internal/drawstate"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
+func withShipped() Option { return WithDataFS(data.Modules()...) }
+
 // newGenerator creates a generator over a single data directory, failing on
 // error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
 // on conflicts).
@@ -18,17 +25,28 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator {
 
 func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
 	t.Helper()
-	all := []Option{WithoutShippedData()}
+	var all []Option
 	for _, dir := range dirs {
 		all = append(all, WithDataPath(dir))
 	}
-	f, err := New(append(all, opts...)...)
+	f, err := New(append(append(all, WithSeed(1)), opts...)...)
 	if err != nil {
 		t.Fatalf("New(%q): %v", dirs, err)
 	}
 	return f
 }
 
+// newBareModule loads the shipped module in data/<module> without its manifest, so its
+// categories sit at the root, failing on error.
+func newBareModule(t *testing.T, module string, opts ...Option) *Generator {
+	t.Helper()
+	f, err := New(append([]Option{WithDataFS(withoutManifest{os.DirFS(filepath.Join("data", module))}), WithSeed(1)}, opts...)...)
+	if err != nil {
+		t.Fatalf("New(%s without its manifest): %v", module, err)
+	}
+	return f
+}
+
 // fake generates a value, failing the test on error.
 func fake(t *testing.T, f *Generator, path string) string {
 	t.Helper()
@@ -40,31 +58,32 @@ func fake(t *testing.T, f *Generator, path string) string {
 }
 
 func TestNewMissingDirectory(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath("data/de_DE"))
+	_, err := New(WithSeed(1), WithDataPath("data/de_DE"))
 	if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
 		t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
 	}
 }
 
 func TestNewRejectsAnEmptyDataPath(t *testing.T) {
-	_, err := New(WithoutShippedData(), WithDataPath(""))
+	_, err := New(WithSeed(1), WithDataPath(""))
 	if err == nil || !strings.Contains(err.Error(), "empty") {
 		t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
 	}
 }
 
-func TestNewReportsAnEntropyFailure(t *testing.T) {
+func TestNewWithoutASeedFailsNamingWithSeed(t *testing.T) {
-	saved := randomBytes
+	for name, opts := range map[string][]Option{"no option": nil, "only data": {WithDataFS()}} {
-	randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") }
+		if _, err := New(opts...); err == nil || !strings.Contains(err.Error(), "WithSeed(") {
-	defer func() { randomBytes = saved }()
+			t.Errorf("New(%s) = %v, want it refused, naming WithSeed", name, err)
-	_, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}}))
+		}
-	if err == nil || !strings.Contains(err.Error(), "no entropy") {
+	}
-		t.Fatalf("New() without entropy = %v, want the failure reported", err)
+	if _, err := New(WithDataFS(nil)); err == nil || !strings.Contains(err.Error(), "WithSeed(") || !strings.Contains(err.Error(), "module 1 is nil") {
+		t.Errorf("New(WithDataFS(nil)) = %v, want both the missing seed and the nil module named", err)
 	}
 }
 
 func TestWithSeedIsDeterministic(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(42)), newGenerator(t, "data", WithSeed(42))
+	a, b := shipped(t, WithSeed(42)), shipped(t, WithSeed(42))
 	for i := 0; i < 50; i++ {
 		if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
 			t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
@@ -73,7 +92,7 @@ func TestWithSeedIsDeterministic(t *testing.T) {
 }
 
 func TestDifferentSeedsDiffer(t *testing.T) {
-	a, b := newGenerator(t, "data", WithSeed(1)), newGenerator(t, "data", WithSeed(2))
+	a, b := shipped(t, WithSeed(1)), shipped(t, WithSeed(2))
 	for i := 0; i < 50; i++ {
 		if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
 			return
@@ -92,3 +111,43 @@ func fakeTemplate(t *testing.T, f *Generator, s string) string {
 	}
 	return got
 }
+
+// engine builds a seeded generator with no loaded categories, for rendering tests.
+func engine(seed uint64) *Generator {
+	return &Generator{drawState: drawstate.New(seed)}
+}
+
+// parse unmarshals a JSON template fragment into its dynamic form.
+func parse(t *testing.T, s string) jsonvalue.Value {
+	t.Helper()
+	v, err := jsonvalue.Decode([]byte(s))
+	if err != nil {
+		t.Fatalf("parse %q: %v", s, err)
+	}
+	return v
+}
+
+// resolved compiles a JSON fragment and resolves it against no data.
+func resolved(t *testing.T, s string) (node, error) {
+	t.Helper()
+	n, err := compile(parse(t, s))
+	if err != nil {
+		return nil, err
+	}
+	return n, resolveInlineTemplates(inlineNodes(n, "template"), nil)
+}
+
+// compiled parses, compiles and resolves a JSON fragment into a node.
+func compiled(t *testing.T, s string) node {
+	t.Helper()
+	n, err := resolved(t, s)
+	if err != nil {
+		t.Fatalf("compile %q: %v", s, err)
+	}
+	return n
+}
+
+func mustRender(t *testing.T, f *Generator, s string) string {
+	t.Helper()
+	return renderOnce(f.drawState, compiled(t, s))
+}

format.go

@@ -0,0 +1,205 @@
+package fejkdata
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// checkFunc validates a call at compile time: a known builtin, its args, and, for calc, the
+// operands held in fields.
+func checkFunc(tok grammar.Token, fields map[string]node) error {
+	err := builtinfunc.Check(tok.Fn, tok.Args)
+	if err == nil && tok.Fn == builtinfunc.CalcName {
+		err = checkCalcFields(tok.Args, fields)
+	}
+	if err != nil {
+		return fmt.Errorf("token {%s}: %w", tok.Body, err)
+	}
+	return nil
+}
+
+func parseChecked(format string, fields map[string]node) ([]grammar.Token, []unboundRead, error) {
+	toks, err := grammar.ParseFormat(format)
+	if err != nil {
+		return nil, nil, err
+	}
+	unbound, err := checkTokens(toks, fields)
+	return toks, unbound, err
+}
+
+// unboundRead is a token reading a head no field holds, which only a name bound around the
+// template can answer; err is the refusal where none does, and body is the token's.
+type unboundRead struct {
+	head, body string
+	err        error
+}
+
+// checkTokens proves every token's grammar, functions and field paths, and returns the reads
+// whose head no field holds, for bindNames to look up among the names.
+func checkTokens(toks []grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	var unbound []unboundRead
+	for _, t := range toks {
+		u, err := checkToken(t, fields)
+		if err != nil {
+			return nil, err
+		}
+		unbound = append(unbound, u...)
+	}
+	return unbound, nil
+}
+
+func checkToken(t grammar.Token, fields map[string]node) ([]unboundRead, error) {
+	switch t.Kind {
+	case grammar.BuiltinCall:
+		if err := checkFunc(t, fields); err != nil {
+			return nil, err
+		}
+		return checkReads(t, fields, true)
+	case grammar.NameBind:
+		return nil, checkBind(t, fields)
+	case grammar.PathRead:
+		return checkReads(t, fields, false)
+	}
+	return nil, nil
+}
+
+// checkReads proves each name t reads, an arm or an operand, is a reference or a path into a
+// field, returning those whose head no field holds.
+func checkReads(t grammar.Token, fields map[string]node, operands bool) ([]unboundRead, error) {
+	var unbound []unboundRead
+	names := tokenReads(t)
+	for _, name := range names {
+		if grammar.IsRef(name) {
+			if _, _, err := grammar.RefShape(name); err != nil {
+				return nil, fmt.Errorf("token {%s}: %w", t.Body, err)
+			}
+			continue // resolveRefs checks its target once the tree is assembled
+		}
+		missing, err := checkArm(name, fields, !operands && len(names) == 1)
+		if err == nil {
+			continue
+		}
+		err = fmt.Errorf("token {%s}: %w", t.Body, err)
+		if !missing {
+			return nil, err
+		}
+		unbound = append(unbound, unboundRead{head: splitArm(name, nil).head, body: t.Body, err: err})
+	}
+	return unbound, nil
+}
+
+// checkBind proves a {x as name} token is spelled once, binds a reference or a path into a
+// field, and names a valid name that is no option.
+func checkBind(t grammar.Token, fields map[string]node) error {
+	ref, name := t.BoundRef, t.Bound
+	if trimmed := strings.TrimSpace(ref) + grammar.AsWord + strings.TrimSpace(name); trimmed != t.Body {
+		return fmt.Errorf("token {%s}: write {%s}", t.Body, trimmed)
+	}
+	if err := checkBound(ref, fields); err != nil {
+		return fmt.Errorf("token {%s}: %w", t.Body, err)
+	}
+	if name == "" {
+		return fmt.Errorf("token {%s}: a binding names nothing; write {%s as n}", t.Body, ref)
+	}
+	if isOption(name) {
+		return fmt.Errorf("token {%s}: %q is an option and can never be a name; rename it", t.Body, name)
+	}
+	if err := grammar.CheckIdentifier(name); err != nil {
+		return fmt.Errorf("token {%s}: name %w", t.Body, err)
+	}
+	return nil
+}
+
+// checkBound proves what a binding binds is a reference or a path into a field of its binder.
+func checkBound(ref string, fields map[string]node) error {
+	if grammar.IsRef(ref) {
+		_, _, err := grammar.RefShape(ref)
+		return err
+	}
+	missing, err := checkArm(ref, fields, false)
+	if missing {
+		return fmt.Errorf("%w; a binding names a field, or a reference, which starts with /, . or ..", err)
+	}
+	return err
+}
+
+// checkArm validates one sibling name or path against a template's fields.
+// wholeToken says the name is the token's entire body, so {/name} would render
+// the same value and can be offered as the reference spelling. missing says no
+// field holds the head, which a name may yet answer.
+func checkArm(name string, fields map[string]node, wholeToken bool) (missing bool, err error) {
+	a := splitArm(name, nil)
+	if err := checkSegments(a); err != nil {
+		return false, err
+	}
+	field, ok := fields[a.head]
+	if !ok {
+		if a.head == "" {
+			return false, fmt.Errorf("a name is never empty, so this token can name no field")
+		}
+		if isOption(a.head) {
+			return false, fmt.Errorf("%q is an option and can never be a field", a.head)
+		}
+		if len(fields) == 0 {
+			hint := ""
+			if wholeToken && hintableRef(name) {
+				hint = fmt.Sprintf(" — write {/%s} to reference the data", name)
+			}
+			return true, fmt.Errorf("no field %q; a token names a sibling field, and this template has none%s", a.head, hint)
+		}
+		return true, fmt.Errorf("no field %q", a.head)
+	}
+	if err := provePath(field, a.tail, a.head, 0); err != nil {
+		return false, fmt.Errorf("field %q: %w", a.head, err)
+	}
+	return false, nil
+}
+
+// hintableRef reports whether {/name} is a reference the grammar accepts, so the
+// hint never names a spelling that fails too.
+func hintableRef(name string) bool {
+	return grammar.CheckPathIdentifiers(name) == nil
+}
+
+// tokenReads lists the names t reads: a read's arms, or the operands a call's builtin reads.
+func tokenReads(t grammar.Token) []string {
+	if t.Kind == grammar.BuiltinCall {
+		return builtinfunc.Operands(t.Fn, t.Args)
+	}
+	return t.Arms
+}
+
+// formatOps is a compiled format: its ops, and the size of its literal text.
+type formatOps struct {
+	ops  []op
+	grow int
+}
+
+// compileFormat compiles t's parsed format against its resolved references and its names'
+// targets; a {ref as n} token compiles to no op. checkTokens proved every token valid.
+func compileFormat(t *template, targets map[*nameBinding]nameTarget) (formatOps, error) {
+	var c formatOps
+	for _, tok := range t.tokens {
+		o := op{Token: tok}
+		var err error
+		switch tok.Kind {
+		case grammar.LiteralRun:
+			c.grow += len(tok.Lit)
+		case grammar.BuiltinCall:
+			o.call = builtinfunc.Prep(tok.Fn, tok.Args)
+			o.operands, err = t.compileArms(builtinfunc.Operands(tok.Fn, tok.Args), targets)
+		case grammar.PathRead:
+			o.arms, err = t.compileArms(tok.Arms, targets)
+		case grammar.NameBind:
+			continue
+		}
+		if err != nil {
+			return c, fmt.Errorf("token {%s}: %w", tok.Body, err)
+		}
+		c.ops = append(c.ops, o)
+	}
+	return c, nil
+}

format_stability_test.go

@@ -0,0 +1,54 @@
+package fejkdata
+
+import "testing"
+
+// TestSeededOutputIsStable pins the exact bytes a seeded generator emits for each
+// piece of the token grammar, so a change to how a format is scanned or rendered
+// cannot quietly shift the rng stream that reproducibility depends on.
+func TestSeededOutputIsStable(t *testing.T) {
+	dir := writeFiles(t, map[string]string{
+		"alt.json":          `{"format":"{a|b}","a":"A","b":"B"}`,
+		"calc.json":         `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"classes.json":      `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`,
+		"escapes.json":      `{"format":"01Aa#{x}","x":"!"}`,
+		"funcs.json":        `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`,
+		"municipality.json": `{"format":"{name}","rows":"municipality.tsv","key":"code","name":"name","parent":"region"}`,
+		"municipality.tsv":  "code\tname\tregion\n0180\tStockholm\t01\n0184\tSolna\t01\n1280\tMalmö\t12\n1281\tLund\t12\n",
+		"named.json":        `{"format":"{p as q}{q.a}{q.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"nested.json":       `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`,
+		"paths.json":        `{"format":"{p.a}{p.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
+		"ref.json":          `"see {/alt}"`,
+		"refpath.json":      `"{/region.municipality.name} in {/region.name}"`,
+		"region.json":       `{"format":"{name}","rows":"region.tsv","key":"code","name":"name","weight":"people"}`,
+		"region.tsv":        "code\tname\tpeople\n01\tStockholms län\t3\n12\tSkåne län\t1\n",
+		"repeat.json":       `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`,
+		"sums.json":         `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`,
+		"weights.json":      `[{"format":"big","weight":9},"tiny"]`,
+	})
+	want := map[string][]string{
+		"alt":                            {"A", "A", "B", "A"},
+		"calc":                           {"100.00 x 3 = 700.00", "5.00 x 7 = 35.00", "5.00 x 3 = 35.00", "100.00 x 2 = 139.93"},
+		"classes":                        {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"},
+		"escapes":                        {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"},
+		"funcs":                          {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"},
+		"named":                          {"ab", "ab", "AB", "AB"},
+		"nested":                         {"i-73", "i-23", "i-67", "i-85"},
+		"paths":                          {"aB", "aB", "ab", "AB"},
+		"paths.p.a":                      {"A", "a", "a", "A"},
+		"ref":                            {"see A", "see A", "see A", "see B"},
+		"refpath":                        {"Solna in Skåne län", "Solna in Stockholms län", "Solna in Stockholms län", "Stockholm in Stockholms län"},
+		"region.municipality[0184].name": {"Solna", "Solna", "Solna", "Solna"},
+		"region[12].municipality.name":   {"Malmö", "Malmö", "Lund", "Malmö"},
+		"repeat":                         {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"},
+		"sums":                           {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"},
+		"weights":                        {"big", "big", "big", "big"},
+	}
+	for path := range want {
+		f := newGenerator(t, dir, WithSeed(42))
+		for i, expect := range want[path] {
+			if got := fake(t, f, path); got != expect {
+				t.Errorf("%s draw %d = %q, want %q", path, i, got, expect)
+			}
+		}
+	}
+}

format_test.go

@@ -0,0 +1,132 @@
+package fejkdata
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func TestStringIsAFormat(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`"Malmö"`:                  "Malmö",
+		`"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0",
+		`"{{x}}"`:                  "{x}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) {
+		t.Errorf(`render("{digits(3)}") = %q, want three digits`, got)
+	}
+	if _, err := resolved(t, `"{x}"`); err == nil || !strings.Contains(err.Error(), `no field "x"`) {
+		t.Errorf(`compile("{x}") = %v, want a no-field error`, err)
+	}
+}
+
+func TestTextIsLiteral(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" {
+		t.Fatalf("text = %q, want it verbatim", got)
+	}
+}
+
+func TestBraceEscapes(t *testing.T) {
+	f := engine(1)
+	for src, want := range map[string]string{
+		`{"format":"{{","x":"v"}`:         "{",
+		`{"format":"}}","x":"v"}`:         "}",
+		`{"format":"{{x}}","x":"v"}`:      "{x}",
+		`{"format":"{{{x}}}","x":"v"}`:    "{v}",
+		`{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}",
+	} {
+		if got := mustRender(t, f, src); got != want {
+			t.Errorf("render(%s) = %q, want %q", src, got, want)
+		}
+	}
+	for src, want := range map[string]string{
+		`"x}y"`:                      "}}",
+		`"}"`:                        "}}",
+		`{"format":"{a{b}","a":"Q"}`: "'{'",
+		`"{x"`:                       "unterminated",
+	} {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want)
+		}
+	}
+}
+
+func TestTokenSubstitution(t *testing.T) {
+	if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" {
+		t.Fatalf("token = %q, want Eriksson", got)
+	}
+}
+
+func TestAlternationDrawsOneField(t *testing.T) {
+	f := engine(3)
+	seen := map[string]bool{}
+	for i := 0; i < 100; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || len(seen) != 2 {
+		t.Fatalf("alternation produced %v, want both A and B", seen)
+	}
+}
+
+func TestAlternationThreeWay(t *testing.T) {
+	f := engine(4)
+	seen := map[string]bool{}
+	for i := 0; i < 200; i++ {
+		seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true
+	}
+	if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 {
+		t.Fatalf("3-way alternation produced %v, want A, B and C", seen)
+	}
+}
+
+func TestHashIsLiteral(t *testing.T) {
+	f := engine(1)
+	cases := map[string]string{
+		`{"format":"","x":"v"}`:         "",
+		`{"format":"#","x":"v"}`:        "#",
+		`{"format":"##","x":"v"}`:       "##",
+		`{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a",
+		`{"format":"#{x}","x":"v"}`:     "#v",
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Errorf("render(%s) = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestMultibyteFormat(t *testing.T) {
+	// Scanning is rune-aware: multibyte literals coexist with class chars and
+	// tokens without corrupting indices.
+	got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`)
+	if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) {
+		t.Fatalf("multibyte format = %q", got)
+	}
+}
+
+func TestFormatCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{x"`,                       // unterminated brace
+		`"x}y"`,                      // a lone } must be written }}
+		`{"format":"{a{b}","a":"Q"}`, // a brace inside a token
+		`"{x}"`,                      // a bare string has no fields to name
+		`{"format":"{y}","x":"Q"}`,   // token names a missing field
+		`"{}"`,                       // empty token name
+		`{"format":"{a|}","a":"Q"}`,  // empty alternation segment
+		`"{luhn(}"`,                  // malformed function token
+		`"{lower()}"`,                // wrong arity
+		`"{nope()}"`,                 // unknown function
+		`"{luhn(x)}"`,                // function given args it takes none of
+		`"{int(1)}"`,                 // wrong arity
+		`"{float(1,2)}"`,             // wrong arity
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}

go.mod

@@ -1,3 +1,3 @@
-module gitea.larvit.se/larvit/fejkdata
+module github.com/larvit/fejkdata
 
 go 1.22

graph.go

@@ -2,128 +2,99 @@ package fejkdata
 
 import (
 	"fmt"
-	"sort"
+	"slices"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// walkNodes calls fn once per contained node, passing the dot path that reaches it,
+// eachNode visits n and every node inside it once, and passes fn each node with the label an
-// visiting keys in sorted order so which of several broken nodes gets reported does
+// error uses for it: the dot path reaching it, then ", line N" for a table cell. It never
-// not depend on map iteration.
+// crosses a reference edge — a {/path} reference is skipped — so a single inline node is
-func walkNodes(root map[string]node, fn func(path string, n node) error) error {
+// walked on its own.
-	for _, name := range sortedNames(root) {
+func eachNode(n node, label string, fn func(label string, n node) error) error {
-		if err := eachNode(root[name], name, fn); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// eachNode visits n and every node contained within it once, passing the dot path
-// that reaches each. It never crosses a reference edge — a bound {/path} field is
-// skipped — so a single inline node is walked on its own.
-func eachNode(n node, path string, fn func(path string, n node) error) error {
 	seen := map[node]bool{}
 	var visit func(string, node) error
-	visit = func(path string, m node) error {
+	visit = func(label string, m node) error {
 		if m == nil || seen[m] {
 			return nil
 		}
 		seen[m] = true
-		if err := fn(path, m); err != nil {
+		if err := fn(label, m); err != nil {
 			return err
 		}
 		for _, c := range contained(m) {
-			if err := visit(join(path, c.name), c.node); err != nil {
+			if err := visit(c.labelIn(label), c.node); err != nil {
 				return err
 			}
 		}
 		return nil
 	}
-	return visit(path, n)
+	return visit(label, n)
 }
 
-// namedNode is a contained child and the segment reaching it; a choice's items carry
+// containedNode is a contained child and the segment reaching it; a choice's items carry
 // no segment, matching how a dot path steps over a choice.
-type namedNode struct {
+type containedNode struct {
 	name string
 	node node
+	line int // a table cell's line in its rows file, else 0
 }
 
-func contained(n node) []namedNode {
+func (c containedNode) labelIn(label string) string {
+	if c.line > 0 {
+		return fmt.Sprintf("%s, line %d", label, c.line)
+	}
+	return join(label, c.name)
+}
+
+func contained(n node) []containedNode {
 	switch n := n.(type) {
 	case *folder:
-		return named(n.children)
+		return containedByName(n.children)
 	case *choice:
-		out := make([]namedNode, len(n.items))
+		out := make([]containedNode, len(n.items))
 		for i, it := range n.items {
-			out[i] = namedNode{node: it}
+			out[i] = containedNode{node: it}
 		}
 		return out
 	case *template:
-		return named(n.fields)
+		return containedByName(n.fields)
 	case *table:
-		return append([]namedNode{{node: n.format}}, named(n.fields)...)
+		return append([]containedNode{{node: n.formatTemplate}}, containedByName(n.formatTemplate.fields)...)
-	case *column:
+	case *tableColumn:
-		if len(n.t.tokens) == 0 {
+		if len(n.t.cellTemplates) == 0 {
 			return nil
 		}
-		var out []namedNode
+		var out []containedNode
-		for r := 0; r < n.t.rows(); r++ {
+		for r := 0; r < n.t.rows.Len(); r++ {
-			if cell := n.t.cellNode(r, n.i); cell != nil {
+			if cell := n.t.cellTemplate(r, n.i); cell != nil {
-				out = append(out, namedNode{node: cell})
+				out = append(out, containedNode{node: cell, line: r + 2})
 			}
 		}
 		return out
+	case *nullItem, *tableRow:
+		return nil
 	default:
-		return nil
+		panic(invariant.Broken("contained has no case for node %T", n))
 	}
 }
 
-// named skips a bound {/path} key: it is a render edge, not containment, so using
+func containedByName(m map[string]node) []containedNode {
-// it as a path segment would report a node under a path that does not reach it. Only
+	out := make([]containedNode, 0, len(m))
-// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a
-// folder's children never carry the prefix — so this one skip serves both.
-func named(m map[string]node) []namedNode {
-	out := make([]namedNode, 0, len(m))
 	for _, name := range sortedNames(m) {
-		if isRef(name) {
+		out = append(out, containedNode{name: name, node: m[name]})
-			continue
-		}
-		out = append(out, namedNode{name: name, node: m[name]})
 	}
 	return out
 }
 
-func sortedNames(m map[string]node) []string {
-	names := make([]string, 0, len(m))
-	for name := range m {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return names
-}
-
 // renderEdge is a child a node renders into, labelled by what reaches it (a field
-// name, reference, or choice index) for a readable cycle report. operand names
+// name, reference, or choice index) for a readable cycle report.
-// the builtin when the label is its operand rather than a token, so an error can
-// name it the way the author wrote it.
 type renderEdge struct {
 	to    node
 	label string
-	operand string
 }
 
-// reached names an edge as the author spelled it, the vocabulary boundReaders uses
+// renderEdges lists the children rendering n recurses into, mirroring render.
-// for the sibling fence.
-func (e renderEdge) reached() string {
-	if e.operand != "" {
-		return fmt.Sprintf("%s operand %q", e.operand, e.label)
-	}
-	return "{" + e.label + "}"
-}
-
-// renderEdges lists the children rendering n recurses into, mirroring expand: a
-// choice's items, and a template's field/reference tokens plus its operands. A
-// folder renders nothing, so it has no edges.
 func renderEdges(n node) []renderEdge {
 	switch n := n.(type) {
 	case *choice:
@@ -134,120 +105,40 @@ func renderEdges(n node) []renderEdge {
 		return es
 	case *template:
 		var es []renderEdge
-		add := func(name, operand string) {
+		for _, o := range n.compiled.ops {
-			a := splitArm(name, n.refs)
+			for _, a := range slices.Concat(o.operands, o.arms) {
-			c, ok := n.fields[a.key]
+				for _, leaf := range a.leaves {
-			if !ok {
+					es = append(es, renderEdge{leaf, a.spelling})
-				return
-			}
-			for _, leaf := range pathLeaves(c, a.tail) {
-				es = append(es, renderEdge{leaf, name, operand})
 				}
 			}
-		_ = eachToken(n.format, func(t ftoken) error {
-			if t.kind != 'b' {
-				return nil
 		}
-			if fn, _, isFunc := funcCall(t.body); isFunc {
-				for _, operand := range tokenOperands(t.body) {
-					add(operand, fn)
-				}
-				return nil
-			}
-			for _, name := range splitOutside(t.body, '|') {
-				add(name, "")
-			}
-			return nil
-		})
 		return es
 	case *table:
-		return []renderEdge{{to: n.format, label: "format"}}
+		return []renderEdge{{to: n.formatTemplate, label: "format"}}
-	case *row:
+	case *tableRow:
-		return []renderEdge{{to: n.t.format, label: "format"}}
+		return []renderEdge{{to: n.t.formatTemplate, label: "format"}}
-	case *column:
+	case *tableColumn:
 		var es []renderEdge
 		for _, c := range contained(n) {
-			es = append(es, renderEdge{to: c.node, label: n.t.columns[n.i]})
+			es = append(es, renderEdge{to: c.node, label: n.name()})
 		}
 		return es
-	default:
+	case *folder, *nullItem:
 		return nil
+	default:
+		panic(invariant.Broken("renderEdges has no case for node %T", n))
 	}
 }
 
-// pathLeaves lists what a token's dotted tail renders: a choice on the way
+type renderCounts map[node]int
-// contributes every variant, since any of them may be the one drawn. checkPath has
-// already proved the tail resolves in every variant.
-func pathLeaves(n node, tail []string) []node {
-	var out []node
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil },
-		leaf:   func(n node) error { out = append(out, n); return nil },
-	})
-	return out
-}
 
-// nodeScope is the set of nodes one validation pass covers: a whole loaded tree,
+func (m renderCounts) renderCount(n node) int {
-// or a single inline node.
-type nodeScope func(fn func(path string, n node) error) error
-
-func treeScope(root map[string]node) nodeScope {
-	return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
-}
-
-func inlineScope(n node, label string) nodeScope {
-	return func(fn func(path string, m node) error) error { return eachNode(n, label, fn) }
-}
-
-func checkScope(s nodeScope) error {
-	if err := checkColumns(s); err != nil {
-		return err
-	}
-	return checkRenders(s)
-}
-
-// checkRenders runs the per-node fences over a scope, each over the whole scope
-// before the next, so which of several broken nodes is reported does not depend on
-// the walk. It runs after checkNoCycles, whose guarantee is what lets the walks
-// terminate.
-func checkRenders(s nodeScope) error {
-	mem := reachMemo{}
-	if err := s(func(path string, n node) error { return repeatCheck(path, n, mem) }); err != nil {
-		return err
-	}
-	if err := s(heldCheck); err != nil {
-		return err
-	}
-	fence := &drawCheck{}
-	refs := false
-	if err := s(func(path string, n node) error {
-		if t, ok := n.(*template); ok && len(t.refs) > 0 {
-			refs = true
-		}
-		return fence.checkDrawGroup(path, n)
-	}); err != nil {
-		return err
-	}
-	if refs {
-		if err := s(fence.checkDraws); err != nil {
-			return err
-		}
-		if err := s(fence.checkRecordDraws); err != nil {
-			return err
-		}
-	}
-	return s((&valueProof{}).checkDatatype)
-}
-
-type reachMemo map[node]int
-
-func (m reachMemo) of(n node) int {
 	if r, done := m[n]; done {
 		return r
 	}
 	r := 1
 	for _, e := range renderEdges(n) {
-		if c := m.of(e.to); c > r {
+		if c := m.renderCount(e.to); c > r {
 			r = c
 		}
 	}
@@ -260,9 +151,10 @@ func (m reachMemo) of(n node) int {
 
 // repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
 // path, so nested repeats cannot build what one repeat may not.
-func repeatCheck(path string, n node, mem reachMemo) error {
+// docs/decisions.md#the-repeat-cap-bounds-renders-not-bytes
-	if t, ok := n.(*template); ok && t.repeat > 1 && mem.of(n) > MaxRepeat {
+func repeatCheck(label string, n node, mem renderCounts) error {
-		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, mem.of(n), MaxRepeat)
+	if t, ok := n.(*template); ok && t.repeat > 1 && mem.renderCount(n) > MaxRepeat {
+		return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", label, t.repeat, mem.renderCount(n), MaxRepeat)
 	}
 	return nil
 }
@@ -270,32 +162,42 @@ func repeatCheck(path string, n node, mem reachMemo) error {
 // checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
 // — directly, mutually, or through a chain — never terminates, so it must fail at
 // New rather than stack-overflow at render. It is a depth-first walk of the render
-// graph (renderEdges); grey marks nodes on the current path so a back-edge to one
+// graph (renderEdges). Every node is a root: a field its parent's format never
-// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped.
+// renders is still reachable by dot path, so a cycle in one would otherwise reach
-// Every node is a root: a field its parent's format never renders is still reachable
+// render and be fatal there.
-// by dot path, so a cycle in one would otherwise reach render and be fatal there.
+func checkNoCycles(s nodeSet) error {
-func checkNoCycles(root map[string]node) error {
 	const (
 		grey  = 1
 		black = 2
 	)
+	inSet := map[node]bool{}
+	_ = s(func(_ string, n node) error {
+		inSet[n] = true
+		if t, isTable := n.(*table); isTable {
+			inSet[t.rowNode] = true // a selector lands on it, and contained lists it nowhere
+		}
+		return nil
+	})
 	color := map[node]int{}
-	var visit func(n node, path string) error
+	var visit func(n node, label string) error
-	visit = func(n node, path string) error {
+	visit = func(n node, label string) error {
+		if !inSet[n] {
+			return nil // loaded before these nodes and proven then, so it never reaches back into it
+		}
 		switch color[n] {
 		case grey:
-			return fmt.Errorf("reference cycle: %s", path)
+			return fmt.Errorf("reference cycle: %s", label)
 		case black:
 			return nil
 		}
 		color[n] = grey
 		for _, e := range renderEdges(n) {
-			if err := visit(e.to, path+" -> "+e.label); err != nil {
+			if err := visit(e.to, label+" -> "+e.label); err != nil {
 				return err
 			}
 		}
 		color[n] = black
 		return nil
 	}
-	return walkNodes(root, func(path string, n node) error { return visit(n, path) })
+	return s(func(label string, n node) error { return visit(n, label) })
 }

hold.go

@@ -1,358 +0,0 @@
-package fejkdata
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// heldCheck rejects every route to a held sibling name except the ones that read its
-// draw. An expansion holds one draw of that name; anything else that renders it draws
-// again, and the two disagree. checkNoOverlap settles the spellings within one format
-// (a token, an operand); this settles the rest — a reference, whether it sits in that
-// format or in anything the format renders, however deep. A reference path is held
-// for the whole render instead, which drawCheck fences.
-func heldCheck(path string, n node) error {
-	t, ok := n.(*template)
-	if !ok || len(t.held) == 0 {
-		return nil
-	}
-	readers := boundReaders(t.format, t.bound, t.refs)
-	for _, head := range heldHeads(t) {
-		if _, isPath := t.bound[head]; isPath && isRef(head) {
-			continue
-		}
-		if err := checkHeadHeld(t, head, readers); err != nil {
-			return fmt.Errorf("%s: %w", path, err)
-		}
-	}
-	return nil
-}
-
-// heldHeads lists a template's held names, operand heads first, then paths, each
-// in name order: a level read both ways is reported by the operand's fence, and
-// which overlap is reported does not vary.
-func heldHeads(t *template) []string {
-	heads := make([]string, 0, len(t.held))
-	for head := range t.held {
-		heads = append(heads, head)
-	}
-	sort.Slice(heads, func(i, j int) bool {
-		_, pi := t.bound[heads[i]]
-		_, pj := t.bound[heads[j]]
-		if pi != pj {
-			return !pi
-		}
-		return heads[i] < heads[j]
-	})
-	return heads
-}
-
-// pinned collects what the hold of head answers for: a path pins the levels it
-// passes through and the leaf it lands on, an operand exactly the value its render
-// produces.
-func pinned(t *template, head string, readers []reader) map[node]bool {
-	held := map[node]bool{}
-	if _, isPath := t.bound[head]; !isPath {
-		operandDraw(t.fields[head], held)
-		return held
-	}
-	for _, r := range readers {
-		if a := splitArm(r.name, t.refs); a.key == head {
-			coverPath(t.fields[head], a.tail, held)
-		}
-	}
-	return held
-}
-
-// checkHeadHeld rejects every route to what head's hold pins except the readers
-// holding it. One seen set across the edges: a node that cannot reach the level
-// cannot reach it by another route either, so it is walked once.
-func checkHeadHeld(t *template, head string, readers []reader) error {
-	held := pinned(t, head, readers)
-	if len(held) == 0 {
-		return nil // a fixed head holds nothing to reach
-	}
-	reader, isPath := t.bound[head]
-	seen := map[node]bool{}
-	for _, e := range renderEdges(t) {
-		if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) {
-			continue
-		}
-		if isPath {
-			return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader)
-		}
-		return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head))
-	}
-	return nil
-}
-
-// operandReader names the builtin whose operand holds head.
-func operandReader(t *template, head string) string {
-	fn := ""
-	_ = eachToken(t.format, func(tok ftoken) error {
-		if tok.kind != 'b' || fn != "" {
-			return nil
-		}
-		if name, _, isFunc := funcCall(tok.body); isFunc {
-			for _, operand := range tokenOperands(tok.body) {
-				if splitArm(operand, t.refs).key == head {
-					fn = name
-				}
-			}
-		}
-		return nil
-	})
-	return fn
-}
-
-// coverPath collects what holding one path pins: every choice level the path
-// passes through, whole, and the leaf it renders.
-func coverPath(n node, tail []string, into map[node]bool) {
-	_, _ = walkPath(n, tail, pathWalk{
-		choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil },
-		leaf:   func(n node) error { cover(n, into, false); return nil },
-	})
-}
-
-// cover collects a level and everything contained in it. A fixed string outside a
-// choice is left out — it cannot disagree with itself — but inside one each
-// variant carries its own, so there it counts.
-func cover(n node, into map[node]bool, inChoice bool) {
-	if isFixed(n) && !inChoice {
-		return
-	}
-	into[n] = true
-	_, isChoice := n.(*choice)
-	for _, c := range contained(n) {
-		cover(c.node, into, inChoice || isChoice)
-	}
-}
-
-// operandDraw collects what one held draw of an operand answers for: the operand
-// and what rendering it settles inside itself. The builtin renders its operand
-// whole, so that draw fixes every value the render produced, and a second route to
-// any of them disagrees with it.
-//
-// The walk stops at a reference edge, which is where the operand's own value ends
-// and a shared source begins: two names referencing one category are two draws, the
-// same rule {word} {word} follows.
-func operandDraw(n node, into map[node]bool) {
-	if isFixed(n) {
-		return
-	}
-	if into[n] {
-		return
-	}
-	into[n] = true
-	for _, e := range renderEdges(n) {
-		if isRef(e.label) {
-			continue
-		}
-		operandDraw(e.to, into)
-	}
-}
-
-// isFixed is a string that varies nothing: fixed text with no fields to read into.
-func isFixed(n node) bool {
-	t, ok := n.(*template)
-	return ok && t.fixed && len(t.fields) == 0
-}
-
-// renders reports whether rendering n can reach anything in want, following the
-// same edges expand does. seen keeps a node shared by several routes from being
-// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk
-// ends.
-func renders(n node, want, seen map[node]bool) bool {
-	if want[n] {
-		return true
-	}
-	if seen[n] {
-		return false
-	}
-	seen[n] = true
-	for _, e := range renderEdges(n) {
-		if renders(e.to, want, seen) {
-			return true
-		}
-	}
-	return false
-}
-
-// checkNoOverlap rejects a format that both renders a sibling level and reads a path
-// into it — {p} beside {p.first}, {p.addr} beside {p.addr.city}. The path reads the
-// level's held draw while rendering the level expands it afresh, so their values would
-// disagree. Reads are compared in sorted order, so which pair is reported does not
-// depend on where the tokens sit.
-func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
-	names := boundReaders(format, bound, refs)
-	// Stable over one format-order scan, so two readers of one name (a token and a
-	// calc operand both naming "p") are reported as the format writes them.
-	sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
-	for i, level := range names {
-		for _, path := range names[i+1:] {
-			if strings.HasPrefix(path.path, level.path+".") {
-				return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name)
-			}
-		}
-	}
-	return nil
-}
-
-// reader is one way a format reaches a bound field: as written, by its one
-// spelling, and how to name it.
-type reader struct{ name, path, label string }
-
-// boundReaders lists every way a format reaches a bound sibling field, in the order the
-// format writes them. An operand renders its field, so it names a level exactly
-// as a token does; one scan finds both, which is what puts them in one order.
-func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
-	var names []reader
-	_ = eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if fn, _, isFunc := funcCall(t.body); isFunc {
-			for _, operand := range tokenOperands(t.body) {
-				a := splitArm(operand, refs)
-				if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-					names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
-				}
-			}
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
-				names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
-			}
-		}
-		return nil
-	})
-	return names
-}
-
-// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside
-// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
-// error names the single-token spelling.
-func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
-	count := map[string]int{}
-	return eachToken(format, func(t ftoken) error {
-		if t.kind != 'b' {
-			return nil
-		}
-		if _, _, isFunc := funcCall(t.body); isFunc {
-			return nil
-		}
-		for _, a := range splitArms(t.body, refs) {
-			if len(a.tail) > 0 || !c.held[a.key] {
-				continue
-			}
-			if count[a.key]++; count[a.key] > 1 {
-				return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
-			}
-		}
-		return nil
-	})
-}
-
-// draws is what has already been drawn for held names: the variant each was drawn as,
-// so every path under it reads one row, and the draw each read made, by its one
-// spelling, so the same read written twice reads one draw. An expansion keeps one for
-// its sibling names, and a render one per group for its reference paths.
-type draws struct {
-	variant map[string]node
-	value   map[string]draw
-	pins    [8]tablePin // the rows pinned, inline so a render over a country's five-deep geo tree stays off the heap
-	npins   int
-	more    map[*table]int // the rows pinned past the inline eight
-	s       *session       // what draws a row; nil where a walk only proves selectors
-}
-
-// draw is what one read drew: its text, and whether it landed on a null.
-type draw struct {
-	text string
-	null bool
-}
-
-// readField renders one arm of a token. An arm's key is a sibling field or a
-// reference linkRefs bound into fields. A name the expansion holds — a level some
-// token addresses by dotted path, or a field an operand reads — is drawn once and
-// kept, so {place.postal-code} and {place.locality} read one row, either read twice
-// gives one value, and a shown operand is the operand computed. Every other name is
-// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
-// linkRefs prove every step, so the walk cannot fail.
-func readField(s *session, t *template, held *draws, sc drawScope, a arm) draw {
-	if !t.held[a.key] {
-		if len(a.tail) > 0 {
-			panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
-		}
-		return draw{text: render(s, t.fields[a.key], sc)}
-	}
-	d := readScope(s, held, sc, a)
-	if r, done := d.value[a.path]; done {
-		return r
-	}
-	leaf, err := walkPath(t.fields[a.key], a.tail, pathWalk{
-		// Hold the draw at every level passed through, so two paths sharing a
-		// prefix share it.
-		choice: func(c *choice, rest []string) ([]node, error) {
-			key := a.key
-			if consumed := len(a.tail) - len(rest); consumed > 0 {
-				key = a.steps[consumed-1]
-			}
-			n, drew := d.variant[key]
-			if !drew {
-				n = drawn(s, c)
-				if d.variant == nil {
-					d.variant = map[string]node{}
-				}
-				d.variant[key] = n
-			}
-			return []node{n}, nil
-		},
-		pins: d,
-	})
-	if err != nil {
-		panic(fmt.Sprintf("fejkdata: %q: %v; a fence should have refused this at New", a.name, err))
-	}
-	r := renderLeaf(s, leaf, sc)
-	if d.value == nil {
-		d.value = map[string]draw{}
-	}
-	d.value[a.path] = r
-	return r
-}
-
-// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
-// the render's draws for its group, so its draw spans the render; for a sibling, or a
-// reference read whole, held.
-func readScope(s *session, held *draws, sc drawScope, a arm) *draws {
-	if isRef(a.key) && len(a.tail) > 0 {
-		return sc.draws(s)
-	}
-	return held
-}
-
-// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
-// reference alone whose read drew null.
-func renderLeaf(s *session, n node, sc drawScope) draw {
-	n = drawn(s, n)
-	if _, isNull := n.(*null); isNull {
-		return draw{null: true}
-	}
-	r := draw{text: render(s, n, sc)}
-	if t, _ := n.(*template); t != nil && t.readsColumn != nil {
-		r.null = sc.in(t).draws(s).value[t.readsColumn.a.path].null
-	}
-	return r
-}
-
-// drawn resolves a choice to one variant, so a bound head is a concrete node the
-// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not
-// another set to pick from.
-func drawn(s *session, n node) node {
-	for c, ok := n.(*choice); ok; c, ok = n.(*choice) {
-		n = pick(s, c)
-	}
-	return n
-}