Make great documentation goal 1.2, and renumber goals 13 to 16 as 12 to 15 wherever they are cited

.github/workflows/packages.yml

@@ -0,0 +1,162 @@
+name: Packages
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        required: true
+    secrets:
+      TAP_GITHUB_TOKEN:
+        required: true
+      PACKAGES_GPG_KEY:
+        required: true
+      PACKAGES_RSA_KEY:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The released tag to package, such as v0.1.0
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: packages
+  cancel-in-progress: false
+
+jobs:
+  packages:
+    name: Binaries, packages and image for ${{ inputs.tag }}
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+      packages: write
+    outputs:
+      latest: ${{ steps.latest.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.tag }}
+          path: src
+          fetch-depth: 0
+          persist-credentials: false
+      # Check out the packaging config at the workflow's commit, so a tag cut before the
+      # config existed can still be packaged.
+      - uses: actions/checkout@v7
+        with:
+          path: tooling
+          persist-credentials: false
+      - name: The Go version the gate tests
+        id: go
+        run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: false
+      - name: Channels holding one version
+        id: latest
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
+          if [ "$latest" = "$TAG" ]; then
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
+            echo "OLDER_TAG=1" >> "$GITHUB_ENV"
+          fi
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: v2.18.3
+          workdir: src
+          args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      - name: The built binary and the published image print the tag
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
+          test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  dockerhub:
+    name: Docker Hub image for ${{ inputs.tag }}
+    needs: packages
+    runs-on: ubuntu-26.04
+    permissions:
+      packages: read
+    steps:
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Copy the GHCR image to Docker Hub, and check it prints the tag
+        env:
+          LATEST: ${{ needs.packages.outputs.latest }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
+          if [ "$LATEST" = true ]; then
+            tags+=(-t docker.io/larvit/fejkdata:latest)
+          fi
+          docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
+          test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  repos:
+    name: apt, dnf and apk repositories for ${{ inputs.tag }}
+    needs: packages
+    if: needs.packages.outputs.latest == 'true'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: The release's Linux packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
+      - uses: docker/setup-buildx-action@v4
+      - name: Install fejkdata from each repository, then export them
+        env:
+          PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+          PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          build() {
+            docker buildx build -f release-tooling/package-repos.Dockerfile \
+              --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
+              --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
+          }
+          build --target checks --output type=local,dest=checks
+          build --target site --output type=local,dest=site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - id: deploy
+        uses: actions/deploy-pages@v5

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * *'
 
 permissions:
   contents: read
@@ -11,9 +14,10 @@ permissions:
 jobs:
   test:
     name: vet + fmt + tests
-    runs-on: ubuntu-24.04
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -26,11 +30,16 @@ jobs:
           else
             base='${{ github.event.before }}'
           fi
-          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
             echo "code=false" >> "$GITHUB_OUTPUT"
           else
             echo "code=true" >> "$GITHUB_OUTPUT"
           fi
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
+            echo "packaging=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "packaging=true" >> "$GITHUB_OUTPUT"
+          fi
           echo "base=$base" >> "$GITHUB_OUTPUT"
       - name: CHANGELOG entry for data changes
         run: |
@@ -48,22 +57,84 @@ jobs:
       # failure is version-specific.
       - name: Lowest supported Go
         if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
-        run: docker build --build-arg GO_VERSION=1.22.12 .
+        run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
+      - name: Package repositories, signed with throwaway keys
+        if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
+        run: |
+          docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
+          PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
+            docker build -f release-tooling/package-repos.Dockerfile --target checks \
+            --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
 
-  release:
+  vulnerabilities:
-    name: Gitea release from CHANGELOG.md
+    name: Known vulnerabilities in this Go version and the required modules
-    needs: test
+    runs-on: ubuntu-26.04
-    if: github.event_name == 'push'
+    timeout-minutes: 10
-    runs-on: ubuntu-24.04
     permissions:
+      actions: write
       contents: write
+      pull-requests: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - run: python3 release-tooling/publish_release.py
+      - name: Fetch govulncheck's report from vuln.go.dev
+        run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
+      - name: File each one in todo.md
+        id: file
+        run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
+      - name: Propose the filed items
+        if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
         env:
-          GITEA_API_URL: ${{ github.api_url }}
+          GH_TOKEN: ${{ github.token }}
-          GITEA_REPOSITORY: ${{ github.repository }}
+        run: |
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
+            exit 0
+          fi
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git switch -C vulnerabilities
+          git commit -qam "File the known vulnerabilities govulncheck reports"
+          git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
+          gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
+            --body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
+          # A pull request the workflow's token opens starts no workflow.
+          gh workflow run test.yml --ref vulnerabilities
+
+  release:
+    name: Release from CHANGELOG.md
+    needs: [test, vulnerabilities]
+    if: github.event_name == 'push'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    outputs:
+      tag: ${{ steps.publish.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - id: publish
+        run: python3 release-tooling/publish_release.py
+        env:
+          FORGE_API_URL: ${{ github.api_url }}
+          FORGE_REPOSITORY: ${{ github.repository }}
+          FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SHA: ${{ github.sha }}
+
+  packages:
+    needs: release
+    if: needs.release.outputs.tag != ''
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/packages.yml
+    with:
+      tag: ${{ needs.release.outputs.tag }}
+    secrets:
+      TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+      PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

AGENTS.md

@@ -1,11 +1,137 @@
 # Rules
 
-- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check.
+- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
 - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
 - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
 - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
+- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
+  only, `gh pr merge --rebase` where the branch is already on top of main.
+- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
+  its status once when a PR is marked ready, answer a review if one is there, and never
+  wait for one.
+- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
+  nothing else merges — no feature, no category, no data — bar a security fix, a
+  dependency bump, a repair of behaviour the README documents, the infrastructure
+  the round itself runs on, and a chunk the comprehension-floor decision lists as
+  shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
+  findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
+  again, until the score passes. At or above 7.0
+  every pull request scores with the four-seat run and answers it in one run. No merge
+  lowers the last score its own panel recorded: the nine-seat round and the four-seat
+  run are two ratchets, never compared with each other. A lower score blocks the merge
+  even where it may be panel noise: fix what the seats name until the score rises. A PR
+  that only records a run's scores and files what its seats name merges even when a
+  score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
 - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
-- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
+- Prose naming a source states what that source states: read the register's own field
-- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
+  before paraphrasing it, and name the register you read before claiming none
+  publishes a spelling.
 - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
-- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed.
+- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
+
+# Decisions
+
+In [docs/decisions.md](docs/decisions.md):
+
+- GitHub is canonical, and the module path names it
+- The Gitea copy stays, as a pull mirror of GitHub
+- The vocabulary sits below `doc.go`'s package clause, not in the package doc
+- Options and fields share one namespace
+- `{a|b}` stays beside nested choices
+- Flags follow getopt_long
+- A struct tag is a template by its shape
+- The CLI renders exactly the template on its stdin, and adds nothing
+- The CLI reads only the library's public API
+- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+- `FakeTemplate` and `NewTemplate` both stay
+- The shipped data is embedded, not discovered
+- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+- Every `{…}` draws afresh, and only a name keeps a pick
+- Reference sigils follow the filesystem
+- A change to what exists is a major; a minor only adds
+- An error is a contract by what it names, not its bytes
+- A load reports every mistake at once
+- Formats are checked after all data is loaded, so their errors name the category
+- Raising the lowest supported Go is a major
+- The format check runs on the latest Go only
+- The changelog heading is the one spelling of a release; CI cuts the tag
+- A `--data-path` override rebinds every reference to the category it replaces
+- The repeat cap bounds renders, not bytes
+- 64-bit targets only
+- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+- Samples say what they emit, transforms what they do
+- A record is a template seen as columns; a Go struct is the one second schema
+- A struct's records follow Go's field access, and compile on first use
+- A category may reference itself, and a record's fences run at load
+- A record's column set is fixed before the first draw
+- Null is a `null` item, not a rate
+- A typed column holds one value, not composed text
+- A column that only reads one reference or name is the column it reads
+- A typed column's calc is refused unless proven
+- `Column` carries text, not a Go value
+- A unit takes the stem of what it is, and a file the stem of the units it holds
+- One name, one meaning
+- The performance gate asserts allocations, not wall-clock time
+- Rows live in a TSV, the shape in JSON
+- A selector is bracketed, and a dot inside it is literal
+- `parent` names the link column and the table alike
+- After a row, a path names a column or a linked table
+- A selected row is fixed, not drawn
+- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+- A table selects by one key column, a code, never a free-form name
+- A parent row with no child row is a load error
+- A table is a record of string columns
+- The key map is built at load, the rest on first draw
+- Two categories may name one TSV
+- A cell may read a reference, and each row its own
+- A path is walked once without drawing before it is walked for real
+- A country's postal codes and streets are siblings under its locality
+- A locale package imports its country's geo package, and its `Modules()` returns both
+- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+- A locale's `address` binds its country's record once and prints it whole
+- A postort's kommun comes from its name, its tätort or its codes, never from distance
+- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+- `--list` stays a plain list of paths
+- A layout is always quoted
+- A title is a table under `sex`
+- A table owns the spelling of a selector on it
+- No builtin reads the clock, so a date is bounded by days, never by an age
+- `misc` is what every locale shares
+- A register's canonical spelling loses to the one its domain writes
+- `misc.territory` is the spine, and a `misc` table naming a territory links to it
+- A table whose register publishes no frequency draws evenly
+- `misc.timezone` weighs a zone by the people living in it
+- `misc.car` is one flat table, not a make linked to its models
+- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+- `misc.tld` keys carry the leading dot, where other tables key on a bare code
+- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+- `misc.territory` carries a currency code, it does not link to `misc.currency`
+- An extension may name two media types
+- The Swedish ids draw Skatteverket's test series
+- The US given names come from a mirror of the SSA file
+- `List` advertises direct descents only
+- A path draws through its compiled steps
+- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+- A country with no open address register covering it whole ships its places and postal codes without streets
+- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
+
+# Scoring run
+
+The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
+
+- `{language}`: Go
+- `{kind}`: a Go library and CLI that renders fake data from JSON templates
+- `{domain}`: fake test-data generation and template languages
+- `{domain docs}`: the documentation of any fake-data library, such as Faker
+- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another

CHANGELOG.md

@@ -8,59 +8,99 @@ replacement, and each removed path, column or flag.
 
 ### Added
 
-- First release: the CLI, the library and the shipped data set.
+- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
-- Table categories: a category JSON naming a `rows` TSV beside it, with the options
+  dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
-  `key`, `name`, `weight` and `parent`; a path selects a row by key or name,
+  the CLI installs without Go. 0.1.0 ships them too; the README's
-  `misc.country[SE]`, and descends to a linked table by name; linked tables draw
+  [Install](https://github.com/larvit/fejkdata#install) lists each command.
-  consistently within one render and draw group. `rows` is an option, so no
+
-  template may carry a field of that name. A `name` without a `key` resolves
+## [0.1.0] - 2026-10-13
-  inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
+
-  `parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any
+The first release: a Go library and a CLI that render locale-aware fake data from JSON
-  table above it, a table whose format or cell references its own family, and,
+templates, and the data they ship with. The
-  within one render and draw group, a path drawing a table another path selects a
+[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
-  row of, or two paths pinning different rows of one table.
+Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
-- `New` refuses a root choice of templates sharing one format and one set of string
+library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
-  fields, naming the rows TSV to write instead.
+and any release may change what a seed draws, so pin one exact version
-- `misc.country`, `misc.currency`, `misc.language`, `misc.httpstatus` and
+([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
-  `misc.mimetype` are tables. `misc.country` is every ISO 3166 country that has a
+
-  capital, a currency and a TLD, with the columns `calling-code`, `capital`,
+### Added
-  `currency`, `flag`, `languages`, `numeric` and `tld` added; `misc.currency` the
+
-  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
+- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
-  added, and its symbols from CLDR; `misc.language` every ISO 639-1 language, with
+  `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
-  a `code3` column carrying its ISO 639-2 code; `misc.httpstatus` every HTTP status
+  repeats a run's output while the version and data stay the same, and without it each
-  code in use in the IANA registry; and `misc.mimetype` every IANA media type that
+  run takes a new seed; `-n` renders the template that many times, and `--list`
-  has a filename extension. `DATA-LICENSES.md` lists each source.
+  prints the paths the data offers.
+  `--data-path` layers a directory of your own data over the shipped set, and
+  `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
+  a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
+  error such as data that fails to load, and `2` on misuse.
+- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
+  `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
+  from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
+  without data, it fails and names `WithDataFS`. The same seed repeats the draws
+  while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
+  and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
+  `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
+  `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
+  `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
+  and `IsTemplate` tells an inline template from a path. Data that fails to load gives
+  an error matching `ErrLoad`, and a record with no fields gives one matching
+  `ErrNoColumns`.
+- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
+  `cli.Main(modules...)` runs it over the modules given, so
+  `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
+  That CLI names itself by the file it runs from, and its `--version` prints its own
+  module's version before fejkdata's.
+- The template language
+  ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
+  Every value in a data file is a string, a choice, a template, a table, a number or a
+  boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
+  root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
+  or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
+  pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
+  digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
+  `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
+  transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
+  grammar, could mean two things or cannot render a valid value.
+- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
+  a template seen as columns. A column is a string unless it declares a `datatype`,
+  `integer`, `number` or `boolean`, and a `null` item draws it as null.
+- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
+  category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
+  `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
+  linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
+  `{/misc.timezone[Europe/Oslo]..territory.name}`.
+- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
+  a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
+  manifest at a source's root may carry a `namespace`, the dot path the source mounts
+  under, and an `index`, with which each category loads on the first call reaching it.
+- The shipped data, in Go packages that each export `FS` and `Modules()`:
+  `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
+  `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
+  `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
+  [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
+  names each source.
+  - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
+    `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
+    the Census Bureau. A person's first name and sex agree, and so does an `en_US`
+    person's title; `person.prefix` is null where a person has no title.
+    `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
+    `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
+    columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
+    assign.
+  - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
+    HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
+    symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
+    zones weighted by the people living in each, user agents and cars, all as tables;
+    and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
+    ObjectIds and UUIDs.
   - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
     `locality`, `postal-code` and `street`, weighted by population and address counts
-  and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
+    and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
-  `address` record over one consistent draw of them. `sv_SE.address` and
+    and an `address` record that draws a street and a postal code inside one
-  `en_US.address` read those records, so `en_US.address.street` no longer carries
+    locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
-  `name` and `suffix`, and a locale folder loads only beside `geo`.
+    (postal town) the import can place ships with all its streets. A US place ships
-- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or
+    only if it is incorporated and holds most of some postal code's land. Of those
-  within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`.
+    places, every state capital but Honolulu ships, as does every place of 100,000
-  `sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so
+    people or more and a tenth of all of them drawn by population. Each US place
-  `date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`,
+    carries a quarter of its streets, at least ten.
-  `time.sec` and `time.ampm` are no longer paths — a part of a date is now its own
-  `{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant.
-- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by
-  bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a
-  name both sexes carry is a row under each. `person` reads them, so its columns are
-  `first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are
-  no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead.
-  `en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms`
-  without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the
-  same shape.
-- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series
-  from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose
-  `ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the
-  ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so
-  `--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new
-  and carries the same three. `person.prefix` is null where a person has no title,
-  where it used to be an empty string, so `--format sql` writes `NULL` and a `string`
-  struct field reading it becomes `*string`.
-- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can
-  answer from the rest.
-- An error names a spelling that runs: a layout is named single-quoted and free of
-  its own quotes, a row of a table with no key is named as the path that selects it,
-  `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record
-  that gives it one.

DATA-LICENSES.md

@@ -9,16 +9,23 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
 | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
 | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
 | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
-| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
+| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
 | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
 | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
 | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
 | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
-| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv` | curated (Skatteverket's test birth numbers are facts) | — | — | — |
+| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
-| `misc/country.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/country.py` |
 | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
+| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
 | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
 | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` |
-| `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | none required | `data-import/mimetype.py` |
+| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from the [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `<syslog.h>` severity macros | facts | none required | `data-import/loglevel.py` |
+| `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` |
+| `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` |
+| `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` |
+| `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` |
+| `misc/timezone.tsv` | [IANA tzdb](https://www.iana.org/time-zones) 2026d `zone.tab` and the standard offset of each zone; weights from [GeoNames](https://www.geonames.org/) `cities15000` populations | [public domain](https://data.iana.org/time-zones/tzdb/LICENSE); [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | "Populations from GeoNames, www.geonames.org" | `data-import/timezone.py` |
+| `misc/tld.tsv` | [IANA Root Zone Database](https://www.iana.org/domains/root/db), checked against the root zone's [TLD list](https://data.iana.org/TLD/tlds-alpha-by-domain.txt) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/tld.py` |
+| `misc/useragent.tsv` | [top-user-agents](https://github.com/microlinkhq/top-user-agents) desktop and mobile lists | [MIT](https://github.com/microlinkhq/top-user-agents/blob/master/LICENSE.md) | "Copyright © 2020 Kiko Beats" | `data-import/useragent.py` |
 
-Every other category is hand-written JSON under [`data/`](data), MIT like the code.
+Every other category is hand-written under [`data/`](data), MIT like the code.

Dockerfile

@@ -1,19 +1,43 @@
-# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
+ARG GO_VERSION=1.27.2
-# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
+FROM golang:${GO_VERSION} AS portable
-# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
-# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
-ARG GO_VERSION=1.26.4
-FROM golang:${GO_VERSION}
 
 WORKDIR /app
 
-# Module layer cached separately from source.
 COPY go.mod ./
 RUN go mod download
 
 COPY . .
 RUN go vet ./... && \
 	go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
-	{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
-	  { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
 	go test -race ./...
+
+FROM golang:${GO_VERSION} AS vulnerabilities
+WORKDIR /app
+COPY . .
+RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
+
+FROM scratch AS vulnerability-report
+COPY --from=vulnerabilities /report.json /
+
+FROM python:3.14.8-slim AS release-tooling
+COPY release-tooling/ /release-tooling/
+RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
+
+FROM python:3.14.8-slim AS vulnerability-filing
+COPY vulnerability-filing/ /vulnerability-filing/
+# The filer's test files an item into the plan itself.
+COPY todo.md /todo.md
+RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
+
+FROM goreleaser/goreleaser:v2.18.3 AS packaging
+WORKDIR /check
+COPY release-tooling/goreleaser.yaml ./
+RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
+
+FROM portable
+# BuildKit skips a stage nothing copies from.
+COPY --from=release-tooling /tested /tmp/release-tooling-tested
+COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
+COPY --from=packaging /tested /tmp/packaging-tested
+RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
+	{ echo "unformatted files:"; echo "$unformatted"; exit 1; }

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2026 Timewave
+Copyright (c) 2026 Larvit
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

arm.go

@@ -0,0 +1,117 @@
+package fejkdata
+
+import (
+	"fmt"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// arm is one alternative of a {a|b} token or one operand, split into the head,
+// whose node `template.startOf` finds, and the tail of a dotted path into it.
+type arm struct {
+	spelling string // as written, for messages
+	head     string
+	tail     []string
+	levels   []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
+	steps    []pathStep
+	leaves   []node // every node the path may land on, one per variant it passes
+	kind     armKind
+	named    *nameBinding // namedRead: the binding of the name it reads through
+}
+
+// armKind is how expand reads an arm, fixed at compile.
+type armKind uint8
+
+const (
+	freshRead armKind = iota // drawn afresh at every read
+	namedRead                // read through a name, kept in its pick
+)
+
+// splitArm splits one name into head and tail. refs maps a reference to what
+// resolveRefs resolved it to; before that, a reference is whole.
+func splitArm(name string, refs map[string]resolvedRef) arm {
+	if grammar.IsRef(name) {
+		if r, resolved := refs[name]; resolved {
+			return pathArm(name, r.head, r.tail)
+		}
+		return pathArm(name, name, nil)
+	}
+	segs, err := grammar.SplitPath(name)
+	if err != nil || len(segs) == 1 {
+		return pathArm(name, name, nil)
+	}
+	return pathArm(name, segs[0], segs[1:])
+}
+
+func pathArm(name, head string, segs []string) arm {
+	return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
+}
+
+// key is the key of the leaf a lands on, the one key every way of writing this read shares.
+func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
+
+func checkSegments(a arm) error {
+	if len(a.tail) == 0 {
+		return nil
+	}
+	return grammar.CheckSegments(append([]string{a.head}, a.tail...))
+}
+
+// op is one compiled unit of a format string: a literal run, a name read,
+// or a builtin already prepared with its args.
+type op struct {
+	grammar.Token
+	arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
+	call builtinfunc.Call
+	// operands are the fields the builtin reads, in the order its operands func
+	// fixed; expand reads them before the call. nil for a builtin that reads none.
+	operands []arm
+}
+
+func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+	arms := make([]arm, len(names))
+	for i, name := range names {
+		a, err := t.compileArm(name, targets)
+		if err != nil {
+			return nil, err
+		}
+		arms[i] = a
+	}
+	return arms, nil
+}
+
+// compileArm compiles one read into a path: from the head it names, or, for a read through a
+// name, from what that name binds.
+func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
+	a := splitArm(name, t.refs.byName)
+	if !t.isName(a.head) {
+		start := t.startOf(a.head)
+		if start == nil {
+			panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
+		}
+		w := compilePath(start, a.tail)
+		a.steps, a.leaves = w.steps, w.leaves
+		return a, nil
+	}
+	b := t.nameScope.lookup(a.head)
+	if grammar.HasSelector(a.tail) {
+		if b.bindsField() {
+			return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
+		}
+		return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
+	}
+	target := targets[b]
+	full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
+	if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
+		return a, err
+	}
+	w := compilePath(target.start, full)
+	a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
+	a.levels = levelKeys(full, 0)
+	return a, nil
+}

bench_test.go

@@ -6,17 +6,17 @@ import (
 	"testing"
 )
 
-func benchGenerator(b *testing.B, dir string) *Generator {
+func benchGenerator(b *testing.B, data Option) *Generator {
 	b.Helper()
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
+	f, err := New(data, WithSeed(1))
 	if err != nil {
 		b.Fatal(err)
 	}
 	return f
 }
 
-func benchPath(b *testing.B, dir, path string) {
+func benchPath(b *testing.B, data Option, path string) {
-	f := benchGenerator(b, dir)
+	f := benchGenerator(b, data)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
 	}
 }
 
-func BenchmarkPerson(b *testing.B)       { benchPath(b, "data", "sv_SE.person") }
+func BenchmarkPerson(b *testing.B)       { benchPath(b, withShipped(), "sv_SE.person") }
-func BenchmarkAddress(b *testing.B)      { benchPath(b, "data", "sv_SE.address") }
+func BenchmarkAddress(b *testing.B)      { benchPath(b, withShipped(), "sv_SE.address") }
-func BenchmarkWord(b *testing.B)         { benchPath(b, "data", "sv_SE.word") }
+func BenchmarkWord(b *testing.B)         { benchPath(b, withShipped(), "sv_SE.word") }
-func BenchmarkCreditcard(b *testing.B)   { benchPath(b, "data", "misc.creditcard") }
+func BenchmarkCreditcard(b *testing.B)   { benchPath(b, withShipped(), "misc.creditcard") }
-func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") }
+func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
-func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, "data", "misc.uuid") }
+func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, withShipped(), "misc.uuid") }
 
 func tmpData(b *testing.B, name, body string) string {
 	b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
 
 func BenchmarkCalc(b *testing.B) {
 	dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
-	benchPath(b, dir, "inv")
+	benchPath(b, WithDataPath(dir), "inv")
 }
 
 func BenchmarkLongLiteral(b *testing.B) {
 	dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "sql")
+	benchPath(b, WithDataPath(dir), "sql")
 }
 
 func BenchmarkRepeat(b *testing.B) {
 	dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "many")
+	benchPath(b, WithDataPath(dir), "many")
 }
 
-// BenchmarkBound measures a format that binds a head and reads two paths from it —
+// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
-// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
+// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
-// two independent fields.
+// independent fields.
-func BenchmarkBound(b *testing.B) {
+func BenchmarkNamed(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
+	dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-func BenchmarkUnbound(b *testing.B) {
+func BenchmarkUnnamed(b *testing.B) {
 	dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
+// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
-// depth question is about.
+func BenchmarkNamedDeep(b *testing.B) {
-func BenchmarkBoundDeep(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
-	dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
+	benchPath(b, WithDataPath(dir), "addr")
-	benchPath(b, dir, "addr")
 }
 
-// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
+// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
-// expansion holds outgrow a single bucket.
+// outgrow a single bucket.
-func BenchmarkBoundWide(b *testing.B) {
+func BenchmarkNamedWide(b *testing.B) {
-	dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
+	dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
 		{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
 		{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
-	benchPath(b, dir, "row")
+	benchPath(b, WithDataPath(dir), "row")
+}
+
+func BenchmarkFirstFake(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		f, err := New(withShipped(), WithSeed(1))
+		if err != nil {
+			b.Fatal(err)
+		}
+		if _, err := f.Fake("sv_SE.address"); err != nil {
+			b.Fatal(err)
+		}
+	}
 }
 
 func BenchmarkNew(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if _, err := New(WithSeed(1)); err != nil {
+		if _, err := New(withShipped(), WithSeed(1)); err != nil {
 			b.Fatal(err)
 		}
 	}

builtins_test.go

@@ -2,11 +2,14 @@ package fejkdata
 
 import (
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 func TestBuiltinIDGenerators(t *testing.T) {
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
 	}
 }
 
-// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
+// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
 // each call, named counters independent, and the whole thing scoped to one
-// session so a fresh Generator restarts at 1.
+// draw state so a fresh Generator restarts at 1.
-func TestBuiltinSeqPerSession(t *testing.T) {
+func TestBuiltinSeqPerGenerator(t *testing.T) {
 	f := engine(1)
 	for i := 1; i <= 5; i++ {
 		if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
 	if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
 		t.Fatalf("repeat seq = %q, want 1,2,3", got)
 	}
-	// ...and a fresh session restarts from 1.
+	// ...and a fresh generator restarts from 1.
 	if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
-		t.Fatalf("new session seq = %q, want 1", got)
+		t.Fatalf("new generator seq = %q, want 1", got)
 	}
 }
 
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
 		`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
 	}
 	for _, tmpl := range bad {
-		if _, err := compile(parse(t, tmpl)); err == nil {
+		if _, err := resolved(t, tmpl); err == nil {
 			t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
 		}
 	}
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
 	return rem == 1
 }
 
-// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
-// supplies prep, and args parsed at compile only behind a check.
+// separates a reported invariant break from a silently wrong value.
-func TestRegistryShapes(t *testing.T) {
-	for name, b := range builtins {
-		if b.prep == nil {
-			t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
-		}
-		if b.arity != 0 && b.check == nil {
-			t.Errorf("builtin %q parses args in prep with no check", name)
-		}
-	}
-}
-
-// TestArgGuardsPanic pins the guards that report a builtin arg its check should
-// have rejected. No data reaches them — checkFunc runs a builtin's check before
-// compileOps ever calls prep — so they are exercised directly.
-func TestArgGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"atoi on an unvalidated arg": func() { atoi("nope") },
-		"atof on an unvalidated arg": func() { atof("nope") },
-	} {
-		mustPanic(t, name, call)
-	}
-}
-
-// mustPanic fails unless call panics, which is what separates a reported
-// invariant break from a silently wrong value.
 func mustPanic(t *testing.T, name string, call func()) {
 	t.Helper()
 	defer func() {
-		if recover() == nil {
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
-			t.Errorf("%s: no panic, want the invariant reported", name)
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
 		}
 	}()
 	call()
 }
 
 func TestClassBuiltinArgs(t *testing.T) {
-	for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
+	for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
-		if _, err := compile(parse(t, bad)); err == nil {
+		if _, err := resolved(t, bad); err == nil {
 			t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
 		}
 	}
 	for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v", ok, err)
 		}
 	}
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
 	}
 }
 
-// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order,
+// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
-// the layout is quoted, names a field, and for time names no date field.
+// layout is quoted.
 func TestBuiltinDateArgs(t *testing.T) {
 	for tmpl, want := range map[string]string{
 		`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`:        "1990-13-01",
+		`"{time('2006-01-02 15:04')}"`:                        "write date(from,to,layout)",
 		`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`:        "is after",
-		`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`:        "write it as text",
 		`"{date(1990-01-01,1990-12-31,2006-01-02)}"`:          "'2006-01-02'",
 		`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`:    "'",
 		`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
-		`"{date(1990-01-01,1990-12-31,'x')}"`:                 "text",
-		`"{date(1990-01-01,1990-12-31,'')}"`:                  "text",
 		`"{date(1990-01-01,1990-12-31)}"`:                     "3 arguments",
 		`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`:     "'January 2, 2006'",
 		`"{time(3:04 PM, Mon)}"`:                              "'3:04 PM, Mon'",
 		`"{time(15:04)}"`:                                     "'15:04'",
-		`"{time('2006-01-02 15:04')}"`:                        "date(",
-		`"{time('x')}"`:                                       "text",
-		`"{date(1990-01-01,1990-12-31,'15:04')}"`:             "time('15:04')",
 	} {
-		_, err := compile(parse(t, tmpl))
+		_, err := resolved(t, tmpl)
 		if err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
 		}
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
 }
 
 // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
-// draws every second of it, and without one it could only emit one value.
+// draws every second of it, and without one it prints that day.
 func TestBuiltinDateSpansOneDay(t *testing.T) {
 	f := engine(1)
 	seen := map[string]bool{}
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
 	if len(seen) < 400 {
 		t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
 	}
-	_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`))
+	if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
-	if err == nil || !strings.Contains(err.Error(), "write it as text") {
+		t.Fatalf("one day in a date-only layout = %q, want that day", got)
-		t.Fatalf("one day in a date-only layout = %v, want it named a constant", err)
+	}
+}
+
+func TestClassBuiltins(t *testing.T) {
+	cases := map[string]*regexp.Regexp{
+		`"{digits(1)}"`:          regexp.MustCompile(`^[0-9]$`),
+		`"{digits(3)}"`:          regexp.MustCompile(`^[0-9]{3}$`),
+		`"{int(1,9)}"`:           regexp.MustCompile(`^[1-9]$`),
+		`"{upper(1)}"`:           regexp.MustCompile(`^[A-Z]$`),
+		`"{lower(1)}"`:           regexp.MustCompile(`^[a-z]$`),
+		`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
+	}
+	f := engine(7)
+	for tmpl, re := range cases {
+		for i := 0; i < 100; i++ {
+			if got := mustRender(t, f, tmpl); !re.MatchString(got) {
+				t.Fatalf("%s produced %q, want %s", tmpl, got, re)
+			}
+		}
+	}
+}
+
+func TestFunctionTokenLuhn(t *testing.T) {
+	// {luhn()} appends a Luhn check digit over the digits emitted so far in the
+	// current expansion (non-digits skipped but kept). It reads the output
+	// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
+	f := engine(1)
+	cases := map[string]string{
+		`"811218987{luhn()}"`:                      "8112189876",  // personnummer body
+		`"7992739871{luhn()}"`:                     "79927398713", // classic Luhn vector
+		`"811218-987{luhn()}"`:                     "811218-9876", // '-' skipped, kept
+		`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876",  // over a rendered token
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Fatalf("%s = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestBuiltinCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{upper(x)}"`,         // count must be an integer
+		`"{int(a,b)}"`,         // non-integer args
+		`"{int(5,1)}"`,         // min > max
+		`"{nanoid(-1)}"`,       // negative count
+		`"{float(1,2,-1)}"`,    // negative decimals
+		`"{float(NaN,NaN,2)}"`, // bounds must be finite
+		`"{float(Inf,Inf,2)}"`, // same-sign infinities
+		`"{float(1,NaN,2)}"`,   // one NaN bound
+		`"{float(-Inf,1,2)}"`,  // one infinite bound
+		`"{iban(US)}"`,         // unsupported country
+		`"{seq(a,b)}"`,         // seq takes at most one name
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestArgErrorsNameTheSpelling(t *testing.T) {
+	if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
+		t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
+	}
+}
+
+func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
+	for long, short := range map[string]string{
+		`"{float(1,2,02)}"`:                 `"{float(1,2,2)}"`,
+		`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
+		`"{digits(+5)}"`:                    `"{digits(5)}"`,
+		`"{digits(05)}"`:                    `"{digits(5)}"`,
+		`"{int(007,9)}"`:                    `"{int(7,9)}"`,
+		`"{int(+1,5)}"`:                     `"{int(1,5)}"`,
+	} {
+		sameRenders(t, long, short)
 	}
 }

calc.go

@@ -2,392 +2,149 @@ package fejkdata
 
 import (
 	"fmt"
-	"math"
+	"slices"
 	"strconv"
 	"strings"
-	"unicode"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// calcNode is a parsed expression node. It evaluates over the operand values expand
+// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
-// read, so the evaluator touches neither the rng nor the node tree.
+// field is a name, which checkCalcNames checks once names resolve.
-type calcNode interface {
+func checkCalcFields(args []string, fields map[string]node) error {
-	eval(operands []string) float64
+	return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
+		if n, ok := fields[name]; ok {
+			return []node{n}
+		}
+		return nil
+	})
 }
 
-type calcNum float64 // a number literal
+// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
-type calcVar string  // a sibling-field name, before indexVars places it
+func checkCalcNames(label string, t *template) error {
-type calcIdx int     // an operand, by its position in the values expand read
+	for _, o := range t.compiled.ops {
-type calcNeg struct{ x calcNode }
+		if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
-type calcBin struct { // a + - * / b
+			continue
-	op   byte
+		}
-	l, r calcNode
+		if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
+			return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
+		}
+	}
+	return nil
 }
 
-func (n calcNum) eval([]string) float64 { return float64(n) }
+// operandNodes lists every node each operand of o may render, once its template and names resolve.
-
+func operandNodes(o op) func(name string) []node {
-func (n calcIdx) eval(operands []string) float64 {
+	return func(name string) []node {
-	v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
+		a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
-	if err != nil {
+		if len(a.leaves) == 0 {
-		return math.NaN() // a non-numeric operand stays visible, never an error
+			panic(invariant.Broken("calc operand %q was read before it was compiled", name))
 		}
-	return v
+		return a.leaves
-}
-
-// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
-// expression it compiles, so only a calcIdx reaches a render. It panics rather
-// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
-// silently wrong number.
-func (n calcVar) eval([]string) float64 {
-	panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
-}
-
-func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
-
-func (n calcBin) eval(operands []string) float64 {
-	l, r := n.l.eval(operands), n.r.eval(operands)
-	switch n.op {
-	case '+':
-		return l + r
-	case '-':
-		return l - r
-	case '*':
-		return l * r
-	default: // '/'
-		return l / r
 	}
 }
 
-// checkCalc validates a calc token at compile time: a parseable expression whose
+// checkOperands refuses an operand that is never a number and a division by a constant zero.
-// operands all name existing fields, and an optional non-negative integer dp. It
+// operand lists every node an operand may render, nil while that is unknown.
-// is a builtin check (fields first), so calc dispatches through the registry like
+func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
-// every other {name(args)} function.
+	for _, name := range c.Operands {
-func checkCalc(fields map[string]node, args []string) error {
+		if rendered, never := allNeverNumeric(operand(name)); never {
-	if len(args) < 1 || len(args) > 2 {
+			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
-		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return fmt.Errorf("calc(%q): %w", args[0], err)
-	}
-	for _, name := range calcVars(expr) {
-		operand, ok := fields[name]
-		if !ok {
-			return fmt.Errorf("calc(%q): no field %q", args[0], name)
-		}
-		if text, never := neverNumeric(operand); never {
-			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
 		}
 	}
-	if divisor, zero := constantZeroDivisor(expr, fields); zero {
+	if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
-		return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
+		return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
-	}
-	if len(args) == 2 {
-		dp, err := plainInt(args[1])
-		if err != nil {
-			return fmt.Errorf("calc decimals %w", err)
-		}
-		if dp < 0 || dp > maxDecimals {
-			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
-		}
 	}
 	return nil
 }
 
 // constantZeroDivisor finds a division whose right side is a constant zero: number
 // literals and fixed operands folded, anything that varies left unknown.
-func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
+func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
 	switch n := n.(type) {
-	case calcNeg:
+	case grammar.CalcNeg:
-		return constantZeroDivisor(n.x, fields)
+		return constantZeroDivisor(n.X, operand)
-	case calcBin:
+	case grammar.CalcBin:
-		if n.op == '/' {
+		if n.Operator == '/' {
-			if v, known := constantValue(n.r, fields); known && v == 0 {
+			if v, known := constantValue(n.R, operand); known && v == 0 {
-				return calcText(n.r), true
+				return grammar.CalcText(n.R), true
 			}
 		}
-		if d, zero := constantZeroDivisor(n.l, fields); zero {
+		if d, zero := constantZeroDivisor(n.L, operand); zero {
 			return d, true
 		}
-		return constantZeroDivisor(n.r, fields)
+		return constantZeroDivisor(n.R, operand)
 	}
 	return "", false
 }
 
 // constantValue evaluates an expression whose every operand is fixed.
-func constantValue(n calcNode, fields map[string]node) (float64, bool) {
+func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
 	switch n := n.(type) {
-	case calcNum:
+	case grammar.CalcNum:
 		return float64(n), true
-	case calcVar:
+	case grammar.CalcVar:
-		t, ok := fields[string(n)].(*template)
+		nodes := operand(n.Name)
-		if !ok || !t.fixed || t.repeat > 1 {
+		if len(nodes) != 1 {
 			return 0, false
 		}
-		v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
+		t, ok := nodes[0].(*template)
+		if !ok || t.repeat != 1 {
+			return 0, false
+		}
+		lit, fixed := t.fixedText()
+		if !fixed {
+			return 0, false
+		}
+		v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
 		return v, err == nil
-	case calcNeg:
+	case grammar.CalcNeg:
-		v, ok := constantValue(n.x, fields)
+		v, ok := constantValue(n.X, operand)
 		return -v, ok
-	case calcBin:
+	case grammar.CalcBin:
-		l, lok := constantValue(n.l, fields)
+		l, lok := constantValue(n.L, operand)
-		r, rok := constantValue(n.r, fields)
+		r, rok := constantValue(n.R, operand)
 		if !lok || !rok {
 			return 0, false
 		}
-		return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
+		return builtinfunc.Arith(n.Operator, l, r), true
 	}
 	return 0, false
 }
 
-// calcText spells an expression node the way an author would read it.
+// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
-func calcText(n calcNode) string {
+// text that does not parse, or a choice of only such items. text is one such render.
-	switch n := n.(type) {
-	case calcNum:
-		return strconv.FormatFloat(float64(n), 'f', -1, 64)
-	case calcVar:
-		return string(n)
-	case calcNeg:
-		return "-" + calcText(n.x)
-	case calcBin:
-		return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
-	}
-	return "?"
-}
-
-// neverNumeric reports a node no render of which is a number: a null, fixed text that
-// does not parse, or a choice of only such items. text is one such render.
 func neverNumeric(n node) (text string, never bool) {
 	switch n := n.(type) {
-	case *null:
+	case *nullItem:
 		return "", true
 	case *template:
-		if !n.fixed || n.repeat > 1 {
+		lit, fixed := n.fixedText()
+		if n.repeat == 0 {
+			return "", true
+		}
+		if !fixed || n.repeat != 1 {
 			return "", false
 		}
-		if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
+		if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
-			return n.lit, true
+			return lit, true
 		}
 	case *choice:
-		for _, it := range n.items {
+		return allNeverNumeric(n.items)
-			t, itemNever := neverNumeric(it)
+	}
-			if !itemNever {
+	return "", false
+}
+
+// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
+func allNeverNumeric(nodes []node) (text string, never bool) {
+	for _, n := range nodes {
+		t, nodeNever := neverNumeric(n)
+		if !nodeNever {
 			return "", false
 		}
 		text = t
 	}
-		return text, true
+	return text, len(nodes) > 0
-	}
-	return "", false
-}
-
-// calcPrep parses the expression and decimals once, at compile time, and places each
-// operand name at the position expand will read it into. checkCalc proved both args
-// valid, so no step here can fail; dp -1 prints the minimal form.
-func calcPrep(args []string) callFn {
-	expr, err := parseCalc(args[0])
-	if err != nil { // a nil AST would be a nil dereference per render, with no message
-		panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
-	}
-	at := make(map[string]int)
-	for i, name := range calcVars(expr) {
-		at[name] = i
-	}
-	placed := indexVars(expr, at)
-	dp := calcDecimals(args)
-	return func(_ *session, _ string, operands []string) string {
-		return formatFloat(placed.eval(operands), dp)
-	}
-}
-
-// calcDecimals is a calc's decimals count, or -1 for the shortest form.
-func calcDecimals(args []string) int {
-	if len(args) == 2 {
-		return atoi(args[1])
-	}
-	return -1
-}
-
-// indexVars replaces each operand name with its position in the values expand reads.
-// Both sides take that order from calcVars, so they cannot drift.
-func indexVars(n calcNode, at map[string]int) calcNode {
-	switch n := n.(type) {
-	case calcVar:
-		i, placed := at[string(n)]
-		if !placed { // calcVars named every operand, so a miss means the two disagree
-			panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
-		}
-		return calcIdx(i)
-	case calcNeg:
-		return calcNeg{indexVars(n.x, at)}
-	case calcBin:
-		return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
-	}
-	return n
-}
-
-// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
-// an expression that does not parse, so one that does not simply names nothing.
-func calcOperands(args []string) []string {
-	if len(args) == 0 {
-		return nil
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return nil
-	}
-	return calcVars(expr)
-}
-
-// calcVars lists the distinct field names an expression reads, in the order it first
-// names each. That order is the contract between expand, which reads the operands
-// into a slice, and indexVars, which places each name at its position in it.
-func calcVars(n calcNode) []string {
-	var out []string
-	seen := map[string]bool{}
-	var walk func(calcNode)
-	walk = func(n calcNode) {
-		switch n := n.(type) {
-		case calcVar:
-			if name := string(n); !seen[name] {
-				seen[name] = true
-				out = append(out, name)
-			}
-		case calcNeg:
-			walk(n.x)
-		case calcBin:
-			walk(n.l)
-			walk(n.r)
-		}
-	}
-	walk(n)
-	return out
-}
-
-// calcParser is a recursive-descent parser over the expression runes, threading
-// expr -> term -> factor for the standard * / before + - precedence.
-type calcParser struct {
-	rs  []rune
-	pos int
-}
-
-// parseCalc parses a whole expression, requiring it to consume all input.
-func parseCalc(expr string) (calcNode, error) {
-	p := &calcParser{rs: []rune(expr)}
-	if p.space(); p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("empty expression")
-	}
-	n, err := p.expr()
-	if err != nil {
-		return nil, err
-	}
-	if p.space(); p.pos != len(p.rs) {
-		return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
-	}
-	return n, nil
-}
-
-func (p *calcParser) space() {
-	for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
-		p.pos++
-	}
-}
-
-func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
-func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
-
-// binary parses a left-associative run of next() operands joined by the given
-// operators, the one shape expr and term share.
-func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
-	n, err := next()
-	if err != nil {
-		return nil, err
-	}
-	for {
-		p.space()
-		if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
-			return n, nil
-		}
-		op := byte(p.rs[p.pos])
-		p.pos++
-		r, err := next()
-		if err != nil {
-			return nil, err
-		}
-		n = calcBin{op, n, r}
-	}
-}
-
-// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
-func (p *calcParser) factor() (calcNode, error) {
-	p.space()
-	if p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("unexpected end of expression")
-	}
-	switch c := p.rs[p.pos]; {
-	case c == '-':
-		p.pos++
-		x, err := p.factor()
-		if err != nil {
-			return nil, err
-		}
-		return calcNeg{x}, nil
-	case c == '(':
-		p.pos++
-		n, err := p.expr()
-		if err != nil {
-			return nil, err
-		}
-		if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
-			return nil, fmt.Errorf("missing ')'")
-		}
-		p.pos++
-		return n, nil
-	case c == '.' || c >= '0' && c <= '9':
-		return p.number()
-	case c == '_' || unicode.IsLetter(c):
-		return p.ident()
-	default:
-		return nil, fmt.Errorf("unexpected %q", string(c))
-	}
-}
-
-func (p *calcParser) number() (calcNode, error) {
-	start, dot := p.pos, false
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c >= '0' && c <= '9' {
-			p.pos++
-		} else if c == '.' && !dot {
-			dot, p.pos = true, p.pos+1
-		} else {
-			break
-		}
-	}
-	v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
-	if err != nil {
-		return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
-	}
-	return calcNum(v), nil
-}
-
-// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
-// is always the minus operator, so a hyphenated field name can't be an operand.
-func (p *calcParser) ident() (calcNode, error) {
-	start := p.pos
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
-			p.pos++
-		} else {
-			break
-		}
-	}
-	return calcVar(string(p.rs[start:p.pos])), nil
-}
-
-func contains(bs []byte, b byte) bool {
-	for _, x := range bs {
-		if x == b {
-			return true
-		}
-	}
-	return false
 }

calc_test.go

@@ -3,8 +3,11 @@ package fejkdata
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
 )
 
 // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
 	if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
 		t.Fatalf("calc(a - b) = %q, want 7", got)
 	}
+	if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
+		t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
+	}
 }
 
 // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
 	}
 }
 
-// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
+// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
-// is not a calc, and for one whose expression does not parse: nothing, either way.
+// call that is not a calc, and for one whose expression does not parse: none, either way.
-// checkCalc is what reports a bad expression, so the callers that run after it
+func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
-// never meet one — but they must not have to depend on that order to be safe.
+	for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
-func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
+		if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
-	for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
+			t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
-		if got := tokenOperands(body); got != nil {
-			t.Errorf("tokenOperands(%q) = %v, want none", body, got)
 		}
 	}
-	if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
+	if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
-		t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
+		t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
-	}
-}
-
-// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
-// the expression before prep sees it, and indexVars places every name calcVars
-// read, so a break in either is reported rather than computed around.
-func TestCalcGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
-		"an operand name never placed":              func() { calcVar("n").eval(nil) },
-		"indexVars on a name it did not read":       func() { indexVars(calcVar("n"), map[string]int{}) },
-	} {
-		mustPanic(t, name, call)
 	}
 }
 
 // --- one draw, one value: a calc operand reads the expansion's draw ---
 
-// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
+// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
-// A field the format renders and a calc reads is drawn once per expansion, so the
+// shows: the format and the calc read one name.
-// operand shown is the operand computed — the correlation the dotted-path rule
+func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
-// already gives a level (see bound_test.go), applied to a plain sibling.
-func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(3))
 	for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	}
 }
 
-// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
+// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
-// expansion, not to the calc, so the bare token rendering the same name reads it too.
+func TestACalcOperandDrawsAfresh(t *testing.T) {
-func TestCalcOperandSharesOneDraw(t *testing.T) {
+	dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
-	dir := writeData(t, map[string]string{
-		"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
-	})
 	f := newGenerator(t, dir, WithSeed(5))
 	for i := 0; i < 200; i++ {
-		got := fake(t, f, "same")
+		if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
-		if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
-			t.Fatalf("same = %q, want one value twice", got)
-		}
-	}
-}
-
-// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
-// held, so an ordinary {w} {w} still draws twice.
-func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
-	dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
-	f := newGenerator(t, dir, WithSeed(5))
-	for i := 0; i < 200; i++ {
-		if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
 			return
 		}
 	}
-	t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
+	t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
 }
 
-// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
+// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
-// its own expansion, so it draws again while staying self-consistent.
+// iteration, staying one value within it.
-func TestCalcHoldIsPerExpansion(t *testing.T) {
+func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
+		"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(11))
 	varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
 	}
 }
 
-// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
+// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
-// {v} and its calc agree with each other, not with the outer pair.
+// pair agrees with itself, not with the outer pair.
-func TestCalcHoldIsPerTemplate(t *testing.T) {
+func TestTwoNamesPickApart(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
+		"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
-			"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
+			"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
 	})
 	f := newGenerator(t, dir, WithSeed(13))
 	differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["a","b"]}`:       `"x"`,
 		`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
 	} {
-		_, err := compile(parse(t, src))
+		_, err := resolved(t, src)
 		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
 			t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
 		}
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
 		`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
 	} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
 		}
 	}
@@ -257,7 +230,7 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 		`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`:   "divides by 0",
 		`{"format":"{calc(a/-0)}","a":"1"}`:              "divides by -0",
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
 		}
 	}
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	}
 	t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
 }
+
+func TestCalcCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{calc(1/0)}"`, // a constant zero divisor
+		`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
+		`"{calc()}"`,        // calc needs an expression
+		`"{calc(1 +)}"`,     // dangling operator
+		`"{calc((1 + 2)}"`,  // unbalanced parenthesis
+		`"{calc(1 2)}"`,     // two operands, no operator
+		`"{calc(price)}"`,   // operand names no field
+		`"{calc(1, 2, 3)}"`, // too many args
+		`"{calc(1, x)}"`,    // decimals arg not an integer
+		`"{calc(1, -1)}"`,   // decimals negative
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestCalcReadsAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3","5","7"]`,
+		"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	for i := 0; i < 50; i++ {
+		var k, double int
+		got := fake(t, f, "card")
+		if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
+			t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
+		}
+	}
+}
+
+func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3"]`,
+		"card": `"{/n as k}{calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	if got := fake(t, f, "card"); got != "4" && got != "6" {
+		t.Fatalf("card = %q, want 4 or 6", got)
+	}
+}
+
+func TestCalcRefusesABadOperand(t *testing.T) {
+	for card, want := range map[string]string{
+		`"{/word as w}{w}{calc(w * 2)}"`:    `operand "w" is never a number: it renders "def"`,
+		`"{/zero as z}{z}{calc(1 / z)}"`:    `divides by z, which is always zero`,
+		`"{calc(nope * 2)}"`:                `no field "nope"`,
+		`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
+	} {
+		dir := writeData(t, map[string]string{
+			"word": `["abc","def"]`,
+			"zero": `"0"`,
+			"card": card,
+		})
+		if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
+		}
+	}
+}
+
+func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
+		"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
+	})))
+	if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
+		t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
+	}
+}
+
+func TestATypedColumnProvesACalcOverAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":     `"{int(1,9)}"`,
+		"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	r, err := f.FakeRecord("order")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
+		t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
+	}
+	if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
+		t.Fatalf("JSON = %s, want double written as a number", r.JSON())
+	}
+	bad := writeData(t, map[string]string{
+		"word":  `["1","x"]`,
+		"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
+	})
+	if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
+		t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
+	}
+}
+
+func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
+	if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
+		t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
+	}
+}

cli/cli.go

@@ -0,0 +1,706 @@
+// Package cli is the fejkdata command: it renders the template on its stdin with fake
+// values from the modules it is given and any directories layered over them. A program
+// passing its own modules to Main is a fejkdata CLI carrying them, for example
+// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
+package cli
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata"
+)
+
+const usage = `Usage: <template> | COMMAND [flags]
+       COMMAND --list | --help | --version
+
+COMMAND reads one template from stdin and renders it:
+
+  echo 'id: {int(1,999)}' | COMMAND
+  echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
+  COMMAND < template.txt
+
+COMMAND <<'EOF'
+born {date(1990-01-01,2010-12-31,'2006-01-02')}
+EOF
+
+A template is a format string, or a JSON object, array, string or null. Its {…} tokens
+reach the data by reference from the root: {/path}, for any path --list prints, shipped
+or layered with --data-path. A reference names a category or a dotted path into one, a
+table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
+links to.
+
+Under --format text, the default, what you send is what renders, and COMMAND adds
+nothing: a format string keeps every byte, so echo's newline ends each render, and
+printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
+object, array or string drops the whitespace around it, and null alone renders nothing;
+42 or true alone is a format string, printed as written. An empty or blank stdin is a
+format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
+a backslash or a quote as written.
+
+With --format json, ndjson, csv or sql the template must be a record — a JSON
+template whose fields are its columns, or a lone reference such as {/path}:
+one reference and nothing else, with one newline after it allowed — and the rows are
+written as one JSON array, one JSON object per line, one CSV row (after a header), or
+one INSERT.
+
+  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
+      --format F         output form: text (default), json, ndjson, csv or sql
+  -h, --help             print this help, then exit
+      --list             list the paths the data offers, each written after {/ in a template, then exit;
+                         --repeat, --separator, --format and --table do nothing here
+      --no-shipped-data  load only the --data-path directories
+  -n, --repeat N         render the template N times, 0..1048576 (default 1); 0 prints nothing,
+                         [] under --format json, or the header under --format csv
+  -s, --seed N           same seed, version and data: identical output (default: a random seed)
+      --separator S      string between repeated renders (default empty); refused with a record
+                         --format, which has its own separators
+      --table T          the INSERT target for --format sql, ignored under any other
+                         (default: a lone reference's last segment, else records)
+      --version          print the version, then exit
+
+A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
+-- ends the flags.
+
+Exit codes: 0 success, 1 runtime error, 2 misuse.
+`
+
+type invocation struct {
+	args         []string
+	dirs         []string
+	format       string
+	help         bool
+	list         bool
+	noShipped    bool
+	repeat       int
+	seed         uint64
+	seeded       bool
+	separator    string
+	separatorSet bool
+	table        string
+	tableSet     bool
+	version      bool
+}
+
+type flagDef struct {
+	long  string
+	short string
+	value bool
+	set   func(*invocation, string) error
+}
+
+var flagDefs = []flagDef{
+	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
+	{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
+	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
+	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
+	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
+	{"repeat", "n", true, func(in *invocation, v string) error {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n > fejkdata.MaxRepeat {
+			return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
+		}
+		in.repeat = n
+		return nil
+	}},
+	{"seed", "s", true, func(in *invocation, v string) error {
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
+		}
+		in.seed, in.seeded = n, true
+		return nil
+	}},
+	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
+	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
+	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
+}
+
+func flagByLong(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].long == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+func flagByShort(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].short == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+// flagArg is one flag as written: its definition and the value attached to it,
+// if any.
+type flagArg struct {
+	def    *flagDef
+	value  string
+	inline bool
+}
+
+// splitFlags reads one argv element as flags: --name or --name=value, or -abc
+// where each letter is a short flag and the first that takes a value takes the
+// rest of the element.
+func splitFlags(arg string) ([]flagArg, error) {
+	if strings.HasPrefix(arg, "--") {
+		name, value, inline := strings.Cut(arg[2:], "=")
+		def := flagByLong(name)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag --%s", name)
+		}
+		return []flagArg{{def, value, inline}}, nil
+	}
+	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
+		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
+	}
+	var flags []flagArg
+	letters := arg[1:]
+	for i, r := range letters {
+		letter := string(r)
+		def := flagByShort(letter)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag -%s", letter)
+		}
+		if !def.value {
+			flags = append(flags, flagArg{def: def})
+			continue
+		}
+		rest := letters[i+len(letter):]
+		if strings.HasPrefix(rest, "=") {
+			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
+				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
+		}
+		return append(flags, flagArg{def, rest, rest != ""}), nil
+	}
+	return flags, nil
+}
+
+func parseArgs(argv []string) (invocation, error) {
+	in := invocation{repeat: 1, format: "text"}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		if arg == "--" {
+			in.args = append(in.args, argv[i+1:]...)
+			return in, nil
+		}
+		if len(arg) < 2 || arg[0] != '-' {
+			in.args = append(in.args, arg)
+			continue
+		}
+		flags, err := splitFlags(arg)
+		if err != nil {
+			return in, err
+		}
+		for _, fl := range flags {
+			if !fl.def.value {
+				if fl.inline {
+					return in, fmt.Errorf("--%s takes no value", fl.def.long)
+				}
+				_ = fl.def.set(&in, "")
+				continue
+			}
+			value := fl.value
+			if !fl.inline {
+				if i++; i >= len(argv) {
+					return in, fmt.Errorf("--%s needs a value", fl.def.long)
+				}
+				value = argv[i]
+			}
+			if err := fl.def.set(&in, value); err != nil {
+				return in, err
+			}
+		}
+	}
+	return in, nil
+}
+
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
+type recordFormat struct {
+	header func(*fejkdata.Record) string
+	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
+	writes string // what it writes between values, which --separator would seem to set
+}
+
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson writes one object per line.
+var recordFormats = map[string]recordFormat{
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
+	"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
+}
+
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
+func (in invocation) writesRecords() bool {
+	return in.format != "text"
+}
+
+// formatNames lists the --format values, text included, for the misuse error.
+func formatNames() string {
+	names := []string{"text"}
+	for name := range recordFormats {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
+}
+
+// checkFlags rejects a flag value or combination that cannot run.
+func (in invocation) checkFlags() error {
+	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
+		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
+	}
+	if in.tableSet && in.table == "" && in.format == "sql" {
+		return errors.New("--table names the INSERT target, so it cannot be empty")
+	}
+	if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
+		return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
+	}
+	return nil
+}
+
+func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
+
+// operandError refuses an argument that is no flag, naming the spelling that reads what
+// it most likely means: dropping it for -, a file for a file's name, else a template
+// piped in.
+func operandError(arg, name string) error {
+	unexpected := "unexpected argument '" + arg + "'"
+	if arg == "-" {
+		return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
+	}
+	if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
+		return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
+	}
+	ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
+	_, argLone := loneReference(arg)
+	inline, err := fejkdata.IsTemplate(arg)
+	_, refLone := loneReference(ref)
+	example := pipeHint(name)
+	switch {
+	case argLone || inline:
+		example = piped(arg, name)
+	case err == nil && refLone:
+		example = piped(ref, name)
+	}
+	return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
+}
+
+// check rejects a flag combination or an operand that cannot run, before any data is
+// read.
+func (in invocation) check(name string) error {
+	if err := in.checkFlags(); err != nil {
+		return err
+	}
+	if len(in.args) > 0 {
+		return operandError(in.args[0], name)
+	}
+	return nil
+}
+
+func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
+	if in.noShipped {
+		modules = nil
+	}
+	opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
+	for _, dir := range in.dirs {
+		opts = append(opts, fejkdata.WithDataPath(dir))
+	}
+	seed := in.seed
+	if !in.seeded {
+		seed = systemSeed
+	}
+	return append(opts, fejkdata.WithSeed(seed))
+}
+
+// write streams the input's renders to w, repeat of them joined by the separator; a write
+// failure surfaces from Flush, bufio keeping the first one.
+func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
+	if in.writesRecords() {
+		return in.writeRecords(f, src, w)
+	}
+	t, err := f.NewTemplate(src.template)
+	if err != nil {
+		return templateError(err)
+	}
+	out := bufio.NewWriter(w)
+	for i := 0; i < in.repeat; i++ {
+		if i > 0 {
+			out.WriteString(in.separator)
+		}
+		out.WriteString(t.Fake())
+	}
+	return out.Flush()
+}
+
+// recordStream builds the record drawer for the input, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
+	record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
+	if src.record == "" {
+		t, err := f.NewRecordTemplate(src.template)
+		if err != nil {
+			return nil, "", templateError(spacedReference(src.template, err))
+		}
+		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
+	}
+	table := in.table
+	if table == "" {
+		table = defaultTable(src.record)
+	}
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
+	record, table, err := in.recordStream(f, src)
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	if in.repeat == 0 {
+		return writeNoRecord(record, format, w)
+	}
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
+		r, err := record()
+		if err != nil {
+			return err
+		}
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
+		}
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
+}
+
+// writeNoRecord writes zero records: the header where the format writes one, an empty document
+// where it frames one, else nothing. It draws one record, which names the header's columns, so a
+// lone reference to nothing fails as it would with records.
+func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
+	r, err := record()
+	switch {
+	case err != nil:
+		return err
+	case format.header != nil:
+		_, err = io.WriteString(w, format.header(r)+"\n")
+	case format.open != "":
+		_, err = io.WriteString(w, format.open+format.close+"\n")
+	}
+	return err
+}
+
+// defaultTable names the INSERT target when --table is absent: the path's last
+// segment, or "records" when stdin is not a lone reference.
+func defaultTable(path string) string {
+	if path == "" {
+		return "records"
+	}
+	segments := strings.Split(withoutSelectors(path), ".")
+	return segments[len(segments)-1]
+}
+
+// withoutSelectors is path with its [selectors] cut out.
+func withoutSelectors(path string) string {
+	var names strings.Builder
+	for depth, i := 0, 0; i < len(path); i++ {
+		switch {
+		case path[i] == '[':
+			depth++
+		case path[i] == ']':
+			depth--
+		case depth == 0:
+			names.WriteByte(path[i])
+		}
+	}
+	return names.String()
+}
+
+// inputError marks a failure that is the input's own fault — nothing piped in, a template
+// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
+// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
+// error (exit 1).
+type inputError struct{ error }
+
+func (e inputError) Unwrap() error { return e.error }
+
+// jsonSpace is the whitespace JSON allows around a value.
+const jsonSpace = " \t\r\n"
+
+// input is what stdin holds: the template, and the path a lone reference names as a
+// record.
+type input struct {
+	template string
+	record   string
+}
+
+// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
+// reference once JSON's surrounding whitespace, or one final newline, is cut.
+func parseInput(raw string) input {
+	trimmed := strings.Trim(raw, jsonSpace)
+	text := raw
+	if json.Valid([]byte(trimmed)) {
+		text = trimmed
+	}
+	text, found := strings.CutSuffix(text, "\n")
+	if found {
+		text = strings.TrimSuffix(text, "\r")
+	}
+	record, _ := loneReference(text)
+	return input{template: raw, record: record}
+}
+
+// loneReference returns the path that text reads when it is one reference from the root
+// and nothing else, bare or as a JSON string.
+func loneReference(text string) (string, bool) {
+	var unquoted string
+	if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
+		text = unquoted
+	}
+	path, opens := strings.CutPrefix(text, "{/")
+	path, closes := strings.CutSuffix(path, "}")
+	if !opens || !closes || !isPath(path) {
+		return "", false
+	}
+	return path, true
+}
+
+// isPath reports whether a reference's body after its / is a path alone: no second /, no
+// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
+// It copies the grammar, since the CLI reads only the library's public API.
+func isPath(body string) bool {
+	if body == "" || strings.ContainsAny(body[:1], "/.") {
+		return false
+	}
+	names := withoutSelectors(body)
+	if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
+		return false
+	}
+	inline, err := fejkdata.IsTemplate(body)
+	return !inline && err == nil
+}
+
+// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
+// step up, and only a step up may end the path.
+func hasEmptySegment(path string) bool {
+	segments := strings.Split(path, ".")
+	stepUp := false
+	for i, seg := range segments[1:] {
+		last := i == len(segments)-2
+		switch {
+		case seg != "":
+			stepUp = false
+		case stepUp != last:
+			return true
+		default:
+			stepUp = !stepUp
+		}
+	}
+	return false
+}
+
+// spacedReference adds to a record refusal that whitespace around a lone reference made
+// the template text.
+func spacedReference(template string, err error) error {
+	if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
+		if path, lone := loneReference(trimmed); lone {
+			return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
+		}
+	}
+	return err
+}
+
+// piped is the command piping template into the command called name; some shells' echo
+// reads a backslash as an escape, so a template holding one goes through printf.
+func piped(template, name string) string {
+	if strings.ContainsRune(template, '\\') {
+		return "printf '%s' " + shellQuoted(template) + " | " + name
+	}
+	return "echo " + shellQuoted(template) + " | " + name
+}
+
+// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
+func shellQuoted(s string) string {
+	if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
+		return s
+	}
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// Main runs the command on the process's arguments and standard streams over modules, each
+// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
+// 2 misuse. --no-shipped-data leaves modules out.
+func Main(modules ...fs.FS) int {
+	var stdin io.Reader = os.Stdin
+	if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+		stdin = nil
+	}
+	return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
+}
+
+// commandName is the name the command runs under: the file arg0 names, less a Windows
+// .exe, or fejkdata where arg0 names none.
+func commandName(arg0 string) string {
+	name := filepath.Base(arg0)
+	if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
+		name = strings.TrimSuffix(name, ext)
+	}
+	if name == "." || name == "" {
+		return "fejkdata"
+	}
+	return name
+}
+
+// run is Main over the given streams, run as the command name, seeded by systemSeed where
+// no --seed is given. A nil stdin is a terminal, which holds no template.
+func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
+	in, err := parseArgs(args)
+	if err != nil {
+		return misuse(stderr, name, err)
+	}
+	if in.help {
+		io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
+		return 0
+	}
+	if in.version {
+		info, _ := debug.ReadBuildInfo()
+		fmt.Fprintln(stdout, versionLine(name, info))
+		return 0
+	}
+	if err := in.check(name); err != nil {
+		return misuse(stderr, name, err)
+	}
+	var src input
+	if !in.list {
+		if src, err = readStdin(stdin, name); err != nil {
+			return fail(stderr, name, err)
+		}
+	}
+	f, err := fejkdata.New(in.options(modules, systemSeed)...)
+	if err != nil {
+		return fail(stderr, name, err)
+	}
+	if in.list {
+		for _, p := range f.List() {
+			fmt.Fprintln(stdout, p)
+		}
+		return 0
+	}
+	if err := in.write(f, src, stdout); err != nil {
+		return fail(stderr, name, err)
+	}
+	return 0
+}
+
+func readStdin(stdin io.Reader, name string) (input, error) {
+	if stdin == nil {
+		return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
+	}
+	return parseInput(string(raw)), nil
+}
+
+// templateError is misuse unless the data the template reads failed to load.
+func templateError(err error) error {
+	if errors.Is(err, fejkdata.ErrLoad) {
+		return err
+	}
+	return inputError{err}
+}
+
+func fail(stderr io.Writer, name string, err error) int {
+	var te inputError
+	if errors.As(err, &te) {
+		return misuse(stderr, name, te.error)
+	}
+	report(stderr, name, err)
+	if errors.Is(err, fejkdata.ErrNoColumns) {
+		fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
+	}
+	return 1
+}
+
+func misuse(stderr io.Writer, name string, err error) int {
+	report(stderr, name, err)
+	fmt.Fprintf(stderr, "try '%s --help'\n", name)
+	return 2
+}
+
+// report writes err under the command name, in place of the library's own on each line.
+func report(stderr io.Writer, name string, err error) {
+	msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
+	fmt.Fprintf(stderr, "%s: %s\n", name, msg)
+}
+
+// fejkdataModule is the path of the module holding the library; the root package's path is
+// the module's.
+var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
+
+// versionLine names the fejkdata version the binary is built with. Where another module's
+// main package carries the CLI, the main module's version comes first, under the command name.
+func versionLine(name string, info *debug.BuildInfo) string {
+	if info == nil {
+		return "fejkdata devel"
+	}
+	if info.Main.Path == fejkdataModule {
+		return "fejkdata " + moduleVersion(info.Main)
+	}
+	for _, m := range info.Deps {
+		if m.Path == fejkdataModule {
+			return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
+		}
+	}
+	return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
+}
+
+// moduleVersion is the version m is built at: the replacement's where a replace directive
+// names one, and "devel" where it carries none, as a local directory does.
+func moduleVersion(m debug.Module) string {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Version == "" || m.Version == "(devel)" {
+		return "devel"
+	}
+	return m.Version
+}

cli/cli_test.go

@@ -0,0 +1,961 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// shipped holds the shipped modules as read from the checkout, each folder under ../data
+// holding a manifest, so these tests link no data package.
+var shipped = func() []fs.FS {
+	var modules []fs.FS
+	err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
+		if err == nil && d.Name() == ".fejkdata.json" {
+			modules = append(modules, os.DirFS(filepath.Dir(path)))
+		}
+		return err
+	})
+	if err != nil || len(modules) == 0 {
+		panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
+	}
+	return modules
+}()
+
+// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
+// layers its categories unprefixed with -d.
+var svSE, enUS, misc string
+
+func TestMain(m *testing.M) {
+	tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
+	if err != nil {
+		panic(err)
+	}
+	svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
+	code := m.Run()
+	os.RemoveAll(tmp)
+	os.Exit(code)
+}
+
+// bareCopy copies the category and rows files of the shipped module in data/<module> into a
+// folder under tmp, and returns that folder.
+func bareCopy(tmp, module string) string {
+	src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
+	if err := os.Mkdir(dst, 0o755); err != nil {
+		panic(err)
+	}
+	for _, pattern := range []string{"*.json", "*.tsv"} {
+		files, err := filepath.Glob(filepath.Join(src, pattern))
+		if err != nil {
+			panic(err)
+		}
+		for _, f := range files {
+			if strings.HasPrefix(filepath.Base(f), ".") {
+				continue
+			}
+			b, err := os.ReadFile(f)
+			if err == nil {
+				err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
+			}
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	return dst
+}
+
+func runOut(stdin string, args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+// runTerminal runs as from a shell with nothing piped in.
+func runTerminal(args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
+	module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
+		t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
+	}
+	out.Reset()
+	if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
+		t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
+	}
+}
+
+func TestRunOutputsValue(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if out == "" || strings.Contains(out, "\n") {
+		t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
+	}
+}
+
+func TestRunDotPath(t *testing.T) {
+	code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person run = %d", code)
+	}
+	code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(last) == "" {
+		t.Fatal("empty person.last output")
+	}
+	if last == full {
+		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
+}
+
+func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
+	var drawn, seeded, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
+		t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
+		t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if drawn.String() != seeded.String() {
+		t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
+	}
+}
+
+func TestRunSeedSpellings(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"--seed=42", "--data-path", svSE},
+		{"-s", "42", "-d", svSE},
+		{"--data-path", svSE, "--seed", "42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+}
+
+func TestRunShortFlagValues(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"-s42", "-d", svSE},
+		{"-s", "42", "-d" + svSE},
+		{"-d", svSE, "-s42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+	_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
+	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
+		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-s=42", "-d", svSE}, "-s42"},
+		{[]string{"-s=42", "-d", svSE}, "--seed=42"},
+		{[]string{"-d=" + svSE}, "--data-path="},
+		{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
+		{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
+	} {
+		code, out, errb := runOut("{/word}", c.args...)
+		if code != 2 || out != "" {
+			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+	code, out, _ := runOut("", "-hd", svSE)
+	if code != 0 || !strings.Contains(out, "Usage") {
+		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
+	}
+}
+
+func TestRunDoubleDashEndsFlags(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
+	if code != 2 || !strings.Contains(errb, "stdin") {
+		t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSingleDashLongIsRejected(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
+		{[]string{"-seed=42", "-d", svSE}, "use --seed"},
+		{[]string{"-data-path", svSE}, "use --data-path"},
+		{[]string{"-list", "-d", svSE}, "use --list"},
+		{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
+		{[]string{"-x", "-d", svSE}, "unknown flag -x"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunFlagValues(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
+		{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
+		{[]string{"--seed", "x", "-d", svSE}, "--seed"},
+		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
+		{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
+		{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunRepeat(t *testing.T) {
+	code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
+	}
+	_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
+	if short != out {
+		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
+	}
+}
+
+func TestRunSeparator(t *testing.T) {
+	code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if n := strings.Count(out, "\n"); n != 0 {
+		t.Errorf("want no newline, got %d: %q", n, out)
+	}
+	if !strings.Contains(out, ",") {
+		t.Errorf("values should be comma-joined: %q", out)
+	}
+}
+
+func TestRunRepeatAdvancesRNG(t *testing.T) {
+	code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	uniq := map[string]bool{}
+	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+		uniq[l] = true
+	}
+	if len(uniq) < 2 {
+		t.Errorf("repeat should vary output, all identical: %q", out)
+	}
+}
+
+func TestRunMisuse(t *testing.T) {
+	for _, c := range []struct {
+		stdin string
+		args  []string
+	}{
+		{"{/person}", []string{"-d", svSE, "word"}},
+		{"", []string{"-d", svSE, "--list", "person"}},
+		{"{/sv_SE.person}", []string{"--no-shipped-data"}},
+	} {
+		code, out, errb := runOut(c.stdin, c.args...)
+		if code != 2 {
+			t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
+		}
+		if !strings.Contains(errb, "try 'fejkdata --help'") {
+			t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
+		}
+		if out != "" {
+			t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
+		}
+	}
+}
+
+func TestRunTakesNoOperand(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "users.tmpl")
+	if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
+		{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
+		{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
+		{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
+		{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
+		{[]string{"-"}, "drop it"},
+		{[]string{file}, "fejkdata < " + file},
+	} {
+		code, out, errb := runOut("", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+type failingReader struct{}
+
+func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
+
+func TestRunUnreadableStdinIsMisuse(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
+		t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
+	}
+}
+
+func TestRunTerminalStdinIsMisuse(t *testing.T) {
+	code, out, errb := runTerminal("--seed", "1")
+	if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
+		t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
+	}
+	for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
+		if code, out, errb := runTerminal(args...); code != 0 || out == "" {
+			t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
+		}
+	}
+}
+
+func TestRunRendersExactlyWhatIsSent(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"hihi":                           "hihi",
+		"hihi\n":                         "hihi\n",
+		"hihi\r\n":                       "hihi\r\n",
+		"hihi\n\n":                       "hihi\n\n",
+		"{{x}}\n":                        "{x}\n",
+		"\"a\"\n":                        "a",
+		" \t\"a\"\r\n\n":                 "a",
+		"{/misc.territory[SE].alpha2}":   "SE",
+		"{/misc.territory[SE].alpha2}\n": "SE\n",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
+		}
+	}
+	if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
+		t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
+	}
+	if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
+		t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
+	}
+}
+
+func TestRunRendersBlankStdin(t *testing.T) {
+	for _, stdin := range []string{"", "\n", " \r\n\t"} {
+		if code, out, errb := runOut(stdin); code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunTextPrintsAsWritten(t *testing.T) {
+	for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
+		code, out, errb := runOut(stdin)
+		if code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunMultipleDataPaths(t *testing.T) {
+	code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(out) == "" {
+		t.Fatal("empty output for multi-dir run")
+	}
+}
+
+func TestRunUnknownCategoryFails(t *testing.T) {
+	code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if !strings.Contains(errb, "nope") {
+		t.Errorf("stderr %q should name the unknown category", errb)
+	}
+}
+
+func TestRunList(t *testing.T) {
+	code, out, errb := runOut("", "--data-path", svSE, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	for _, want := range []string{"person", "person.last", "address"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("list output missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRunHelpOnStdout(t *testing.T) {
+	for _, h := range []string{"-h", "--help"} {
+		code, out, errb := runOut("", h)
+		if code != 0 {
+			t.Errorf("%s = %d, want 0", h, code)
+		}
+		if !strings.Contains(out, "Usage") || errb != "" {
+			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
+		}
+	}
+}
+
+func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
+	fejkdata := func(version string, replace *debug.Module) *debug.Module {
+		return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
+	}
+	mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
+	for _, c := range []struct {
+		info *debug.BuildInfo
+		want string
+	}{
+		{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
+		{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
+		{nil, "fejkdata devel"},
+	} {
+		if got := versionLine("mycli", c.info); got != c.want {
+			t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
+		}
+	}
+}
+
+func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
+	for arg0, want := range map[string]string{
+		"/usr/local/bin/mycli": "mycli",
+		"mycli.EXE":            "mycli",
+		"mycli.exe":            "mycli",
+		"my.cli":               "my.cli",
+		"":                     "fejkdata",
+	} {
+		if got := commandName(arg0); got != want {
+			t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
+		}
+	}
+}
+
+func TestRunNamesItselfByTheCommandName(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
+		t.Errorf("mycli --nope = %d, %q", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
+		t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
+	}
+	if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
+		t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	code, out, errb := runOut("", "--version")
+	if code != 0 {
+		t.Fatalf("--version = %d, stderr=%q", code, errb)
+	}
+	version, ok := strings.CutPrefix(out, "fejkdata ")
+	if !ok || strings.TrimSpace(version) == "" {
+		t.Errorf("--version = %q, want the command name and a version on stdout", out)
+	}
+}
+
+func TestRunMissingDirFails(t *testing.T) {
+	code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if errb == "" {
+		t.Error("want an error message on stderr")
+	}
+}
+
+func TestRunShippedDataByDefault(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, list, _ := runOut("", "--list")
+	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
+		t.Errorf("--list without --data-path = %d, %q", code, list)
+	}
+	code, _, errb = runOut("{/person}")
+	if code != 2 || !strings.Contains(errb, "person") {
+		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSelectsATableRow(t *testing.T) {
+	dir := t.TempDir()
+	for name, content := range map[string]string{
+		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
+		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
+		t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
+	}
+	if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
+		t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
+	}
+	if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
+		t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
+	}
+	if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
+		t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
+	}
+	if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
+		t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
+	}
+}
+
+func TestUsageExamplesRenderWithNoData(t *testing.T) {
+	examples := regexp.MustCompile(`(?m)^  echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n  "+pipeHint("COMMAND")+"\n", -1)
+	if len(examples) < 3 {
+		t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
+	}
+	for _, e := range examples {
+		if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
+			t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
+		}
+	}
+	for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
+		if token != "{/path}" && token != "{/table[KEY]}" {
+			t.Errorf("usage names %s, a path a custom CLI may not carry", token)
+		}
+	}
+	if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
+		t.Errorf("usage does not list the exit codes")
+	}
+}
+
+func TestRunOperandMisuseBeforeLoad(t *testing.T) {
+	code, _, errb := runOut("", "--no-shipped-data", "x")
+	if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
+		t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
+	}
+}
+
+func TestRunInlineTemplate(t *testing.T) {
+	code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
+	if code != 0 || (out != "name: bosse" && out != "name: lina") {
+		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
+	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
+		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
+	if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
+		t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
+	}
+}
+
+func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
+	for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
+		if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
+			t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
+		}
+	}
+	if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
+		t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
+	}
+}
+
+func TestRunTemplateMisuse(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"{bad":                  "unterminated",
+		"name: {/no.such.path}": "no entry",
+		"a } b":                 "}}",
+		"{//sv_SE.person}":      "write {/sv_SE.person}",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
+			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
+		}
+		if strings.Contains(errb, "fejkdata: fejkdata:") {
+			t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
+		}
+	}
+}
+
+func TestRunNoShippedData(t *testing.T) {
+	code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
+		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
+	}
+	code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Errorf("run = %d, out=%q", code, out)
+	}
+	if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
+		t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
+	}
+	if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
+		t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
+	}
+}
+
+func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{nil, ""},
+		{[]string{"--format", "json"}, "[]\n"},
+		{[]string{"--format", "ndjson"}, ""},
+		{[]string{"--format", "csv"}, "first,last\n"},
+		{[]string{"--format", "sql"}, ""},
+	} {
+		if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
+			t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
+		}
+	}
+	if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
+		t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
+	}
+}
+
+func TestRunRepeatIsBounded(t *testing.T) {
+	for _, args := range [][]string{
+		{"--repeat", "9223372036854775807"},
+		{"--repeat", "1048577"},
+		{"-n", "99999999999"},
+	} {
+		code, out, errb := runOut("{/sv_SE.word}", args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
+			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
+		}
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
+	if code != 0 || len(out) != 1048576 {
+		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
+	}
+}
+
+func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		stdin         string
+		base, ignored []string
+	}{
+		{"", []string{"--list"}, []string{"-n", "3"}},
+		{"", []string{"--list"}, []string{"--separator", ","}},
+		{"", []string{"--list"}, []string{"--format", "json"}},
+		{"", []string{"--list"}, []string{"--table", "t"}},
+		{"{/users}", nil, []string{"--table", "t"}},
+		{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
+	} {
+		args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
+		_, want, _ := runOut(c.stdin, args...)
+		code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
+		if code != 0 || want == "" || got != want {
+			t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
+		}
+	}
+}
+
+func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
+	if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
+		t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
+	}
+}
+
+func TestRunRendersAnyJSONValue(t *testing.T) {
+	for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
+		if code, out, errb := runOut(stdin); code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
+		}
+	}
+}
+
+func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "-ä")
+	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
+		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
+	}
+}
+
+func TestRunEmptyDataPathIsNamed(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
+	if code != 1 || !strings.Contains(errb, "empty") {
+		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
+	}
+}
+
+func recordDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
+	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestRunRecordJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
+		}
+	}
+}
+
+func TestRunRecordCSVRoundTrips(t *testing.T) {
+	code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 4 {
+		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
+	}
+	if lines[0] != "first,last" {
+		t.Fatalf("csv header = %q, want first,last", lines[0])
+	}
+	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
+	if err != nil {
+		t.Fatalf("csv output is not valid CSV: %v", err)
+	}
+	if len(r) != 4 || len(r[0]) != 2 {
+		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
+	}
+}
+
+func TestRunRecordSQL(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
+		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
+	}
+}
+
+func TestRunRecordDefaultTable(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
+	}
+}
+
+func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
+	code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
+	}
+	code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
+	if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
+		t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
+	}
+}
+
+func TestRunRecordInlineTemplate(t *testing.T) {
+	code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
+	if code != 0 {
+		t.Fatalf("inline record = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
+	}
+}
+
+func TestRunRecordMisuse(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
+		{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
+		{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
+		{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
+	} {
+		code, out, errb := runOut("{/users}", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
+	if code != 1 {
+		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
+	}
+}
+
+func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
+	dir := t.TempDir()
+	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
+	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
+	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
+	}
+	code, _, errb = runOut(content, "--format", "json")
+	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
+	}
+	if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
+		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
+	}
+}
+
+func TestRunNumbersFollowTheShell(t *testing.T) {
+	_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
+	code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
+	if code != 0 || got != want {
+		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	}
+}
+
+// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
+// for columns: the record to write, and the quoting a shell needs to take it. The
+// template is well-formed and only the data cannot serve it, so the code is 1.
+func TestRunRecordOfAFieldlessPath(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
+	}
+	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
+		if !strings.Contains(errb, want) {
+			t.Errorf("stderr = %q, want it to name %s", errb, want)
+		}
+	}
+	if out != "" {
+		t.Errorf("stdout = %q, want nothing", out)
+	}
+}
+
+func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
+	dir := t.TempDir()
+	for name, data := range map[string]string{
+		".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
+		"bad.json":       `"{/nope}"`,
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
+		if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
+			t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
+		}
+	}
+}

cli/lone_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// grammarLoneReference is the path a format reads when the grammar parses it as one path
+// read of one reference from the root, and nothing else.
+func grammarLoneReference(text string) (string, bool) {
+	toks, err := grammar.ParseFormat(text)
+	if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
+		return "", false
+	}
+	sigil, path, err := grammar.RefShape(toks[0].Arms[0])
+	if err != nil || sigil != "/" {
+		return "", false
+	}
+	return path, true
+}
+
+func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
+	for _, text := range []string{
+		" {/x}",
+		"x",
+		"{.x}",
+		"{/.x}",
+		"{//x}",
+		"{/a...b}",
+		"{/a..b}",
+		"{/a..}",
+		"{/a.[x]}",
+		"{/a.}",
+		"{/a|/b}",
+		"{/a}{/b}",
+		"{/misc.currency[US Dollar (Next day)]}",
+		"{/sv_SE.person}",
+		"{/t[a as b].c}",
+		"{/t[f(x)]}",
+		"{/x as p}",
+		"{/x(1)}",
+		"{/x}}",
+		"{/}",
+	} {
+		path, lone := loneReference(text)
+		want, wantLone := grammarLoneReference(text)
+		if lone != wantLone || path != want {
+			t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
+		}
+	}
+}

cmd/fejkdata/main.go

@@ -1,512 +1,12 @@
-// Command fejkdata prints fake values from the shipped data and any directories
+// Command fejkdata renders the template on its stdin with fake values from every shipped
-// layered over it.
+// module; package cli documents it.
-//
-//	fejkdata sv_SE.person                        # a full person
-//	fejkdata sv_SE.person.last                   # just the surname
-//	fejkdata --data-path ./mydata sv_SE.person   # layer custom data; the last dir wins
-//	fejkdata --seed 42 sv_SE.address
 package main
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"runtime/debug"
-	"sort"
-	"strconv"
-	"strings"
 
-	"gitea.larvit.se/larvit/fejkdata"
+	"github.com/larvit/fejkdata/cli"
+	"github.com/larvit/fejkdata/data"
 )
 
-const usage = `Usage: fejkdata [flags] <path|template>
+func main() { os.Exit(cli.Main(data.Modules()...)) }
-
-  <path>                 a category, or a dotted path into one (person, person.last);
-                         a table's row by key or name: 'misc.country[SE]', 'misc.country[Sweden].tld'
-  <template>             a format string or JSON value to render inline, e.g.
-                         'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
-
-A layout inside a template is single-quoted, so quote the whole argument with " to
-keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
-
-An argument containing a { token, or a JSON object, array or string, is a
-template; any other argument is a path (a path never contains a brace or a quote,
-and a bracket only as a [selector] after a table's name). Templates reach the
-data by reference from the root — {/sv_SE.person.last} — whether the data is
-shipped or layered with --data-path. An argument carrying a closing brace or a
-quote but no valid JSON names neither.
-
-With --format json, ndjson, csv or sql the argument must name a record — a
-template whose fields are its columns — and the rows are written as one JSON
-array, one JSON object per line, one CSV row (after a header), or one INSERT.
-
-  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, ndjson, csv or sql
-  -h, --help             print this help, then exit
-      --list             list the paths the data offers, then exit
-      --no-shipped-data  load only the --data-path directories
-  -n, --repeat N         render the value N times, 1..1048576 (default 1)
-  -s, --seed N           seed for reproducible output
-      --separator S      string between repeated values (default newline)
-      --table T          the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
-      --version          print the version, then exit
-
-Flags may come before or after <path|template>; -- ends the flags. A short flag's
-value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
-`
-
-type invocation struct {
-	dirs         []string
-	format       string
-	formatSet    bool
-	help         bool
-	list         bool
-	noShipped    bool
-	paths        []string
-	repeat       int
-	repeatSet    bool
-	seed         uint64
-	seeded       bool
-	separator    string
-	separatorSet bool
-	table        string
-	tableSet     bool
-	version      bool
-}
-
-type flagDef struct {
-	long  string
-	short string
-	value bool
-	set   func(*invocation, string) error
-}
-
-var flagDefs = []flagDef{
-	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
-	{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
-	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
-	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
-	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
-	{"repeat", "n", true, func(in *invocation, v string) error {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > fejkdata.MaxRepeat {
-			return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
-		}
-		in.repeat, in.repeatSet = n, true
-		return nil
-	}},
-	{"seed", "s", true, func(in *invocation, v string) error {
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
-		}
-		in.seed, in.seeded = n, true
-		return nil
-	}},
-	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
-	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
-	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
-}
-
-func flagByLong(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].long == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-func flagByShort(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].short == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-// flagArg is one flag as written: its definition and the value attached to it,
-// if any.
-type flagArg struct {
-	def    *flagDef
-	value  string
-	inline bool
-}
-
-// splitFlags reads one argv element as flags: --name or --name=value, or -abc
-// where each letter is a short flag and the first that takes a value takes the
-// rest of the element.
-func splitFlags(arg string) ([]flagArg, error) {
-	if strings.HasPrefix(arg, "--") {
-		name, value, inline := strings.Cut(arg[2:], "=")
-		def := flagByLong(name)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag --%s", name)
-		}
-		return []flagArg{{def, value, inline}}, nil
-	}
-	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
-		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
-	}
-	var flags []flagArg
-	letters := arg[1:]
-	for i, r := range letters {
-		letter := string(r)
-		def := flagByShort(letter)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag -%s", letter)
-		}
-		if !def.value {
-			flags = append(flags, flagArg{def: def})
-			continue
-		}
-		rest := letters[i+len(letter):]
-		if strings.HasPrefix(rest, "=") {
-			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
-				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
-		}
-		return append(flags, flagArg{def, rest, rest != ""}), nil
-	}
-	return flags, nil
-}
-
-func parseArgs(argv []string) (invocation, error) {
-	in := invocation{repeat: 1, separator: "\n", format: "text"}
-	for i := 0; i < len(argv); i++ {
-		arg := argv[i]
-		if arg == "--" {
-			in.paths = append(in.paths, argv[i+1:]...)
-			return in, nil
-		}
-		if len(arg) < 2 || arg[0] != '-' {
-			in.paths = append(in.paths, arg)
-			continue
-		}
-		flags, err := splitFlags(arg)
-		if err != nil {
-			return in, err
-		}
-		for _, fl := range flags {
-			if !fl.def.value {
-				if fl.inline {
-					return in, fmt.Errorf("--%s takes no value", fl.def.long)
-				}
-				_ = fl.def.set(&in, "")
-				continue
-			}
-			value := fl.value
-			if !fl.inline {
-				if i++; i >= len(argv) {
-					return in, fmt.Errorf("--%s needs a value", fl.def.long)
-				}
-				value = argv[i]
-			}
-			if err := fl.def.set(&in, value); err != nil {
-				return in, err
-			}
-		}
-	}
-	return in, nil
-}
-
-// recordFormat is one way to write a record out: the line each record renders,
-// the header that precedes the first one, and the open/close frame plus the
-// between-record separator a document form needs.
-type recordFormat struct {
-	header func(*fejkdata.Record) string
-	line   func(r *fejkdata.Record, table string) string
-	open   string
-	close  string
-	sep    string
-}
-
-// recordFormats is every --format that writes records. json frames the records
-// as one array document; ndjson is the same column, one object per line.
-var recordFormats = map[string]recordFormat{
-	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
-	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
-	"ndjson": {line: jsonLine, sep: "\n"},
-	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
-}
-
-func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
-
-// writesRecords reports whether the format writes records rather than plain text.
-func (in invocation) writesRecords() bool {
-	return in.format != "text"
-}
-
-// formatNames lists the --format values, text included, for the misuse error.
-func formatNames() string {
-	names := []string{"text"}
-	for name := range recordFormats {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
-}
-
-// checkFlags rejects a flag value or combination that cannot run.
-func (in invocation) checkFlags() error {
-	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
-		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
-	}
-	if in.tableSet && in.table == "" {
-		return errors.New("--table names the INSERT target, so it cannot be empty")
-	}
-	if in.tableSet && in.format != "sql" {
-		return errors.New("--table names the INSERT target, so it needs --format sql")
-	}
-	if in.writesRecords() && in.separatorSet {
-		return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
-	}
-	return nil
-}
-
-// check rejects a flag combination or an argument that cannot run, and reports
-// what the argument names, so its shape is settled before any data is read.
-func (in invocation) check() (argKind, error) {
-	if err := in.checkFlags(); err != nil {
-		return argPath, err
-	}
-	if in.list && len(in.paths) > 0 {
-		return argPath, errors.New("--list takes no path")
-	}
-	if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
-		return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
-	}
-	if in.list {
-		return argPath, nil
-	}
-	if len(in.paths) != 1 {
-		return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
-	}
-	return classify(in.paths[0])
-}
-
-func (in invocation) options() []fejkdata.Option {
-	var opts []fejkdata.Option
-	if in.noShipped {
-		opts = append(opts, fejkdata.WithoutShippedData())
-	}
-	for _, dir := range in.dirs {
-		opts = append(opts, fejkdata.WithDataPath(dir))
-	}
-	if in.seeded {
-		opts = append(opts, fejkdata.WithSeed(in.seed))
-	}
-	return opts
-}
-
-// write streams the argument's renders to w, repeat of them joined by the
-// separator and ended by a newline. A value that renders once renders every time,
-// so a render failure comes before anything is written; a write failure surfaces
-// from Flush, bufio keeping the first one.
-func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	if in.writesRecords() {
-		return in.writeRecords(f, kind, w)
-	}
-	draw, err := in.textDraw(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	out := bufio.NewWriter(w)
-	for i := 0; i < in.repeat; i++ {
-		v, err := draw()
-		if err != nil {
-			return err
-		}
-		if i > 0 {
-			out.WriteString(in.separator)
-		}
-		out.WriteString(v)
-	}
-	out.WriteString("\n")
-	return out.Flush()
-}
-
-// textDraw builds what one text render yields: the value, from a path or an
-// inline template.
-func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if kind != argTemplate {
-		return func() (string, error) { return f.Fake(arg) }, nil
-	}
-	t, err := f.NewTemplate(arg)
-	if err != nil {
-		return nil, templateError{err}
-	}
-	return func() (string, error) { return t.Fake(), nil }, nil
-}
-
-// recordStream builds the record drawer for the argument, plus the INSERT table
-// a sql format names.
-func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
-	record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
-	if kind == argTemplate {
-		t, err := f.NewRecordTemplate(arg)
-		if err != nil {
-			return nil, "", templateError{err}
-		}
-		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
-	}
-	table := in.table
-	if table == "" {
-		table = defaultTable(arg, kind)
-	}
-	return record, table, nil
-}
-
-// writeRecords streams a record per line in the chosen format, framing a document
-// form with its open/close brackets and a header preceding the first record.
-func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	record, table, err := in.recordStream(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	format := recordFormats[in.format]
-	out := bufio.NewWriter(w)
-	if format.open != "" {
-		out.WriteString(format.open)
-		out.WriteByte('\n')
-	}
-	for i := 0; i < in.repeat; i++ {
-		r, err := record()
-		if err != nil {
-			return err
-		}
-		if i == 0 && format.header != nil {
-			out.WriteString(format.header(r))
-			out.WriteByte('\n')
-		}
-		if i > 0 {
-			out.WriteString(format.sep)
-		}
-		out.WriteString(format.line(r, table))
-	}
-	if format.close != "" {
-		out.WriteByte('\n')
-		out.WriteString(format.close)
-	}
-	out.WriteByte('\n')
-	return out.Flush()
-}
-
-// defaultTable names the INSERT target when --table is absent: the path's last
-// segment, or "records" for an inline template that sits in no folder.
-func defaultTable(arg string, kind argKind) string {
-	if kind == argTemplate {
-		return "records"
-	}
-	var names strings.Builder // the path with its [selectors] cut out
-	for depth, i := 0, 0; i < len(arg); i++ {
-		switch {
-		case arg[i] == '[':
-			depth++
-		case arg[i] == ']':
-			depth--
-		case depth == 0:
-			names.WriteByte(arg[i])
-		}
-	}
-	segments := strings.Split(names.String(), ".")
-	return segments[len(segments)-1]
-}
-
-// templateError marks a render failure that is the argument's own fault — an
-// inline template that does not compile. run reports it as misuse (exit 2, with a
-// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
-type templateError struct{ error }
-
-func (e templateError) Unwrap() error { return e.error }
-
-type argKind int
-
-const (
-	argPath argKind = iota
-	argTemplate
-)
-
-// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
-func classify(arg string) (argKind, error) {
-	inline, err := fejkdata.IsTemplate(arg)
-	if inline {
-		return argTemplate, nil
-	}
-	return argPath, err
-}
-
-func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
-
-// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
-func run(args []string, stdout, stderr io.Writer) int {
-	in, err := parseArgs(args)
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	if in.help {
-		fmt.Fprint(stdout, usage)
-		return 0
-	}
-	if in.version {
-		fmt.Fprintln(stdout, "fejkdata "+buildVersion())
-		return 0
-	}
-	kind, err := in.check()
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	f, err := fejkdata.New(in.options()...)
-	if errors.Is(err, fejkdata.ErrNoData) {
-		return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
-	}
-	if err != nil {
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	if in.list {
-		for _, p := range f.List() {
-			fmt.Fprintln(stdout, p)
-		}
-		return 0
-	}
-	if err := in.write(f, kind, stdout); err != nil {
-		var te templateError
-		if errors.As(err, &te) {
-			return misuse(stderr, te.error)
-		}
-		fmt.Fprintln(stderr, err)
-		if errors.Is(err, fejkdata.ErrNoColumns) {
-			fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
-		}
-		return 1
-	}
-	return 0
-}
-
-// templateSplit names the cure for a template a shell split on its spaces, which is
-// what several arguments carrying a token mean.
-func templateSplit(paths []string) string {
-	for _, p := range paths {
-		if strings.Contains(p, "{") {
-			return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
-		}
-	}
-	return ""
-}
-
-func misuse(stderr io.Writer, err error) int {
-	// A library error already names the program, so the prefix is not doubled.
-	fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
-	return 2
-}
-
-// buildVersion is the module version go install stamps into the binary, or "devel".
-func buildVersion() string {
-	if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
-		return info.Main.Version
-	}
-	return "devel"
-}

cmd/fejkdata/main_test.go

@@ -2,640 +2,46 @@ package main
 
 import (
 	"bytes"
-	"encoding/csv"
-	"encoding/json"
 	"os"
-	"path/filepath"
+	"os/exec"
-	"regexp"
 	"strings"
 	"testing"
 )
 
-const (
+func TestMain(m *testing.M) {
-	svSE = "../../data/sv_SE"
+	if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
-	enUS = "../../data/en_US"
+		main()
-	misc = "../../data/misc"
+	}
-)
+	os.Exit(m.Run())
-
-func runOut(args ...string) (int, string, string) {
-	var out, errb bytes.Buffer
-	code := run(args, &out, &errb)
-	return code, out.String(), errb.String()
 }
 
-func TestRunOutputsValue(t *testing.T) {
+func TestMainRendersTheShippedData(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "person")
+	cmd := exec.Command(os.Args[0], "--seed", "1")
-	if code != 0 {
+	cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-		t.Fatalf("run = %d, stderr=%q", code, errb)
+	cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
-	}
+	var stderr bytes.Buffer
-	if strings.TrimSpace(out) == "" {
+	cmd.Stderr = &stderr
-		t.Fatalf("empty output, stderr=%q", errb)
+	out, err := cmd.Output()
-	}
+	if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
-	if !strings.HasSuffix(out, "\n") {
+		t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
-		t.Errorf("output should end with newline, got %q", out)
 	}
 }
 
-func TestRunDotPath(t *testing.T) {
+func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
-	code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
+	var outs [2]string
-	if code != 0 {
+	for i := range outs {
-		t.Fatalf("person run = %d", code)
+		cmd := exec.Command(os.Args[0])
-	}
+		cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
-	code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
+		cmd.Stdin = strings.NewReader("{uuid()}")
-	if code != 0 {
+		var stderr bytes.Buffer
-		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+		cmd.Stderr = &stderr
-	}
+		out, err := cmd.Output()
-	if strings.TrimSpace(last) == "" {
+		if err != nil || len(out) == 0 {
-		t.Fatal("empty person.last output")
+			t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
-	}
+		}
-	if last == full {
+		outs[i] = string(out)
-		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
-	}
+	if outs[0] == outs[1] {
-}
+		t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
-
-func TestRunSeedSpellings(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"--seed=42", "--data-path", svSE, "address"},
-		{"-s", "42", "-d", svSE, "address"},
-		{"--data-path", svSE, "address", "--seed", "42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-}
-
-func TestRunShortFlagValues(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"-s42", "-d", svSE, "address"},
-		{"-s", "42", "-d" + svSE, "address"},
-		{"-d", svSE, "address", "-s42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-	_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
-	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
-		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
-	}
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
-		{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
-		{[]string{"-d=" + svSE, "address"}, "--data-path="},
-		{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
-		{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" {
-			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-	code, out, _ := runOut("-hd", svSE)
-	if code != 0 || !strings.Contains(out, "Usage") {
-		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
-	}
-}
-
-func TestRunDoubleDashEndsFlags(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--", "person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, _, errb = runOut("--data-path", svSE, "--", "--list")
-	if code != 1 || !strings.Contains(errb, "--list") {
-		t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestRunSingleDashLongIsRejected(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-data-path", svSE, "person"}, "use --data-path"},
-		{[]string{"-list", "-d", svSE}, "use --list"},
-		{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
-		{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunFlagValues(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
-		{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
-		{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
-		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
-		{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
-		{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunRepeat(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 3 {
-		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
-	}
-	_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
-	if short != out {
-		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
-	}
-}
-
-func TestRunSeparator(t *testing.T) {
-	code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if n := strings.Count(out, "\n"); n != 1 {
-		t.Errorf("want one trailing newline, got %d: %q", n, out)
-	}
-	if !strings.Contains(out, ",") {
-		t.Errorf("values should be comma-joined: %q", out)
-	}
-}
-
-func TestRunRepeatAdvancesRNG(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	uniq := map[string]bool{}
-	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
-		uniq[l] = true
-	}
-	if len(uniq) < 2 {
-		t.Errorf("repeat should vary output, all identical: %q", out)
-	}
-}
-
-func TestRunMisuse(t *testing.T) {
-	for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
-		code, out, errb := runOut(args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", args, code)
-		}
-		if !strings.Contains(errb, "try 'fejkdata --help'") {
-			t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
-		}
-		if out != "" {
-			t.Errorf("run(%v) stdout = %q, want nothing", args, out)
-		}
-	}
-	// A template a shell split names both quotings, since which one cures it turns
-	// on whether the argument carries a double-quoted JSON.
-	_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
-	for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
-		}
-	}
-}
-
-func TestRunMultipleDataPaths(t *testing.T) {
-	code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatal("empty output for multi-dir run")
-	}
-}
-
-func TestRunUnknownCategoryFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", svSE, "nope")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if !strings.Contains(errb, "nope") {
-		t.Errorf("stderr %q should name the unknown category", errb)
-	}
-}
-
-func TestRunList(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	for _, want := range []string{"person", "person.last", "address"} {
-		if !strings.Contains(out, want) {
-			t.Errorf("list output missing %q:\n%s", want, out)
-		}
-	}
-}
-
-func TestRunHelpOnStdout(t *testing.T) {
-	for _, h := range []string{"-h", "--help"} {
-		code, out, errb := runOut(h)
-		if code != 0 {
-			t.Errorf("%s = %d, want 0", h, code)
-		}
-		if !strings.Contains(out, "Usage") || errb != "" {
-			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
-		}
-	}
-}
-
-func TestRunVersion(t *testing.T) {
-	code, out, errb := runOut("--version")
-	if code != 0 {
-		t.Fatalf("--version = %d, stderr=%q", code, errb)
-	}
-	version, ok := strings.CutPrefix(out, "fejkdata ")
-	if !ok || strings.TrimSpace(version) == "" {
-		t.Errorf("--version = %q, want the command name and a version on stdout", out)
-	}
-}
-
-func TestRunMissingDirFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", "../../data/nope", "person")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if errb == "" {
-		t.Error("want an error message on stderr")
-	}
-}
-
-func TestRunShippedDataByDefault(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "sv_SE.person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, list, _ := runOut("--list")
-	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
-		t.Errorf("--list without --data-path = %d, %q", code, list)
-	}
-	code, _, errb = runOut("person")
-	if code != 1 || !strings.Contains(errb, "person") {
-		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestClassify(t *testing.T) {
-	for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
-		if got, err := classify(arg); err != nil || got != want {
-			t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
-		}
-	}
-	if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
-		t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
-	}
-	if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
-		t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
-	}
-}
-
-func TestRunSelectsATableRow(t *testing.T) {
-	dir := t.TempDir()
-	for name, content := range map[string]string{
-		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
-		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
-	} {
-		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
-		t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
-		t.Fatalf("region[Skåne län].code = %d, %q", code, out)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
-		t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
-		t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
-		t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.country[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
-		t.Fatalf("--format csv misc.country[SE] = %d, %q", code, out)
-	}
-}
-
-func TestUsageReferencesResolve(t *testing.T) {
-	for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
-		path := token[2 : len(token)-1]
-		code, out, errb := runOut("--seed", "1", path)
-		if code != 0 || strings.TrimSpace(out) == "" {
-			t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
-		}
-	}
-}
-
-func TestRunShapeMisuseBeforeLoad(t *testing.T) {
-	code, _, errb := runOut("--no-shipped-data", "[abc]")
-	if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
-		t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
-	}
-}
-
-func TestRunInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
-	if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
-		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
-	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
-		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
-	}
-}
-
-func TestRunTemplateMisuse(t *testing.T) {
-	for arg, want := range map[string]string{
-		"{bad":                  "unterminated",
-		"[red,green]":           `starts with "["`,
-		`{"format":"x"}`:        "is a string",
-		"name: {/no.such.path}": "no entry",
-		"{/sv_SE.person}":       "write sv_SE.person",
-		"x[1]y":                 `"]"`,
-		` ["a","b"] `:           "may not be padded",
-	} {
-		code, out, errb := runOut("--seed", "1", arg)
-		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
-			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
-		}
-		if strings.Contains(errb, "fejkdata: fejkdata:") {
-			t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
-		}
-	}
-}
-
-func TestRunNoShippedData(t *testing.T) {
-	code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
-		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
-	}
-	code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Errorf("run = %d, out=%q", code, out)
-	}
-	code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
-	if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
-		t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
-	}
-}
-
-func TestRunRepeatIsBounded(t *testing.T) {
-	for _, args := range [][]string{
-		{"--repeat", "9223372036854775807", "sv_SE.word"},
-		{"--repeat", "1048577", "sv_SE.word"},
-		{"-n", "99999999999", "sv_SE.word"},
-	} {
-		code, out, errb := runOut(args...)
-		if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
-			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
-		}
-	}
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
-	if code != 0 || len(out) != 1048576+1 {
-		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
-	}
-}
-
-func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
-	for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
-		code, _, errb := runOut(args...)
-		if code != 2 || !strings.Contains(errb, "--list takes no") {
-			t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
-		}
-	}
-}
-
-func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
-	code, _, errb := runOut("-ä", "sv_SE.word")
-	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
-		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
-	}
-}
-
-func TestRunEmptyDataPathIsNamed(t *testing.T) {
-	code, _, errb := runOut("--data-path=", "sv_SE.word")
-	if code != 1 || !strings.Contains(errb, "empty") {
-		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
-	}
-}
-
-func recordDir(t *testing.T) string {
-	t.Helper()
-	dir := t.TempDir()
-	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
-	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
-func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil {
-		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
-	}
-	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
-		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
-	}
-}
-
-func TestRunRecordNDJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
-	}
-	for _, line := range lines {
-		var m map[string]string
-		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
-			t.Fatalf("ndjson line %q is not one object: %v", line, err)
-		}
-	}
-}
-
-func TestRunRecordCSVRoundTrips(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 4 {
-		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
-	}
-	if lines[0] != "first,last" {
-		t.Fatalf("csv header = %q, want first,last", lines[0])
-	}
-	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
-	if err != nil {
-		t.Fatalf("csv output is not valid CSV: %v", err)
-	}
-	if len(r) != 4 || len(r[0]) != 2 {
-		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
-	}
-}
-
-func TestRunRecordSQL(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
-		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
-	}
-}
-
-func TestRunRecordDefaultTable(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
-	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
-		t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
-	}
-}
-
-func TestRunRecordInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
-	if code != 0 {
-		t.Fatalf("inline record = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
-		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
-	}
-}
-
-func TestRunRecordMisuse(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
-		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
-		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
-		}
-	}
-}
-
-func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
-	if code != 1 {
-		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
-	}
-}
-
-func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
-	dir := t.TempDir()
-	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
-	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
-	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
-	}
-	code, _, errb = runOut("--format", "json", content)
-	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
-		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
-	}
-}
-
-func TestRunNumbersFollowTheShell(t *testing.T) {
-	_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
-	code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
-	if code != 0 || got != want {
-		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
-	}
-}
-
-// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
-// for columns: the record to write, and the quoting a shell needs to take it. The
-// argument is well-formed and only the data cannot serve it, so the code is 1.
-func TestRunRecordOfAFieldlessPath(t *testing.T) {
-	code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
-	if code != 1 {
-		t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
-	}
-	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
-		if !strings.Contains(errb, want) {
-			t.Errorf("stderr = %q, want it to name %s", errb, want)
-		}
-	}
-	if out != "" {
-		t.Errorf("stdout = %q, want nothing", out)
 	}
 }

compose.yaml

@@ -1,21 +1,5 @@
-# Docker-based dev workflow — no local tooling beyond Docker is required.
-#
-#   docker compose run --rm test
-#   docker compose run --rm vet
-#   docker compose run --rm build
-#
-# The full gate — vet, formatting and tests — is `docker build .`, which is what
-# CI runs.
-#
-# Build caches persist in the `gocache` volume, so re-runs are fast and the
-# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
-# ownership when run with `--user "$(id -u):$(id -g)"`.
-#
-# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
-# supported Go with: GO_VERSION=1.22 docker compose run --rm test
-
 x-go: &go
-  image: golang:${GO_VERSION:-1.26.4}
+  image: golang:${GO_VERSION:-1.27.2}
   working_dir: /app
   volumes:
     - .:/app
@@ -33,12 +17,16 @@ services:
 
   cover:
     <<: *go
-    command: go test -cover ./...
+    command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
 
   bench:
     <<: *go
     command: go test -run XXX -bench . -benchmem ./...
 
+  generate:
+    <<: *go
+    command: go generate ./...
+
   vet:
     <<: *go
     command: go vet ./...
@@ -47,6 +35,10 @@ services:
     <<: *go
     command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
 
+  vulncheck:
+    <<: *go
+    command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
+
   build:
     <<: *go
     command: go build ./...
@@ -59,13 +51,12 @@ services:
     <<: *go
     command: go mod tidy
 
-  # Interactive shell for ad-hoc work: docker compose run --rm dev
   dev:
     <<: *go
     command: sh
 
   data-import:
-    image: python:3.14.7-slim
+    image: python:3.14.8-slim
     working_dir: /app
     volumes:
       - .:/app

data-import/geo-se.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
 
-    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
+    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import csv
 import io
 import json
-import math
 import os
 import re
 import sys
@@ -16,6 +14,7 @@ import urllib.request
 import zipfile
 from pathlib import Path
 
+import geo
 import source
 import tsv
 import xlsx
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
 POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
 NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
 NVDB_PAGE = 50000
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
-CACHE = Path(__file__).resolve().parent / "cache"
 TIMEZONE = "Europe/Stockholm"
+# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
 ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
+# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
 UNMATCHED_POPULATION = 200
+# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
+# or a description in one.
+NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
+PARTICLES = {"af", "de", "von"}
 
 
 def scb_codes(cache):
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
         yield name, float(lat), float(lon)
 
 
-class Nearest:
-    """Nearest point by an equirectangular distance, over a degree grid."""
-
-    def __init__(self, points, cell=0.05):
-        self.cell = cell
-        self.grid = collections.defaultdict(list)
-        for lat, lon, value in points:
-            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
-
-    def find(self, lat, lon):
-        ci, cj = int(lat // self.cell), int(lon // self.cell)
-        best, best_d = None, math.inf
-        ring = 0
-        while ring < 400:
-            for i in range(ci - ring, ci + ring + 1):
-                for j in range(cj - ring, cj + ring + 1):
-                    if max(abs(i - ci), abs(j - cj)) != ring:
-                        continue
-                    for plat, plon, value in self.grid.get((i, j), ()):
-                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
-                        if d < best_d:
-                            best, best_d = value, d
-            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
-                return best
-            ring += 1
-        return best
-
-
 def street_delivery(name, codes):
-    """The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
+    """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
+    least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
+    for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
+    docs/research/research-geo-se.md, Postal codes → Structure."""
     if name in ONE_POSITION:
         return [c for c in codes if c[1] != "0"]
     largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
 
 
 def well_cased(name):
+    """Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
+    capitals or lower case."""
     return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
 
 
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
         municipality, method = municipality_of(name, rows, tatorter, municipalities)
         how[method] += 1
         kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
-        with_point = [r for r in rows if r["lat"] is not None]
+        point = geo.centroid(rows)
-        if municipality is None or not kept or not with_point or not well_cased(name):
+        if municipality is None or not kept or point is None or not well_cased(name):
             continue
-        lat = sum(r["lat"] for r in with_point) / len(with_point)
+        lat, lon = point
-        lon = sum(r["lon"] for r in with_point) / len(with_point)
         out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
-    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr)
+    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
     return out
 
 
-def streets(segments, codes, localities, per_locality):
+def street_name(name):
+    # A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
+    opening = name[0].isupper() or name.split()[0] in PARTICLES
+    return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
+
+
+def streets(segments, codes, localities, share, minimum):
     """The names with most segments per locality, each segment at its nearest code centroid."""
-    nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
+    nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
     count = collections.Counter()
     for name, lat, lon in segments:
-        if name[0].isalpha():
+        if street_name(name):
             count[(nearest.find(lat, lon), name)] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "segments")
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
     key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
     if not key:
         sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
-    cache, out = Path(a.cache), Path(a.out)
+    cache, out = geo.directories(a)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
-    regions, municipalities = scb_codes(cache)
+    region_names, municipality_names = scb_codes(cache)
     population = scb_population(cache)
     codes = geonames(cache)
-    places = localities(codes, scb_tatorter(cache), municipalities, population)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
-    named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
+    named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
-    places = {name: l for name, l in places.items() if name in named}
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
-    if empty:
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
-        sys.exit(f"municipalities without a locality: {empty}")
+    geo.log_dropped_localities(dropped["locality"])
+    if dropped["municipality"] or dropped["region"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo-us.py

@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
-"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
+"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
 
-    data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
+    data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import concurrent.futures
 import csv
 import io
+import math
+import random
 import re
 import struct
 import sys
+import urllib.parse
 import zipfile
-from pathlib import Path
 
+import geo
 import source
 import tsv
 
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
 PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
 ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
 TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
-CACHE = Path(__file__).resolve().parent / "cache"
 ESTIMATE = "POPESTIMATE2025"
 CDP = "57"
+# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
+CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
+SELECT ?iso ?capitalLabel ?gnis WHERE {
+  ?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
+  OPTIONAL { ?capital wdt:P590 ?gnis. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}"""})
+SHIPPED_POPULATION = 100000
 HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
 SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
-# Places whose Census name is a merged government's; the postal city is what an address carries.
+# Places whose Census name is not the postal city an address carries.
-NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
+NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
 TIMEZONES = {
     "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
     "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -107,8 +115,8 @@ def place_name(geoid, name):
     return stripped
 
 
-def localities(cache, min_population, counties):
+def localities(cache, counties):
-    """Each shipped place with its county, population and centroid."""
+    """Each incorporated place with its county, population and centroid."""
     place_population, county_part = {}, collections.defaultdict(list)
     for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
         if r["SUMLEV"] == "162":
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
     out = {}
     for r in gazetteer(cache, "place"):
         geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
-        if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid):
+        if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
             continue
         county = max(county_part[geoid])[1]
         if county not in counties:
             print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
             continue
-        out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
+        out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
     return out
 
 
-def postal_codes(cache, localities):
+def capitals(cache, places, counties):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
+    """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
+    by_gnis = {p["gnis"]: code for code, p in places.items()}
+    named = collections.defaultdict(list)
+    for code, p in places.items():
+        named[(counties[p["municipality"]]["region"], p["name"])].append(code)
+    by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
+    gnis_of = collections.defaultdict(set)
+    for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
+        gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
+    out = set()
+    for (state, name), ids in sorted(gnis_of.items()):
+        code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
+        if code:
+            out.add(code)
+        else:
+            print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
+    return out
+
+
+def drawn(places, share):
+    """The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
+    sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
+    the sources moves only the places near the cut."""
+    peopled = [code for code, p in places.items() if p["population"] > 0]
+    key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
+    return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
+
+
+def zcta_localities(cache, localities):
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
     return {zcta: place for zcta, place in largest.items() if place in localities}
 
 
-def streets(cache, counties, locality_of_zcta, per_locality):
+def streets(cache, counties, locality_of_zcta, share, minimum):
     """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
     with concurrent.futures.ThreadPoolExecutor(3) as pool:
         list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
             if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
                 for z in zips.get(r["TLID"], ()):
                     count[(locality_of_zcta[z], r["FULLNAME"])] += 1
-    of = collections.defaultdict(list)
+    return geo.top_streets(count, share, minimum, "addresses"), addresses
-    for (locality, name), n in count.items():
-        of[locality].append((n, name))
-    named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
-    return named, addresses
 
 
 def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
-    p.add_argument("--cache", default=str(CACHE))
+    cache, out = geo.directories(a)
-    p.add_argument("--min-population", type=int, default=25000)
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
 
     state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
     regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
@@ -182,20 +208,29 @@ def main():
         counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
 
-    places = localities(cache, a.min_population, counties)
+    places = localities(cache, counties)
-    locality_of_zcta = postal_codes(cache, places)
+    owner_of_zcta = zcta_localities(cache, places)
-    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
+    capital_codes = capitals(cache, places, counties)
-    for geoid in [l for l in places if l not in named]:
+    owners = {p: places[p] for p in set(owner_of_zcta.values())}
-        print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr)
+    for code in sorted(capital_codes - set(owners)):
-        del places[geoid]
+        print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
-    kept_counties = {l["municipality"] for l in places.values()}
+    places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
+    locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
+    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped_localities(dropped["locality"])
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
-    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo.py

@@ -0,0 +1,98 @@
+"""The steps a country's geo import may share with another's."""
+import argparse
+import collections
+import math
+import sys
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
+CACHE = Path(__file__).resolve().parent / "cache"
+
+
+def parser(doc, country, street_share, **own):
+    """A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
+    "_" standing for "-"."""
+    options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
+    p = argparse.ArgumentParser(description=doc.splitlines()[0])
+    for name in sorted(options):
+        p.add_argument("--" + name.replace("_", "-"), **options[name])
+    return p
+
+
+def directories(args):
+    """The cache and out directories of the parsed options, both created."""
+    cache, out = Path(args.cache), Path(args.out)
+    cache.mkdir(parents=True, exist_ok=True)
+    out.mkdir(parents=True, exist_ok=True)
+    return cache, out
+
+
+def centroid(rows):
+    """The mean lat and lon of the rows with a point, or None where no row has one."""
+    points = [r for r in rows if r["lat"] is not None]
+    if not points:
+        return None
+    return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
+
+
+class Nearest:
+    """Nearest point by an equirectangular distance, over a degree grid."""
+
+    def __init__(self, points, cell=0.05):
+        self.cell = cell
+        self.grid = collections.defaultdict(list)
+        for lat, lon, value in points:
+            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
+
+    def find(self, lat, lon):
+        ci, cj = int(lat // self.cell), int(lon // self.cell)
+        best, best_d = None, math.inf
+        ring = 0
+        while ring < 400:
+            for i in range(ci - ring, ci + ring + 1):
+                for j in range(cj - ring, cj + ring + 1):
+                    if max(abs(i - ci), abs(j - cj)) != ring:
+                        continue
+                    for plat, plon, value in self.grid.get((i, j), ()):
+                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
+                        if d < best_d:
+                            best, best_d = value, d
+            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
+                return best
+            ring += 1
+        return best
+
+
+def top_streets(count, share, minimum, column):
+    """count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
+    them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
+    of = collections.defaultdict(list)
+    for (locality, name), n in count.items():
+        of[locality].append((n, name))
+    return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
+
+
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
+
+
+def log_dropped_localities(rows):
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)

data-import/httpmethod.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/httpmethod.tsv from the IANA HTTP Method Registry.
+
+    data-import/httpmethod.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A method ships when the register cites RFC 9110 section 9.3 or RFC 5789 for it, which
+is the eight HTTP defines and PATCH; the register's other entries do not. The
+register's `yes` and `no` ship as `true` and `false`, which a consumer's boolean reads.
+"""
+import argparse
+import csv
+import io
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/http-methods/methods.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpmethod.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["idempotent", "method", "safe"]
+CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
+BOOLEAN = {"yes": "true", "no": "false"}
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+EXPECTED = 9
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        method = (r["Method Name"] or "").strip()
+        if not CORE.search(r["Reference"] or ""):
+            continue
+        if set(method) & set(SELECTOR):
+            sys.exit(f"{method}: a method holding one of {SELECTOR} is one a selector cannot spell")
+        answers = {}
+        for column in ("Safe", "Idempotent"):
+            answer = (r[column] or "").strip()
+            if answer not in BOOLEAN:
+                sys.exit(f"{method}: {column} {answer!r} is neither yes nor no")
+            answers[column] = BOOLEAN[answer]
+        yield {"idempotent": answers["Idempotent"], "method": method, "safe": answers["Safe"]}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "http-methods.csv").decode("utf-8")), key=lambda r: r["method"])
+    if len(table) != EXPECTED:
+        sys.exit(f"{len(table)} methods cite RFC 9110 section 9.3 or RFC 5789, not {EXPECTED}; the register's Reference column has moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/loglevel.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/loglevel.tsv from RFC 5424 and POSIX <syslog.h>.
+
+    data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE]
+
+Every severity in Table 2 ships, paired to POSIX's keywords by the order each source
+lists them, which holds only because a keyword prefixes the severity RFC 5424 numbers
+alike.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt"
+POSIX = "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["code", "keyword", "severity"]
+TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S)
+ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M)
+SECTION = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S)
+MACRO = re.compile(r"\bLOG_([A-Z]+)\b")
+EXPECTED = 8
+
+
+def severities(text):
+    table = TABLE.search(text)
+    if not table:
+        sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved")
+    return ROW.findall(table.group(1))
+
+
+def keywords(page):
+    section = SECTION.search(html.unescape(re.sub(r"<[^>]+>", "", page)))
+    if not section:
+        sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved")
+    return [m.lower() for m in MACRO.findall(section.group(1))]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--posix", default=POSIX)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(severities(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")))
+    codes = [code for code, _ in table]
+    if codes != [str(i) for i in range(EXPECTED)]:
+        sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}")
+    words = keywords(source.fetch(a.posix, a.cache, "posix-syslog.h.html").decode("utf-8"))
+    if len(words) != EXPECTED:
+        sys.exit(f"POSIX lists {len(words)} severity macros, not {EXPECTED}: {words}")
+    rows = []
+    for (code, severity), keyword in zip(table, words):
+        if not severity.lower().startswith(keyword):
+            sys.exit(f"severity {code} is {severity} in the RFC and {keyword} in POSIX, which is no prefix of it")
+        rows.append({"code": code, "keyword": keyword, "severity": severity})
+    tsv.write(a.out, COLUMNS, rows)
+
+
+if __name__ == "__main__":
+    main()

data-import/mimetype.py

@@ -3,7 +3,7 @@
 
     data-import/mimetype.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
 
-Only IANA-registered types with a filename extension ship; the first extension is the one listed.
+Only types mime-db records as IANA-registered, with a filename extension, ship.
 """
 import argparse
 import json
@@ -16,12 +16,13 @@ SOURCE = "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
 OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "mimetype.tsv"
 CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["ext", "type"]
+FIXUPS = {"application/mp4": "mp4s", "audio/mpeg": "mp3", "audio/ogg": "ogg", "video/quicktime": "mov"}
 
 
 def rows(db):
     for mimetype, entry in db.items():
         if entry.get("source") == "iana" and entry.get("extensions"):
-            yield {"ext": "." + entry["extensions"][0], "type": mimetype}
+            yield {"ext": "." + FIXUPS.get(mimetype, entry["extensions"][0]), "type": mimetype}
 
 
 def main():

data-import/port.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/port.tsv from the IANA Service Name and Transport Protocol Port Number Registry.
+
+    data-import/port.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a service name and a numeric TCP port. A port the registry lists more than
+once keeps the first service it describes, or the first of them where it describes none,
+so a number selects one row.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "port.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["number", "service"]
+FLOOR = 4000
+
+
+def rows(text):
+    best, described = {}, set()
+    for r in csv.DictReader(io.StringIO(text)):
+        service, number = (r["Service Name"] or "").strip(), (r["Port Number"] or "").strip()
+        transport, description = (r["Transport Protocol"] or "").strip(), (r["Description"] or "").strip()
+        if transport != "tcp" or not service or not number.isdigit():
+            continue
+        if not 1 <= int(number) <= 65535:
+            sys.exit(f"{service}: port {number} is outside 1-65535")
+        if number not in best or (description and number not in described):
+            best[number] = {"number": number, "service": service}
+        if description:
+            described.add(number)
+    return sorted(best.values(), key=lambda r: int(r["number"]))
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(source.fetch(a.source, a.cache, "service-names-port-numbers.csv").decode("utf-8"))
+    if len(table) < FLOOR:
+        sys.exit(f"only {len(table)} ports named a TCP service; the registry's columns have moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/protocol.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/protocol.tsv from the IANA Protocol Numbers registry.
+
+    data-import/protocol.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a number and a keyword, which drops the seven numbers assigned to a class of
+protocols rather than to one; a keyword the registry reserves or marks deprecated goes
+too. Where the registry names no protocol beside the keyword, the keyword is the name.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["keyword", "name", "number"]
+SKIP = ("Reserved", "deprecated")
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        number, keyword = (r["Decimal"] or "").strip(), (r["Keyword"] or "").strip()
+        if not number.isdigit() or not keyword or any(s in keyword for s in SKIP):
+            continue
+        if not 0 <= int(number) <= 255:
+            sys.exit(f"{keyword}: protocol number {number} is outside 0-255")
+        yield {"keyword": keyword, "name": " ".join((r["Protocol"] or "").split()) or keyword, "number": number}
+
+
+def refuse_an_unselectable_row(table):
+    """What the loader refuses at New, and a name that names two rows rather than one."""
+    keywords, names = {}, {}
+    for r in table:
+        for column in ("keyword", "name"):
+            if set(r[column]) & set(SELECTOR):
+                sys.exit(f"{r['keyword']}: {column} {r[column]!r} holds one of {SELECTOR}, which a selector cannot spell")
+        if r["keyword"] in keywords:
+            sys.exit(f"{r['keyword']}: repeats the keyword of protocol {keywords[r['keyword']]}")
+        keywords[r["keyword"]] = r["number"]
+    for r in table:
+        if r["name"] != r["keyword"] and r["name"] in keywords:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} is another protocol's keyword")
+        if r["name"] in names:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} repeats {names[r['name']]}'s, so it selects neither")
+        names[r["name"]] = r["keyword"]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "protocol-numbers-1.csv").decode("utf-8")), key=lambda r: int(r["number"]))
+    refuse_an_unselectable_row(table)
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/source.py

@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
     if path.exists():
         return path.read_bytes()
     for attempt in range(1, 6):
-        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
+        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
         try:
             with urllib.request.urlopen(req, timeout=600) as r:
                 body = r.read()

data-import/territory.py

@@ -1,21 +1,24 @@
 #!/usr/bin/env python3
-"""Rebuild data/misc/country.tsv from datasets/country-codes (PDDL).
+"""Rebuild data/misc/territory.tsv from datasets/country-codes (PDDL).
 
-    data-import/country.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+    data-import/territory.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
 """
 import argparse
 import csv
 import io
 import re
+import sys
 from pathlib import Path
 
 import source
 import tsv
 
 SOURCE = "https://raw.githubusercontent.com/datasets/country-codes/main/data/country-codes.csv"
-OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "country.tsv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
 CACHE = Path(__file__).resolve().parent / "cache"
-COLUMNS = ["alpha2", "alpha3", "calling-code", "capital", "currency", "flag", "languages", "name", "numeric", "tld"]
+COLUMNS = ["alpha2", "alpha3", "calling-code", "capital", "country", "currency", "flag", "languages", "name", "numeric", "tld"]
+SOVEREIGN = re.compile(r"^(?:Part of|Territor(?:y|ies) of|Crown dependency of|Commonwealth of|Associated with) ([A-Z]{2})$")
+STANDALONE = {"In contention", "International"}
 # Gaps in the source, keyed by alpha2.
 FIXUPS = {"TR": {"currency": "TRY"}}
 
@@ -33,6 +36,16 @@ def languages(field):
     return ",".join(p for p in field.split(",") if p)
 
 
+def country(alpha2, independent):
+    """The sovereign state the register records; a territory it records none for stands alone."""
+    m = SOVEREIGN.match(independent)
+    if m:
+        return m.group(1)
+    if independent == "Yes" or independent in STANDALONE:
+        return alpha2
+    sys.exit(f"{alpha2}: is_independent {independent!r} names no sovereign this script can read")
+
+
 def rows(text):
     for r in csv.DictReader(io.StringIO(text)):
         alpha2 = r["ISO3166-1-Alpha-2"]
@@ -41,6 +54,7 @@ def rows(text):
             "alpha3": r["ISO3166-1-Alpha-3"],
             "calling-code": calling_code(r["Dial"]),
             "capital": r["Capital"],
+            "country": country(alpha2, r["is_independent"]),
             "currency": r["ISO4217-currency_alphabetic_code"].split(",")[0],
             "flag": flag(alpha2),
             "languages": languages(r["Languages"]),
@@ -59,8 +73,12 @@ def main():
     p.add_argument("--source", default=SOURCE)
     p.add_argument("--out", default=str(OUT))
     a = p.parse_args()
-    table = rows(source.fetch(a.source, a.cache, "country-codes.csv").decode("utf-8"))
+    table = sorted(rows(source.fetch(a.source, a.cache, "country-codes.csv").decode("utf-8")), key=lambda r: r["alpha2"])
-    tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["alpha2"]))
+    shipped = {r["alpha2"] for r in table}
+    for r in table:
+        if r["country"] not in shipped:
+            sys.exit(f"{r['alpha2']}: country {r['country']} is no shipped territory; a sovereign the set drops leaves its territories unreadable")
+    tsv.write(a.out, COLUMNS, table)
 
 
 if __name__ == "__main__":

data-import/timezone.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/timezone.tsv from the IANA tzdb tarball (public domain) and GeoNames (CC BY 4.0).
+
+    data-import/timezone.py [--source URL_OR_FILE] [--cities URL_OR_FILE] [--cache DIR] [--out FILE] [--territories FILE]
+
+The offset is the zone's standard offset, the first field of its Zone rule's last
+continuation line, with a Link resolved to its target. The weight is the population
+GeoNames records in the zone's cities of 15,000 or more, never below that threshold,
+so a drawn zone is one people live in.
+"""
+import argparse
+import csv
+import io
+import re
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
+CITIES = "https://download.geonames.org/export/dump/cities15000.zip"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "timezone.tsv"
+TERRITORIES = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["offset", "territory", "weight", "zone"]
+REGIONS = ["africa", "antarctica", "asia", "australasia", "backward", "etcetera", "europe", "northamerica", "southamerica"]
+# GeoNames' own cutoff, so a zone it records no city in weighs less than one would.
+FLOOR = 15000
+
+
+def member(tar, name):
+    try:
+        f = tar.extractfile(name)
+    except KeyError:
+        f = None
+    if f is None:
+        sys.exit(f"{name}: the tarball no longer holds it as a file; the tzdb layout has moved")
+    return f.read().decode("utf-8")
+
+
+def offsets(tar):
+    """Every zone's standard offset, and every link's target."""
+    std, links = {}, {}
+    for name in REGIONS:
+        zone = None
+        for raw in member(tar, name).splitlines():
+            line = raw.split("#")[0].rstrip()
+            if not line.strip():
+                continue
+            if line.startswith("Zone"):
+                f = line.split()
+                zone, std[f[1]] = f[1], f[2]
+            elif line.startswith("Link"):
+                f = line.split()
+                links[f[2]], zone = f[1], None
+            elif line[0] in " \t" and zone:
+                std[zone] = line.split()[0]
+            else:
+                zone = None
+    return std, links
+
+
+def resolve(zone, std, links):
+    for _ in range(10):
+        if zone in std:
+            return std[zone]
+        if zone not in links:
+            return None
+        zone = links[zone]
+    return None
+
+
+def utc_offset(raw):
+    """±HH:MM from a tzdb STDOFF field, which writes the minutes and seconds only when it has them."""
+    m = re.match(r"^(-)?(\d{1,2})(?::(\d{2})(?::(\d{2}))?)?$", raw)
+    if not m or (m.group(4) or "00") != "00":
+        return None
+    return f"{'-' if m.group(1) else '+'}{int(m.group(2)):02d}:{m.group(3) or '00'}"
+
+
+def population(body):
+    """The population GeoNames records per timezone, summed over its cities."""
+    with zipfile.ZipFile(io.BytesIO(body)) as z:
+        text = z.read("cities15000.txt").decode("utf-8")
+    per = {}
+    for city in csv.reader(io.StringIO(text), delimiter="\t", quoting=csv.QUOTE_NONE):
+        if len(city) > 17 and city[17] and city[14].isdigit():
+            per[city[17]] = per.get(city[17], 0) + int(city[14])
+    return per
+
+
+def rows(tar, territories, pop):
+    std, links = offsets(tar)
+    for line in member(tar, "zone.tab").splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        fields = line.split("\t")
+        if len(fields) < 3:
+            sys.exit(f"zone.tab: {line!r} has {len(fields)} fields; a row names a territory, a location and a zone")
+        territory, zone = fields[0], fields[2]
+        if territory not in territories:
+            continue
+        raw = resolve(zone, std, links)
+        if raw is None:
+            sys.exit(f"{zone}: the tarball gives it no Zone rule and no Link to one")
+        offset = utc_offset(raw)
+        if offset is None:
+            sys.exit(f"{zone}: standard offset {raw!r} is not a ±HH:MM the table can spell")
+        yield {"offset": offset, "territory": territory, "weight": max(pop.get(zone, 0), FLOOR), "zone": zone}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    p.add_argument("--cities", default=CITIES)
+    p.add_argument("--territories", default=str(TERRITORIES))
+    a = p.parse_args()
+    shipped = {r["alpha2"] for r in csv.DictReader(io.StringIO(Path(a.territories).read_text(encoding="utf-8")), delimiter="\t")}
+    pop = population(source.fetch(a.cities, a.cache, "cities15000.zip", magic=b"PK"))
+    body = source.fetch(a.source, a.cache, "tzdata-latest.tar.gz", magic=b"\x1f\x8b")
+    with tarfile.open(fileobj=io.BytesIO(body)) as tar:
+        table = sorted(rows(tar, shipped, pop), key=lambda r: r["zone"])
+        print(f"tzdb {tar.extractfile('version').read().decode('utf-8').strip()}", file=sys.stderr)
+    linked = {r["territory"] for r in table}
+    if missing := shipped - linked:
+        sys.exit(f"no zone for {sorted(missing)}; a parent row without a child is a load error")
+    if (floored := sum(1 for r in table if r["weight"] == FLOOR)) > len(table) // 2:
+        sys.exit(f"{floored} of {len(table)} zones fell to the floor; GeoNames' population or timezone column has moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/tld.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/tld.tsv from the IANA Root Zone Database.
+
+    data-import/tld.py [--source URL_OR_FILE] [--zone URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A TLD ships when the database names a manager for it; one it marks "Not assigned" is a
+delegation the root zone no longer holds, and the zone file is fetched to prove the two
+agree. The key is the A-label the root zone holds; `unicode` is the form the database
+displays, less the bidi marks it wraps a right-to-left label in, and proved to punycode
+back to the key.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/domains/root/db"
+ZONE = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "tld.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["tld", "type", "unicode"]
+TYPES = ("country-code", "generic", "generic-restricted", "infrastructure", "sponsored")
+BIDI = {0x200E: None, 0x200F: None}
+ROW = re.compile(r'<a href="/domains/root/db/([^"]+)\.html">([^<]+)</a>.*?</td>\s*<td>([^<]*)</td>\s*<td>(.*?)</td>', re.S)
+
+
+def rows(page):
+    for a_label, shown, kind, manager in ROW.findall(page):
+        if "Not assigned" in manager:
+            continue
+        kind = kind.strip()
+        if kind not in TYPES:
+            sys.exit(f"{a_label}: the database types it {kind!r}, which is none of {TYPES}")
+        yield {"tld": "." + a_label, "type": kind, "unicode": html.unescape(shown).strip().translate(BIDI)}
+
+
+def prove_each_unicode_form_is_its_key(table):
+    """The displayed form is the key's own spelling, which punycode encodes back to."""
+    for r in table:
+        label = r["unicode"][1:]
+        encoded = label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
+        if "." + encoded != r["tld"]:
+            sys.exit(f"{r['tld']}: the database displays {r['unicode']!r}, which is no spelling of it")
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--zone", default=ZONE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "root-db.html").decode("utf-8")), key=lambda r: r["tld"])
+    prove_each_unicode_form_is_its_key(table)
+    listing = source.fetch(a.zone, a.cache, "tlds-alpha-by-domain.txt").decode("utf-8")
+    delegated = {"." + line.strip().lower() for line in listing.splitlines() if line.strip() and not line.startswith("#")}
+    shipped = {r["tld"] for r in table}
+    if len(table) != len(delegated) or shipped != delegated:
+        sys.exit(f"the database assigns {len(table)} rows where the root zone holds {len(delegated)} TLDs: {sorted(shipped ^ delegated)[:10]}")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/tsv.py

@@ -5,12 +5,14 @@ from pathlib import Path
 
 
 def write(path, columns, rows):
-    """Write the rows as a TSV; every cell must be non-empty and free of tabs, newlines and braces."""
+    """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
     lines = ["\t".join(columns)]
     for row in rows:
-        cells = [str(row[c]) for c in columns]
+        cells = [str(row[c]).strip() for c in columns]
         if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
             raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
         lines.append("\t".join(cells))
+    if len(lines) < 2:
+        raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
     Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
     print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)

data-import/useragent.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/useragent.tsv from microlinkhq/top-user-agents (MIT).
+
+    data-import/useragent.py [--source URL] [--cache DIR] [--out FILE]
+
+The desktop and mobile lists partition the whole, so `device` is read rather than
+parsed. A row ships when the string names a browser and an operating system both.
+"""
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "useragent.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["browser", "device", "os", "ua"]
+# First match wins: a Chromium fork this list does not name ships as Chrome.
+BROWSERS = [("Edge", r"Edg(A|iOS)?/"), ("Opera", r"OPR/"), ("Samsung Internet", r"SamsungBrowser/"),
+            ("Chrome", r"(Chrome|CriOS)/"), ("Firefox", r"(Firefox|FxiOS)/"), ("Safari", r"Version/[\d.]+ .*Safari")]
+SYSTEMS = [("iOS", r"iPhone|iPad|CPU OS "), ("Android", r"Android"), ("ChromeOS", r"CrOS"),
+           ("Windows", r"Windows NT"), ("macOS", r"Macintosh|Mac OS X"), ("Linux", r"X11.*Linux|Ubuntu")]
+
+
+def named(table, ua):
+    for name, pattern in table:
+        if re.search(pattern, ua):
+            return name
+    return ""
+
+
+def rows(desktop, mobile):
+    seen, kept = set(), []
+    for device, uas in (("desktop", desktop), ("mobile", mobile)):
+        for ua in uas:
+            browser, os = named(BROWSERS, ua), named(SYSTEMS, ua)
+            if not browser or not os:
+                print(f"dropped, naming no {'browser' if not browser else 'operating system'}: {ua}", file=sys.stderr)
+                continue
+            if ua in seen:
+                sys.exit(f"{ua}: listed twice, which would draw it twice as often")
+            seen.add(ua)
+            kept.append({"browser": browser, "device": device, "os": os, "ua": ua})
+    if len(kept) < 0.8 * (len(desktop) + len(mobile)):
+        sys.exit(f"only {len(kept)} of {len(desktop) + len(mobile)} strings named both; the tokens have moved")
+    return kept
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    lists = [json.loads(source.fetch(f"{a.source}/{n}.json", a.cache, f"top-user-agents-{n}.json")) for n in ("desktop", "mobile")]
+    tsv.write(a.out, COLUMNS, sorted(rows(*lists), key=lambda r: (r["browser"], r["os"], r["ua"])))
+
+
+if __name__ == "__main__":
+    main()

data.go

@@ -1,195 +1,175 @@
 package fejkdata
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
+// loadSources merges sources in order, the last winning a clash. A source whose manifest
-// label prefixes file names in errors; onDisk marks path as a directory that must
+// carries an index leaves its categories unloaded; every other source loads here, with
-// exist.
+// the unloaded categories it needs.
-type dataSource struct {
+// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
-	fsys   fs.FS
+func loadSources(sources []datafiles.Source) (folder, error) {
-	label  string
+	root := folder{children: map[string]node{}}
-	onDisk bool
-	path   string
-	root   string
-}
-
-func (s dataSource) name(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-// loadData loads every source into one namespace tree and returns its root
-// children. A directory becomes a folder; each *.json file in it compiles to a node
-// keyed by its base name (address.json -> "address"); each subdirectory becomes a
-// nested folder, so folders turn into dot-path segments. Sources merge left to right:
-// matching folders merge by their children, and any other clash is won by the last
-// source loaded. Once merged, linkRefs binds every reference against the
-// final tree.
-func loadData(sources []dataSource) (map[string]node, error) {
-	root := map[string]node{}
 	for _, src := range sources {
-		if src.onDisk {
+		m, err := src.Manifest()
-			if src.path == "" {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.path)
 		if err != nil {
-				return nil, fmt.Errorf("data path %s: %w", src.path, err)
+			return folder{}, err
 		}
-			if !info.IsDir() {
+		if err := addSource(&root, src, m); err != nil {
-				return nil, fmt.Errorf("%s is not a directory", src.path)
+			return folder{}, err
 		}
 	}
-		dir := src.root
+	return root, loadReached(&root, categorySites(&root), nil)
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
-		if err != nil {
-			return nil, err
-		}
-		mergeChildren(root, g.children)
-	}
-	if len(root) == 0 {
-		return nil, fmt.Errorf("no .json data found")
-	}
-	setTablePaths(root)
-	if err := linkTables(root); err != nil {
-		return nil, err
-	}
-	if err := linkRefs(root); err != nil {
-		return nil, err
-	}
-	if err := checkNoCycles(root); err != nil {
-		return nil, err
-	}
-	if err := checkScope(treeScope(root)); err != nil {
-		return nil, err
-	}
-	return root, nil
 }
 
-// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
+func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
-// by name, so the tree is built deterministically. Empty subdirectories (no JSON
+	src = src.Mounted(m.Namespace)
-// anywhere under them) are skipped rather than added as empty namespaces.
+	if m.Index != nil {
-func loadDir(src dataSource, dir string) (*folder, error) {
+		placeIndex(root, src, m.Index)
-	entries, err := fs.ReadDir(src.fsys, dir)
+		return nil
-	if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.name(dir), err)
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
-	for _, e := range entries {
+		return err
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
 	}
-	}
+	mergeFolder(root, g)
-	for _, e := range entries {
+	return nil
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
-		}
-		full := path.Join(dir, e.Name())
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
-		}
-	}
-	return g, nil
 }
 
-// categoryFiles is what a category may name beside itself: the rows files of its
+func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
-// directory, each marked once a category names it.
+	return func(c datafiles.Category) error {
-type categoryFiles struct {
+		n, err := compileCategory(c)
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-// rows reads the rows file a category names beside it.
-func (c *categoryFiles) rows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
 		if err != nil {
 			return err
 		}
-	if len(child.children) == 0 {
+		place(c.Folders).put(c.Name, n)
 		return nil
 	}
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.name(full), err)
-	}
-	g.children[name] = child
-	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
+// madeFolder is the folder that dir names under root, made where it is missing or a category.
-// is skipped.
+func madeFolder(root *folder, dir []string) *folder {
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
+	g := root
-	if !strings.HasSuffix(file, ".json") {
+	for _, seg := range dir {
-		return nil
+		sub, isFolder := g.children[seg].(*folder)
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.put(seg, sub)
 		}
-	name := strings.TrimSuffix(file, ".json")
+		g = sub
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.name(full), err)
 	}
-	b, err := fs.ReadFile(src.fsys, full)
+	return g
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	g.children[name] = n
-	return nil
 }
 
-// mergeChildren overlays src onto dst. Two folders under the same key merge
+func mergeFolder(dst, src *folder) {
-// recursively (so locales/categories from several paths combine); every other
+	for k, v := range src.children {
-// key is replaced, making the last-loaded directory win on a conflict.
+		if dg, ok := dst.children[k].(*folder); ok {
-func mergeChildren(dst, src map[string]node) {
-	for k, v := range src {
-		if dg, ok := dst[k].(*folder); ok {
 			if sg, ok := v.(*folder); ok {
-				mergeChildren(dg.children, sg.children)
+				mergeFolder(dg, sg)
 				continue
 			}
 		}
-		dst[k] = v
+		dst.put(k, v)
 	}
 }
+
+// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
+// single inline node.
+type nodeSet func(fn func(label string, n node) error) error
+
+// categorySite is a loaded category, where it sits and its own dot path.
+type categorySite struct {
+	categoryAt
+	n    node
+	path string
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, siteIn(dir, g, name))
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func siteIn(dir []string, in *folder, name string) categorySite {
+	return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
+}
+
+func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
+
+func siteNodes(sites []categorySite) nodeSet {
+	return func(fn func(label string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func inlineNodes(n node, label string) nodeSet {
+	return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
+}
+
+// pipeline is the one load pipeline, run over one nodeSet; its fields are where
+// entry points differ.
+type pipeline struct {
+	nodes   nodeSet
+	resolve func() error
+	// Set by a caller that proves the columns against their Go types itself.
+	typedByGo bool
+}
+
+// categoryPipeline is the pipeline over the categories of one load, their references
+// resolving against root.
+func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
+	return pipeline{
+		nodes: siteNodes(sites),
+		resolve: func() error {
+			if err := linkTables(sites); err != nil {
+				return err
+			}
+			return resolveCategoryTemplates(sites, root)
+		},
+	}
+}
+
+func (p pipeline) run() error {
+	if err := p.resolve(); err != nil {
+		return err
+	}
+	if err := checkNoCycles(p.nodes); err != nil {
+		return err
+	}
+	settleRecords(p.nodes)
+	if !p.typedByGo {
+		if err := checkColumns(p.nodes); err != nil {
+			return err
+		}
+	}
+	return checkNodeFences(p.nodes)
+}
+
+// checkNodeFences runs the per-node fences over s, each over every node before the next,
+// so which of several broken nodes is reported does not depend on the walk. Its walks
+// recurse unguarded, so pipeline.run refuses cycles first.
+func checkNodeFences(s nodeSet) error {
+	mem := renderCounts{}
+	if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
+		return err
+	}
+	return s((&valueProof{}).checkDatatype)
+}

data/en_US/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"en_US.color": {"paths":[""]},
+		"en_US.company": {"paths":["","base","suffix"]},
+		"en_US.date": {"paths":[""]},
+		"en_US.email": {"paths":["","domain","local","local.n"]},
+		"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"en_US.ip": {"paths":[""]},
+		"en_US.itin": {"paths":["","area","group","serial"]},
+		"en_US.last-name": {"paths":["","count","name"]},
+		"en_US.person": {"paths":["","first","last","prefix","sex"]},
+		"en_US.phone": {"paths":["","area","exchange","line"]},
+		"en_US.phone-exchange": {"paths":[""]},
+		"en_US.price": {"paths":["","amt","cents"]},
+		"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
+		"en_US.ssn": {"paths":["","area","group","serial"]},
+		"en_US.time": {"paths":[""]},
+		"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
+		"en_US.url": {"paths":["","host","path"]},
+		"en_US.username": {"paths":["","n"]},
+		"en_US.version": {"paths":["","n","pre","suffix"]},
+		"en_US.word": {"paths":[""]}
+	},
+	"namespace": "en_US"
+}

data/en_US/address.json

@@ -1,8 +1,8 @@
 {
-  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
+  "format": "{/geo.US.address as a}{a}",
-  "street": "{/geo.US.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.US.address.street-number}",
+  "street-number": "{a.street-number}",
-  "locality": "{/geo.US.address.locality}",
+  "locality": "{a.locality}",
-  "region": "{/geo.US.address.region}",
+  "region": "{a.region}",
-  "postal-code": "{/geo.US.address.postal-code}"
+  "postal-code": "{a.postal-code}"
 }

data/en_US/fs.go

@@ -0,0 +1,19 @@
+// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
+// geo.US, which package geous holds.
+package enus
+
+import (
+	"embed"
+	"io/fs"
+
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.US.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }

data/en_US/fs_test.go

@@ -0,0 +1,22 @@
+package enus_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	enus "github.com/larvit/fejkdata/data/en_US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/en_US/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
-  "prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }],
+  "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data/en_US/phone-exchange.json

@@ -0,0 +1,17 @@
+[
+  { "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
+  { "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
+  { "format": "{int(2,9)}10", "weight": 8 },
+  { "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
+  { "format": "55{int(0,4)}", "weight": 5 },
+  { "format": "55{int(6,9)}", "weight": 4 },
+  { "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
+  { "format": "95{int(1,7)}", "weight": 7 },
+  { "format": "96{digits(1)}", "weight": 10 },
+  { "format": "97{int(0,5)}", "weight": 6 },
+  { "format": "97{int(7,9)}", "weight": 3 }
+]

data/en_US/phone.json

@@ -1,15 +1,15 @@
 [
   {
-    "format": "({area}) {exch}-{line}",
+    "format": "({area}) {exchange}-{line}",
     "weight": 2,
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   },
   {
-    "format": "{area}-{exch}-{line}",
+    "format": "{area}-{exchange}-{line}",
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   }
 ]

data/geo/SE/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
+		"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
+		"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
+		"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
+	},
+	"namespace": "geo.SE"
+}

data/geo/SE/address.json

@@ -1,6 +1,6 @@
 {
   "format": "{street} {street-number}\n{postal-code} {locality}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(1,9)}",
     "{int(10,99)}",
@@ -9,6 +9,6 @@
     { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
     { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
   ],
-  "postal-code": "{.postal-code.code}",
+  "postal-code": "{l.postal-code.code}",
-  "locality": "{.locality.name}"
+  "locality": "{l.name}"
 }

data/geo/SE/fs.go

@@ -0,0 +1,15 @@
+// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
+package geose
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/SE/fs_test.go

@@ -0,0 +1,22 @@
+package geose_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/geo/SE/locality.tsv

@@ -866,7 +866,6 @@ Nynäshamn	0192	16447	58.9034	17.9479
 Nyvång	1277	200	56.1333	12.9000
 Nyåker	2403	200	63.7500	19.3167
 Nälden	2309	986	63.3500	14.2500
-Nämdö	0120	200	59.1833	18.6833
 Nässjö	0682	18639	57.6531	14.6968
 Näsum	1272	1274	56.1667	14.4833
 Näsviken	2184	994	61.7667	16.8667

data/geo/SE/postal-code.tsv

@@ -949,7 +949,6 @@ code	locality
 130 25	Ingmarsö
 130 33	Gällnöby
 130 34	Skälvik
-130 36	Nämdö
 130 37	Stavsnäs
 130 38	Runmarö
 130 39	Sandhamn

data/geo/US/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
+		"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
+		"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
+		"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
+	},
+	"namespace": "geo.US"
+}

data/geo/US/address.json

@@ -1,13 +1,13 @@
 {
   "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{.street.name}",
+  "street": "{.locality as l}{l.street.name}",
   "street-number": [
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 2 },
     { "format": "{int(1000,9999)}", "weight": 0.5 },
     { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
   ],
-  "locality": "{.locality.name}",
+  "locality": "{l.name}",
-  "region": "{.region.abbr}",
+  "region": "{l..municipality..region.abbr}",
-  "postal-code": "{.postal-code.code}"
+  "postal-code": "{l.postal-code.code}"
 }

data/geo/US/fs.go

@@ -0,0 +1,15 @@
+// Package geous is fejkdata's US places module, whose categories are geo.US.*.
+package geous
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/US/fs_test.go

@@ -0,0 +1,22 @@
+package geous_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/.fejkdata.json

@@ -0,0 +1,25 @@
+{
+	"index": {
+		"misc.car": {"paths":["","make","model"]},
+		"misc.coordinate": {"paths":["","lat","lon"]},
+		"misc.creditcard": {"paths":["","d"]},
+		"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
+		"misc.datetime": {"paths":[""]},
+		"misc.emoji": {"paths":[""]},
+		"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
+		"misc.httpstatus": {"paths":["","code","reason"]},
+		"misc.language": {"paths":["","code","code3","name"]},
+		"misc.loglevel": {"paths":["","code","keyword","severity"]},
+		"misc.mac": {"paths":[""]},
+		"misc.mimetype": {"paths":["","ext","type"]},
+		"misc.objectid": {"paths":[""]},
+		"misc.port": {"paths":["","number","service"]},
+		"misc.protocol": {"paths":["","keyword","name","number"]},
+		"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
+		"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
+		"misc.tld": {"paths":["","tld","type","unicode"]},
+		"misc.useragent": {"paths":["","browser","device","os","ua"]},
+		"misc.uuid": {"paths":["","variant"]}
+	},
+	"namespace": "misc"
+}

data/misc/car.json

@@ -1,10 +1 @@
-[
+{ "format": "{make} {model}", "rows": "car.tsv" }
-  { "format": "{maker} {model}", "maker": "BMW", "model": ["3 Series", "5 Series", "X3", "X5"] },
-  { "format": "{maker} {model}", "maker": "Ford", "model": ["Fiesta", "Focus", "Mustang", "Explorer"] },
-  { "format": "{maker} {model}", "maker": "Honda", "model": ["Civic", "Accord", "CR-V", "Jazz"] },
-  { "format": "{maker} {model}", "maker": "Mercedes-Benz", "model": ["A-Class", "C-Class", "E-Class", "GLC"] },
-  { "format": "{maker} {model}", "maker": "Tesla", "model": ["Model 3", "Model S", "Model X", "Model Y"] },
-  { "format": "{maker} {model}", "maker": "Toyota", "model": ["Corolla", "Camry", "RAV4", "Yaris"] },
-  { "format": "{maker} {model}", "maker": "Volkswagen", "model": ["Golf", "Passat", "Polo", "Tiguan"] },
-  { "format": "{maker} {model}", "maker": "Volvo", "model": ["XC40", "XC60", "XC90", "V60"] }
-]

data/misc/car.tsv

@@ -0,0 +1,33 @@
+make	model
+BMW	3 Series
+BMW	5 Series
+BMW	X3
+BMW	X5
+Ford	Explorer
+Ford	Fiesta
+Ford	Focus
+Ford	Mustang
+Honda	Accord
+Honda	CR-V
+Honda	Civic
+Honda	Jazz
+Mercedes-Benz	A-Class
+Mercedes-Benz	C-Class
+Mercedes-Benz	E-Class
+Mercedes-Benz	GLC
+Tesla	Model 3
+Tesla	Model S
+Tesla	Model X
+Tesla	Model Y
+Toyota	Camry
+Toyota	Corolla
+Toyota	RAV4
+Toyota	Yaris
+Volkswagen	Golf
+Volkswagen	Passat
+Volkswagen	Polo
+Volkswagen	Tiguan
+Volvo	V60
+Volvo	XC40
+Volvo	XC60
+Volvo	XC90

data/misc/country.json

@@ -1 +0,0 @@
-{ "format": "{name}", "rows": "country.tsv", "key": "alpha2", "name": "name" }

data/misc/country.tsv

@@ -1,242 +0,0 @@
-alpha2	alpha3	calling-code	capital	currency	flag	languages	name	numeric	tld
-AD	AND	376	Andorra la Vella	EUR	🇦🇩	ca	Andorra	020	.ad
-AE	ARE	971	Abu Dhabi	AED	🇦🇪	ar-AE,fa,en,hi,ur	United Arab Emirates	784	.ae
-AF	AFG	93	Kabul	AFN	🇦🇫	fa-AF,ps,uz-AF,tk	Afghanistan	004	.af
-AG	ATG	1-268	St. John's	XCD	🇦🇬	en-AG	Antigua & Barbuda	028	.ag
-AI	AIA	1-264	The Valley	XCD	🇦🇮	en-AI	Anguilla	660	.ai
-AL	ALB	355	Tirana	ALL	🇦🇱	sq,el	Albania	008	.al
-AM	ARM	374	Yerevan	AMD	🇦🇲	hy	Armenia	051	.am
-AO	AGO	244	Luanda	AOA	🇦🇴	pt-AO	Angola	024	.ao
-AR	ARG	54	Buenos Aires	ARS	🇦🇷	es-AR,en,it,de,fr,gn	Argentina	032	.ar
-AS	ASM	1-684	Pago Pago	USD	🇦🇸	en-AS,sm,to	American Samoa	016	.as
-AT	AUT	43	Vienna	EUR	🇦🇹	de-AT,hr,hu,sl	Austria	040	.at
-AU	AUS	61	Canberra	AUD	🇦🇺	en-AU	Australia	036	.au
-AW	ABW	297	Oranjestad	AWG	🇦🇼	nl-AW,pap,es,en	Aruba	533	.aw
-AX	ALA	358	Mariehamn	EUR	🇦🇽	sv-AX	Åland Islands	248	.ax
-AZ	AZE	994	Baku	AZN	🇦🇿	az,ru,hy	Azerbaijan	031	.az
-BA	BIH	387	Sarajevo	BAM	🇧🇦	bs,hr-BA,sr-BA	Bosnia	070	.ba
-BB	BRB	1-246	Bridgetown	BBD	🇧🇧	en-BB	Barbados	052	.bb
-BD	BGD	880	Dhaka	BDT	🇧🇩	bn-BD,en	Bangladesh	050	.bd
-BE	BEL	32	Brussels	EUR	🇧🇪	nl-BE,fr-BE,de-BE	Belgium	056	.be
-BF	BFA	226	Ouagadougou	XOF	🇧🇫	fr-BF,mos	Burkina Faso	854	.bf
-BG	BGR	359	Sofia	EUR	🇧🇬	bg,tr-BG,rom	Bulgaria	100	.bg
-BH	BHR	973	Manama	BHD	🇧🇭	ar-BH,en,fa,ur	Bahrain	048	.bh
-BI	BDI	257	Gitega	BIF	🇧🇮	fr-BI,rn	Burundi	108	.bi
-BJ	BEN	229	Porto-Novo	XOF	🇧🇯	fr-BJ	Benin	204	.bj
-BL	BLM	590	Gustavia	EUR	🇧🇱	fr	St. Barthélemy	652	.gp
-BM	BMU	1-441	Hamilton	BMD	🇧🇲	en-BM,pt	Bermuda	060	.bm
-BN	BRN	673	Bandar Seri Begawan	BND	🇧🇳	ms-BN,en-BN	Brunei	096	.bn
-BO	BOL	591	Sucre	BOB	🇧🇴	es-BO,qu,ay	Bolivia	068	.bo
-BR	BRA	55	Brasilia	BRL	🇧🇷	pt-BR,es,en,fr	Brazil	076	.br
-BS	BHS	1-242	Nassau	BSD	🇧🇸	en-BS	Bahamas	044	.bs
-BT	BTN	975	Thimphu	INR	🇧🇹	dz	Bhutan	064	.bt
-BW	BWA	267	Gaborone	BWP	🇧🇼	en-BW,tn-BW	Botswana	072	.bw
-BY	BLR	375	Minsk	BYN	🇧🇾	be,ru	Belarus	112	.by
-BZ	BLZ	501	Belmopan	BZD	🇧🇿	en-BZ,es	Belize	084	.bz
-CA	CAN	1	Ottawa	CAD	🇨🇦	en-CA,fr-CA,iu	Canada	124	.ca
-CC	CCK	61	West Island	AUD	🇨🇨	ms-CC,en	Cocos Islands	166	.cc
-CD	COD	243	Kinshasa	CDF	🇨🇩	fr-CD,ln,ktu,kg,sw,lua	Congo - Kinshasa	180	.cd
-CF	CAF	236	Bangui	XAF	🇨🇫	fr-CF,sg,ln,kg	Central African Republic	140	.cf
-CG	COG	242	Brazzaville	XAF	🇨🇬	fr-CG,kg,ln-CG	Congo - Brazzaville	178	.cg
-CH	CHE	41	Bern	CHF	🇨🇭	de-CH,fr-CH,it-CH,rm	Switzerland	756	.ch
-CI	CIV	225	Yamoussoukro	XOF	🇨🇮	fr-CI	Côte d’Ivoire	384	.ci
-CK	COK	682	Avarua	NZD	🇨🇰	en-CK,mi	Cook Islands	184	.ck
-CL	CHL	56	Santiago	CLP	🇨🇱	es-CL	Chile	152	.cl
-CM	CMR	237	Yaounde	XAF	🇨🇲	en-CM,fr-CM	Cameroon	120	.cm
-CN	CHN	86	Beijing	CNY	🇨🇳	zh-CN,yue,wuu,dta,ug,za	China	156	.cn
-CO	COL	57	Bogota	COP	🇨🇴	es-CO	Colombia	170	.co
-CR	CRI	506	San Jose	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
-CU	CUB	53	Havana	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
-CV	CPV	238	Praia	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
-CW	CUW	599	 Willemstad	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
-CX	CXR	61	Flying Fish Cove	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
-CY	CYP	357	Nicosia	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
-CZ	CZE	420	Prague	CZK	🇨🇿	cs,sk	Czechia	203	.cz
-DE	DEU	49	Berlin	EUR	🇩🇪	de	Germany	276	.de
-DJ	DJI	253	Djibouti	DJF	🇩🇯	fr-DJ,ar,so-DJ,aa	Djibouti	262	.dj
-DK	DNK	45	Copenhagen	DKK	🇩🇰	da-DK,en,fo,de-DK	Denmark	208	.dk
-DM	DMA	1-767	Roseau	XCD	🇩🇲	en-DM	Dominica	212	.dm
-DO	DOM	1-809	Santo Domingo	DOP	🇩🇴	es-DO	Dominican Republic	214	.do
-DZ	DZA	213	Algiers	DZD	🇩🇿	ar-DZ	Algeria	012	.dz
-EC	ECU	593	Quito	USD	🇪🇨	es-EC	Ecuador	218	.ec
-EE	EST	372	Tallinn	EUR	🇪🇪	et,ru	Estonia	233	.ee
-EG	EGY	20	Cairo	EGP	🇪🇬	ar-EG,en,fr	Egypt	818	.eg
-EH	ESH	212	El-Aaiun	MAD	🇪🇭	ar,mey	Western Sahara	732	.eh
-ER	ERI	291	Asmara	ERN	🇪🇷	aa-ER,ar,tig,kun,ti-ER	Eritrea	232	.er
-ES	ESP	34	Madrid	EUR	🇪🇸	es-ES,ca,gl,eu,oc	Spain	724	.es
-ET	ETH	251	Addis Ababa	ETB	🇪🇹	am,en-ET,om-ET,ti-ET,so-ET,sid	Ethiopia	231	.et
-FI	FIN	358	Helsinki	EUR	🇫🇮	fi-FI,sv-FI,smn	Finland	246	.fi
-FJ	FJI	679	Suva	FJD	🇫🇯	en-FJ,fj	Fiji	242	.fj
-FK	FLK	500	Stanley	FKP	🇫🇰	en-FK	Falkland Islands	238	.fk
-FM	FSM	691	Palikir	USD	🇫🇲	en-FM,chk,pon,yap,kos,uli,woe,nkr,kpg	Micronesia	583	.fm
-FO	FRO	298	Torshavn	DKK	🇫🇴	fo,da-FO	Faroe Islands	234	.fo
-FR	FRA	33	Paris	EUR	🇫🇷	fr-FR,frp,br,co,ca,eu,oc	France	250	.fr
-GA	GAB	241	Libreville	XAF	🇬🇦	fr-GA	Gabon	266	.ga
-GB	GBR	44	London	GBP	🇬🇧	en-GB,cy-GB,gd	UK	826	.uk
-GD	GRD	1-473	St. George's	XCD	🇬🇩	en-GD	Grenada	308	.gd
-GE	GEO	995	Tbilisi	GEL	🇬🇪	ka,ru,hy,az	Georgia	268	.ge
-GF	GUF	594	Cayenne	EUR	🇬🇫	fr-GF	French Guiana	254	.gf
-GG	GGY	44	St Peter Port	GBP	🇬🇬	en,nrf	Guernsey	831	.gg
-GH	GHA	233	Accra	GHS	🇬🇭	en-GH,ak,ee,tw	Ghana	288	.gh
-GI	GIB	350	Gibraltar	GIP	🇬🇮	en-GI,es,it,pt	Gibraltar	292	.gi
-GL	GRL	299	Nuuk	DKK	🇬🇱	kl,da-GL,en	Greenland	304	.gl
-GM	GMB	220	Banjul	GMD	🇬🇲	en-GM,mnk,wof,wo,ff	Gambia	270	.gm
-GN	GIN	224	Conakry	GNF	🇬🇳	fr-GN	Guinea	324	.gn
-GP	GLP	590	Basse-Terre	EUR	🇬🇵	fr-GP	Guadeloupe	312	.gp
-GQ	GNQ	240	Ciudad de la Paz	XAF	🇬🇶	es-GQ,fr,pt	Equatorial Guinea	226	.gq
-GR	GRC	30	Athens	EUR	🇬🇷	el-GR,en,fr	Greece	300	.gr
-GT	GTM	502	Guatemala City	GTQ	🇬🇹	es-GT	Guatemala	320	.gt
-GU	GUM	1-671	Hagatna	USD	🇬🇺	en-GU,ch-GU	Guam	316	.gu
-GW	GNB	245	Bissau	XOF	🇬🇼	pt-GW,pov	Guinea-Bissau	624	.gw
-GY	GUY	592	Georgetown	GYD	🇬🇾	en-GY	Guyana	328	.gy
-HK	HKG	852	Hong Kong	HKD	🇭🇰	zh-HK,yue,zh,en	Hong Kong	344	.hk
-HN	HND	504	Tegucigalpa	HNL	🇭🇳	es-HN,cab,miq	Honduras	340	.hn
-HR	HRV	385	Zagreb	EUR	🇭🇷	hr-HR,sr	Croatia	191	.hr
-HT	HTI	509	Port-au-Prince	HTG	🇭🇹	ht,fr-HT	Haiti	332	.ht
-HU	HUN	36	Budapest	HUF	🇭🇺	hu-HU	Hungary	348	.hu
-ID	IDN	62	Jakarta	IDR	🇮🇩	id,en,nl,jv	Indonesia	360	.id
-IE	IRL	353	Dublin	EUR	🇮🇪	en-IE,ga-IE	Ireland	372	.ie
-IL	ISR	972	Jerusalem	ILS	🇮🇱	he,ar-IL,en-IL	Israel	376	.il
-IM	IMN	44	Douglas	GBP	🇮🇲	en,gv	Isle of Man	833	.im
-IN	IND	91	New Delhi	INR	🇮🇳	en-IN,hi,bn,te,mr,ta,ur,gu,kn,ml,or,pa,as,bh,sat,ks,ne,sd,kok,doi,mni,sit,sa,fr,lus,inc	India	356	.in
-IO	IOT	246	Diego Garcia	USD	🇮🇴	en-IO	British Indian Ocean Territory	086	.io
-IQ	IRQ	964	Baghdad	IQD	🇮🇶	ar-IQ,ku,hy	Iraq	368	.iq
-IR	IRN	98	Tehran	IRR	🇮🇷	fa-IR,ku	Iran	364	.ir
-IS	ISL	354	Reykjavik	ISK	🇮🇸	is,en,de,da,sv,no	Iceland	352	.is
-IT	ITA	39	Rome	EUR	🇮🇹	it-IT,de-IT,fr-IT,sc,ca,co,sl	Italy	380	.it
-JE	JEY	44	Saint Helier	GBP	🇯🇪	en,fr,nrf	Jersey	832	.je
-JM	JAM	1-876	Kingston	JMD	🇯🇲	en-JM	Jamaica	388	.jm
-JO	JOR	962	Amman	JOD	🇯🇴	ar-JO,en	Jordan	400	.jo
-JP	JPN	81	Tokyo	JPY	🇯🇵	ja	Japan	392	.jp
-KE	KEN	254	Nairobi	KES	🇰🇪	en-KE,sw-KE	Kenya	404	.ke
-KG	KGZ	996	Bishkek	KGS	🇰🇬	ky,uz,ru	Kyrgyzstan	417	.kg
-KH	KHM	855	Phnom Penh	KHR	🇰🇭	km,fr,en	Cambodia	116	.kh
-KI	KIR	686	Tarawa	AUD	🇰🇮	en-KI,gil	Kiribati	296	.ki
-KM	COM	269	Moroni	KMF	🇰🇲	ar,fr-KM	Comoros	174	.km
-KN	KNA	1-869	Basseterre	XCD	🇰🇳	en-KN	St. Kitts & Nevis	659	.kn
-KP	PRK	850	Pyongyang	KPW	🇰🇵	ko-KP	North Korea	408	.kp
-KR	KOR	82	Seoul	KRW	🇰🇷	ko-KR,en	South Korea	410	.kr
-KW	KWT	965	Kuwait City	KWD	🇰🇼	ar-KW,en	Kuwait	414	.kw
-KY	CYM	1-345	George Town	KYD	🇰🇾	en-KY	Cayman Islands	136	.ky
-KZ	KAZ	7	Nur-Sultan	KZT	🇰🇿	kk,ru	Kazakhstan	398	.kz
-LA	LAO	856	Vientiane	LAK	🇱🇦	lo,fr,en	Laos	418	.la
-LB	LBN	961	Beirut	LBP	🇱🇧	ar-LB,fr-LB,en,hy	Lebanon	422	.lb
-LC	LCA	1-758	Castries	XCD	🇱🇨	en-LC	St. Lucia	662	.lc
-LI	LIE	423	Vaduz	CHF	🇱🇮	de-LI	Liechtenstein	438	.li
-LK	LKA	94	Colombo	LKR	🇱🇰	si,ta,en	Sri Lanka	144	.lk
-LR	LBR	231	Monrovia	LRD	🇱🇷	en-LR	Liberia	430	.lr
-LS	LSO	266	Maseru	LSL	🇱🇸	en-LS,st,zu,xh	Lesotho	426	.ls
-LT	LTU	370	Vilnius	EUR	🇱🇹	lt,ru,pl	Lithuania	440	.lt
-LU	LUX	352	Luxembourg	EUR	🇱🇺	lb,de-LU,fr-LU	Luxembourg	442	.lu
-LV	LVA	371	Riga	EUR	🇱🇻	lv,ru,lt	Latvia	428	.lv
-LY	LBY	218	Tripoli	LYD	🇱🇾	ar-LY,it,en	Libya	434	.ly
-MA	MAR	212	Rabat	MAD	🇲🇦	ar-MA,ber,fr	Morocco	504	.ma
-MC	MCO	377	Monaco	EUR	🇲🇨	fr-MC,en,it	Monaco	492	.mc
-MD	MDA	373	Chisinau	MDL	🇲🇩	ro,ru,gag,tr	Moldova	498	.md
-ME	MNE	382	Podgorica	EUR	🇲🇪	sr,hu,bs,sq,hr,rom	Montenegro	499	.me
-MF	MAF	590	Marigot	EUR	🇲🇫	fr	St. Martin	663	.gp
-MG	MDG	261	Antananarivo	MGA	🇲🇬	fr-MG,mg	Madagascar	450	.mg
-MH	MHL	692	Majuro	USD	🇲🇭	mh,en-MH	Marshall Islands	584	.mh
-MK	MKD	389	Skopje	MKD	🇲🇰	mk,sq,tr,rmm,sr	North Macedonia	807	.mk
-ML	MLI	223	Bamako	XOF	🇲🇱	fr-ML,bm	Mali	466	.ml
-MM	MMR	95	Nay Pyi Taw	MMK	🇲🇲	my	Myanmar	104	.mm
-MN	MNG	976	Ulaanbaatar	MNT	🇲🇳	mn,ru	Mongolia	496	.mn
-MO	MAC	853	Macao	MOP	🇲🇴	zh,zh-MO,pt	Macao	446	.mo
-MP	MNP	1-670	Saipan	USD	🇲🇵	fil,tl,zh,ch-MP,en-MP	Northern Mariana Islands	580	.mp
-MQ	MTQ	596	Fort-de-France	EUR	🇲🇶	fr-MQ	Martinique	474	.mq
-MR	MRT	222	Nouakchott	MRU	🇲🇷	ar-MR,fuc,snk,fr,mey,wo	Mauritania	478	.mr
-MS	MSR	1-664	Plymouth	XCD	🇲🇸	en-MS	Montserrat	500	.ms
-MT	MLT	356	Valletta	EUR	🇲🇹	mt,en-MT	Malta	470	.mt
-MU	MUS	230	Port Louis	MUR	🇲🇺	en-MU,bho,fr	Mauritius	480	.mu
-MV	MDV	960	Male	MVR	🇲🇻	dv,en	Maldives	462	.mv
-MW	MWI	265	Lilongwe	MWK	🇲🇼	ny,yao,tum,swk	Malawi	454	.mw
-MX	MEX	52	Mexico City	MXN	🇲🇽	es-MX	Mexico	484	.mx
-MY	MYS	60	Kuala Lumpur	MYR	🇲🇾	ms-MY,en,zh,ta,te,ml,pa,th	Malaysia	458	.my
-MZ	MOZ	258	Maputo	MZN	🇲🇿	pt-MZ,vmw	Mozambique	508	.mz
-NA	NAM	264	Windhoek	NAD	🇳🇦	en-NA,af,de,hz,naq	Namibia	516	.na
-NC	NCL	687	Noumea	XPF	🇳🇨	fr-NC	New Caledonia	540	.nc
-NE	NER	227	Niamey	XOF	🇳🇪	fr-NE,ha,kr,dje	Niger	562	.ne
-NF	NFK	672	Kingston	AUD	🇳🇫	en-NF	Norfolk Island	574	.nf
-NG	NGA	234	Abuja	NGN	🇳🇬	en-NG,ha,yo,ig,ff	Nigeria	566	.ng
-NI	NIC	505	Managua	NIO	🇳🇮	es-NI,en	Nicaragua	558	.ni
-NL	NLD	31	Amsterdam	EUR	🇳🇱	nl-NL,fy-NL	Netherlands	528	.nl
-NO	NOR	47	Oslo	NOK	🇳🇴	no,nb,nn,se,fi	Norway	578	.no
-NP	NPL	977	Kathmandu	NPR	🇳🇵	ne,en	Nepal	524	.np
-NR	NRU	674	Yaren	AUD	🇳🇷	na,en-NR	Nauru	520	.nr
-NU	NIU	683	Alofi	NZD	🇳🇺	niu,en-NU	Niue	570	.nu
-NZ	NZL	64	Wellington	NZD	🇳🇿	en-NZ,mi	New Zealand	554	.nz
-OM	OMN	968	Muscat	OMR	🇴🇲	ar-OM,en,bal,ur	Oman	512	.om
-PA	PAN	507	Panama City	PAB	🇵🇦	es-PA,en	Panama	591	.pa
-PE	PER	51	Lima	PEN	🇵🇪	es-PE,qu,ay	Peru	604	.pe
-PF	PYF	689	Papeete	XPF	🇵🇫	fr-PF,ty	French Polynesia	258	.pf
-PG	PNG	675	Port Moresby	PGK	🇵🇬	en-PG,ho,meu,tpi	Papua New Guinea	598	.pg
-PH	PHL	63	Manila	PHP	🇵🇭	tl,en-PH,fil,ceb,ilo,hil,war,pam,bik,bcl,pag,mrw,tsg,mdh,cbk,krj,sgd,msb,akl,ibg,yka,mta,abx	Philippines	608	.ph
-PK	PAK	92	Islamabad	PKR	🇵🇰	ur-PK,en-PK,pa,sd,ps,brh	Pakistan	586	.pk
-PL	POL	48	Warsaw	PLN	🇵🇱	pl	Poland	616	.pl
-PM	SPM	508	Saint-Pierre	EUR	🇵🇲	fr-PM	St. Pierre & Miquelon	666	.pm
-PN	PCN	870	Adamstown	NZD	🇵🇳	en-PN	Pitcairn	612	.pn
-PR	PRI	1	San Juan	USD	🇵🇷	en-PR,es-PR	Puerto Rico	630	.pr
-PT	PRT	351	Lisbon	EUR	🇵🇹	pt-PT,mwl	Portugal	620	.pt
-PW	PLW	680	Melekeok	USD	🇵🇼	pau,sov,en-PW,tox,ja,fil,zh	Palau	585	.pw
-PY	PRY	595	Asuncion	PYG	🇵🇾	es-PY,gn	Paraguay	600	.py
-QA	QAT	974	Doha	QAR	🇶🇦	ar-QA,es	Qatar	634	.qa
-RE	REU	262	Saint-Denis	EUR	🇷🇪	fr-RE	Réunion	638	.re
-RO	ROU	40	Bucharest	RON	🇷🇴	ro,hu,rom	Romania	642	.ro
-RS	SRB	381	Belgrade	RSD	🇷🇸	sr,hu,bs,rom	Serbia	688	.rs
-RU	RUS	7	Moscow	RUB	🇷🇺	ru,tt,xal,cau,ady,kv,ce,tyv,cv,udm,tut,mns,bua,myv,mdf,chm,ba,inh,kbd,krc,av,sah,nog	Russia	643	.ru
-RW	RWA	250	Kigali	RWF	🇷🇼	rw,en-RW,fr-RW,sw	Rwanda	646	.rw
-SA	SAU	966	Riyadh	SAR	🇸🇦	ar-SA	Saudi Arabia	682	.sa
-SB	SLB	677	Honiara	SBD	🇸🇧	en-SB,tpi	Solomon Islands	090	.sb
-SC	SYC	248	Victoria	SCR	🇸🇨	en-SC,fr-SC	Seychelles	690	.sc
-SD	SDN	249	Khartoum	SDG	🇸🇩	ar-SD,en,fia	Sudan	729	.sd
-SE	SWE	46	Stockholm	SEK	🇸🇪	sv-SE,se,sma,fi-SE	Sweden	752	.se
-SG	SGP	65	Singapore	SGD	🇸🇬	cmn,en-SG,ms-SG,ta-SG,zh-SG	Singapore	702	.sg
-SH	SHN	290	Jamestown	SHP	🇸🇭	en-SH	St. Helena	654	.sh
-SI	SVN	386	Ljubljana	EUR	🇸🇮	sl,sh	Slovenia	705	.si
-SJ	SJM	47	Longyearbyen	NOK	🇸🇯	no,ru	Svalbard & Jan Mayen	744	.sj
-SK	SVK	421	Bratislava	EUR	🇸🇰	sk,hu	Slovakia	703	.sk
-SL	SLE	232	Freetown	SLE	🇸🇱	en-SL,men,tem	Sierra Leone	694	.sl
-SM	SMR	378	San Marino	EUR	🇸🇲	it-SM	San Marino	674	.sm
-SN	SEN	221	Dakar	XOF	🇸🇳	fr-SN,wo,fuc,mnk	Senegal	686	.sn
-SO	SOM	252	Mogadishu	SOS	🇸🇴	so-SO,ar-SO,it,en-SO	Somalia	706	.so
-SR	SUR	597	Paramaribo	SRD	🇸🇷	nl-SR,en,srn,hns,jv	Suriname	740	.sr
-SS	SSD	211	Juba	SSP	🇸🇸	en	South Sudan	728	.ss
-ST	STP	239	Sao Tome	STN	🇸🇹	pt-ST	São Tomé & Príncipe	678	.st
-SV	SLV	503	San Salvador	SVC	🇸🇻	es-SV	El Salvador	222	.sv
-SX	SXM	1-721	Philipsburg	XCG	🇸🇽	nl,en	Sint Maarten	534	.sx
-SY	SYR	963	Damascus	SYP	🇸🇾	ar-SY,ku,hy,arc,fr,en	Syria	760	.sy
-SZ	SWZ	268	Mbabane	SZL	🇸🇿	en-SZ,ss-SZ	Eswatini	748	.sz
-TC	TCA	1-649	Cockburn Town	USD	🇹🇨	en-TC	Turks & Caicos Islands	796	.tc
-TD	TCD	235	N'Djamena	XAF	🇹🇩	fr-TD,ar-TD,sre	Chad	148	.td
-TF	ATF	262	Port-aux-Francais	EUR	🇹🇫	fr	French Southern Territories	260	.tf
-TG	TGO	228	Lome	XOF	🇹🇬	fr-TG,ee,hna,kbp,dag,ha	Togo	768	.tg
-TH	THA	66	Bangkok	THB	🇹🇭	th,en	Thailand	764	.th
-TJ	TJK	992	Dushanbe	TJS	🇹🇯	tg,ru	Tajikistan	762	.tj
-TL	TLS	670	Dili	USD	🇹🇱	tet,pt-TL,id,en	Timor-Leste	626	.tl
-TM	TKM	993	Ashgabat	TMT	🇹🇲	tk,ru,uz	Turkmenistan	795	.tm
-TN	TUN	216	Tunis	TND	🇹🇳	ar-TN,fr	Tunisia	788	.tn
-TO	TON	676	Nuku'alofa	TOP	🇹🇴	to,en-TO	Tonga	776	.to
-TR	TUR	90	Ankara	TRY	🇹🇷	tr-TR,ku,diq,az,av	Türkiye	792	.tr
-TT	TTO	1-868	Port of Spain	TTD	🇹🇹	en-TT,hns,fr,es,zh	Trinidad & Tobago	780	.tt
-TV	TUV	688	Funafuti	AUD	🇹🇻	tvl,en,sm,gil	Tuvalu	798	.tv
-TW	TWN	886	Taipei	TWD	🇹🇼	zh-TW,zh,nan,hak	Taiwan	158	.tw
-TZ	TZA	255	Dodoma	TZS	🇹🇿	sw-TZ,en,ar	Tanzania	834	.tz
-UA	UKR	380	Kyiv	UAH	🇺🇦	uk,ru-UA,rom,pl,hu	Ukraine	804	.ua
-UG	UGA	256	Kampala	UGX	🇺🇬	en-UG,lg,sw,ar	Uganda	800	.ug
-US	USA	1	Washington	USD	🇺🇸	en-US,es-US,haw,fr	US	840	.us
-UY	URY	598	Montevideo	UYU	🇺🇾	es-UY	Uruguay	858	.uy
-UZ	UZB	998	Tashkent	UZS	🇺🇿	uz,ru,tg	Uzbekistan	860	.uz
-VA	VAT	39	Vatican City	EUR	🇻🇦	la,it,fr	Vatican City	336	.va
-VC	VCT	1-784	Kingstown	XCD	🇻🇨	en-VC,fr	St. Vincent & Grenadines	670	.vc
-VE	VEN	58	Caracas	VES	🇻🇪	es-VE	Venezuela	862	.ve
-VG	VGB	1-284	Road Town	USD	🇻🇬	en-VG	British Virgin Islands	092	.vg
-VI	VIR	1-340	Charlotte Amalie	USD	🇻🇮	en-VI	U.S. Virgin Islands	850	.vi
-VN	VNM	84	Hanoi	VND	🇻🇳	vi,en,fr,zh,km	Vietnam	704	.vn
-VU	VUT	678	Port Vila	VUV	🇻🇺	bi,en-VU,fr-VU	Vanuatu	548	.vu
-WF	WLF	681	Mata Utu	XPF	🇼🇫	wls,fud,fr-WF	Wallis & Futuna	876	.wf
-WS	WSM	685	Apia	WST	🇼🇸	sm,en-WS	Samoa	882	.ws
-YE	YEM	967	Sanaa	YER	🇾🇪	ar-YE	Yemen	887	.ye
-YT	MYT	262	Mamoudzou	EUR	🇾🇹	fr-YT	Mayotte	175	.yt
-ZA	ZAF	27	Pretoria	ZAR	🇿🇦	zu,xh,af,nso,en-ZA,tn,st,ts,ss,ve,nr	South Africa	710	.za
-ZM	ZMB	260	Lusaka	ZMW	🇿🇲	en-ZM,bem,loz,lun,lue,ny,toi	Zambia	894	.zm
-ZW	ZWE	263	Harare	ZWG	🇿🇼	en-ZW,sn,nr,nd	Zimbabwe	716	.zw

data/misc/fs.go

@@ -0,0 +1,15 @@
+// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
+package misc
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/misc/fs_test.go

@@ -0,0 +1,22 @@
+package misc_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	"github.com/larvit/fejkdata/data/misc"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/httpmethod.json

@@ -0,0 +1 @@
+{ "format": "{method}", "rows": "httpmethod.tsv", "key": "method" }

data/misc/httpmethod.tsv

@@ -0,0 +1,10 @@
+idempotent	method	safe
+false	CONNECT	false
+true	DELETE	false
+true	GET	true
+true	HEAD	true
+true	OPTIONS	true
+false	PATCH	false
+false	POST	false
+true	PUT	false
+true	TRACE	true

data/misc/loglevel.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "loglevel.tsv", "key": "code", "name": "keyword" }

data/misc/loglevel.tsv

@@ -0,0 +1,9 @@
+code	keyword	severity
+0	emerg	Emergency
+1	alert	Alert
+2	crit	Critical
+3	err	Error
+4	warning	Warning
+5	notice	Notice
+6	info	Informational
+7	debug	Debug

data/misc/mimetype.tsv

@@ -62,7 +62,7 @@ ext	type
 .musd	application/mmt-usd+xml
 .mods	application/mods+xml
 .m21	application/mp21
-.mp4	application/mp4
+.mp4s	application/mp4
 .doc	application/msword
 .mxf	application/mxf
 .nq	application/n-quads
@@ -511,8 +511,8 @@ ext	type
 .mka	audio/matroska
 .mxmf	audio/mobile-xmf
 .m4a	audio/mp4
-.mpga	audio/mpeg
+.mp3	audio/mpeg
-.oga	audio/ogg
+.ogg	audio/ogg
 .uva	audio/vnd.dece.audio
 .eol	audio/vnd.digital-winds
 .dra	audio/vnd.dra
@@ -688,7 +688,7 @@ ext	type
 .mp4	video/mp4
 .mpeg	video/mpeg
 .ogv	video/ogg
-.qt	video/quicktime
+.mov	video/quicktime
 .uvh	video/vnd.dece.hd
 .uvm	video/vnd.dece.mobile
 .uvp	video/vnd.dece.pd

data/misc/port.json

@@ -0,0 +1 @@
+{ "format": "{number}", "rows": "port.tsv", "key": "number" }

data/misc/protocol.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "protocol.tsv", "key": "keyword", "name": "name" }

data/misc/protocol.tsv

@@ -0,0 +1,140 @@
+keyword	name	number
+HOPOPT	IPv6 Hop-by-Hop Option	0
+ICMP	Internet Control Message	1
+IGMP	Internet Group Management	2
+GGP	Gateway-to-Gateway	3
+IPv4	IPv4 encapsulation	4
+ST	Stream	5
+TCP	Transmission Control	6
+CBT	CBT	7
+EGP	Exterior Gateway Protocol	8
+IGP	any private interior gateway (used by Cisco for their IGRP)	9
+BBN-RCC-MON	BBN RCC Monitoring	10
+NVP-II	Network Voice Protocol	11
+PUP	PUP	12
+EMCON	EMCON	14
+XNET	Cross Net Debugger	15
+CHAOS	Chaos	16
+UDP	User Datagram	17
+MUX	Multiplexing	18
+DCN-MEAS	DCN Measurement Subsystems	19
+HMP	Host Monitoring	20
+PRM	Packet Radio Measurement	21
+XNS-IDP	XEROX NS IDP	22
+TRUNK-1	Trunk-1	23
+TRUNK-2	Trunk-2	24
+LEAF-1	Leaf-1	25
+LEAF-2	Leaf-2	26
+RDP	Reliable Data Protocol	27
+IRTP	Internet Reliable Transaction	28
+ISO-TP4	ISO Transport Protocol Class 4	29
+NETBLT	Bulk Data Transfer Protocol	30
+MFE-NSP	MFE Network Services Protocol	31
+MERIT-INP	MERIT Internodal Protocol	32
+DCCP	Datagram Congestion Control Protocol	33
+3PC	Third Party Connect Protocol	34
+IDPR	Inter-Domain Policy Routing Protocol	35
+XTP	XTP	36
+DDP	Datagram Delivery Protocol	37
+IDPR-CMTP	IDPR Control Message Transport Proto	38
+TP++	TP++ Transport Protocol	39
+IL	IL Transport Protocol	40
+IPv6	IPv6 encapsulation	41
+SDRP	Source Demand Routing Protocol	42
+IPv6-Route	Routing Header for IPv6	43
+IPv6-Frag	Fragment Header for IPv6	44
+IDRP	Inter-Domain Routing Protocol	45
+RSVP	Reservation Protocol	46
+GRE	Generic Routing Encapsulation	47
+DSR	Dynamic Source Routing Protocol	48
+BNA	BNA	49
+ESP	Encap Security Payload	50
+AH	Authentication Header	51
+I-NLSP	Integrated Net Layer Security TUBA	52
+NARP	NBMA Address Resolution Protocol	54
+Min-IPv4	Minimal IPv4 Encapsulation	55
+TLSP	Transport Layer Security Protocol using Kryptonet key management	56
+SKIP	SKIP	57
+IPv6-ICMP	ICMP for IPv6	58
+IPv6-NoNxt	No Next Header for IPv6	59
+IPv6-Opts	Destination Options for IPv6	60
+CFTP	CFTP	62
+SAT-EXPAK	SATNET and Backroom EXPAK	64
+KRYPTOLAN	Kryptolan	65
+RVD	MIT Remote Virtual Disk Protocol	66
+IPPC	Internet Pluribus Packet Core	67
+SAT-MON	SATNET Monitoring	69
+VISA	VISA Protocol	70
+IPCV	Internet Packet Core Utility	71
+CPNX	Computer Protocol Network Executive	72
+CPHB	Computer Protocol Heart Beat	73
+WSN	Wang Span Network	74
+PVP	Packet Video Protocol	75
+BR-SAT-MON	Backroom SATNET Monitoring	76
+SUN-ND	SUN ND PROTOCOL-Temporary	77
+WB-MON	WIDEBAND Monitoring	78
+WB-EXPAK	WIDEBAND EXPAK	79
+ISO-IP	ISO Internet Protocol	80
+VMTP	VMTP	81
+SECURE-VMTP	SECURE-VMTP	82
+VINES	VINES	83
+IPTM	Internet Protocol Traffic Manager	84
+NSFNET-IGP	NSFNET-IGP	85
+DGP	Dissimilar Gateway Protocol	86
+TCF	TCF	87
+EIGRP	EIGRP	88
+OSPFIGP	OSPFIGP	89
+Sprite-RPC	Sprite RPC Protocol	90
+LARP	Locus Address Resolution Protocol	91
+MTP	Multicast Transport Protocol	92
+AX.25	AX.25 Frames	93
+IPIP	IP-within-IP Encapsulation Protocol	94
+SCC-SP	Semaphore Communications Sec. Pro.	96
+ETHERIP	Ethernet-within-IP Encapsulation	97
+ENCAP	Encapsulation Header	98
+GMTP	GMTP	100
+IFMP	Ipsilon Flow Management Protocol	101
+PNNI	PNNI over IP	102
+PIM	Protocol Independent Multicast	103
+ARIS	ARIS	104
+SCPS	SCPS	105
+QNX	QNX	106
+A/N	Active Networks	107
+IPComp	IP Payload Compression Protocol	108
+SNP	Sitara Networks Protocol	109
+Compaq-Peer	Compaq Peer Protocol	110
+IPX-in-IP	IPX in IP	111
+VRRP	Virtual Router Redundancy Protocol	112
+PGM	PGM Reliable Transport Protocol	113
+L2TP	Layer Two Tunneling Protocol	115
+DDX	D-II Data Exchange (DDX)	116
+IATP	Interactive Agent Transfer Protocol	117
+STP	Schedule Transfer Protocol	118
+SRP	SpectraLink Radio Protocol	119
+UTI	UTI	120
+SMP	Simple Message Protocol	121
+PTP	Performance Transparency Protocol	123
+ISIS over IPv4	ISIS over IPv4	124
+FIRE	FIRE	125
+CRTP	Combat Radio Transport Protocol	126
+CRUDP	Combat Radio User Datagram	127
+SSCOPMCE	SSCOPMCE	128
+IPLT	IPLT	129
+SPS	Secure Packet Shield	130
+PIPE	Private IP Encapsulation within IP	131
+SCTP	Stream Control Transmission Protocol	132
+FC	Fibre Channel	133
+RSVP-E2E-IGNORE	RSVP-E2E-IGNORE	134
+Mobility Header	Mobility Header	135
+UDPLite	UDPLite	136
+MPLS-in-IP	MPLS-in-IP	137
+manet	MANET Protocols	138
+HIP	Host Identity Protocol	139
+Shim6	Shim6 Protocol	140
+WESP	Wrapped Encapsulating Security Payload	141
+ROHC	Robust Header Compression	142
+Ethernet	Ethernet	143
+AGGFRAG	AGGFRAG encapsulation payload for ESP	144
+NSH	Network Service Header	145
+Homa	Homa	146
+BIT-EMU	Bit-stream Emulation	147

data/misc/territory.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "territory.tsv", "key": "alpha2", "name": "name" }

data/misc/territory.tsv

@@ -0,0 +1,242 @@
+alpha2	alpha3	calling-code	capital	country	currency	flag	languages	name	numeric	tld
+AD	AND	376	Andorra la Vella	AD	EUR	🇦🇩	ca	Andorra	020	.ad
+AE	ARE	971	Abu Dhabi	AE	AED	🇦🇪	ar-AE,fa,en,hi,ur	United Arab Emirates	784	.ae
+AF	AFG	93	Kabul	AF	AFN	🇦🇫	fa-AF,ps,uz-AF,tk	Afghanistan	004	.af
+AG	ATG	1-268	St. John's	AG	XCD	🇦🇬	en-AG	Antigua & Barbuda	028	.ag
+AI	AIA	1-264	The Valley	GB	XCD	🇦🇮	en-AI	Anguilla	660	.ai
+AL	ALB	355	Tirana	AL	ALL	🇦🇱	sq,el	Albania	008	.al
+AM	ARM	374	Yerevan	AM	AMD	🇦🇲	hy	Armenia	051	.am
+AO	AGO	244	Luanda	AO	AOA	🇦🇴	pt-AO	Angola	024	.ao
+AR	ARG	54	Buenos Aires	AR	ARS	🇦🇷	es-AR,en,it,de,fr,gn	Argentina	032	.ar
+AS	ASM	1-684	Pago Pago	US	USD	🇦🇸	en-AS,sm,to	American Samoa	016	.as
+AT	AUT	43	Vienna	AT	EUR	🇦🇹	de-AT,hr,hu,sl	Austria	040	.at
+AU	AUS	61	Canberra	AU	AUD	🇦🇺	en-AU	Australia	036	.au
+AW	ABW	297	Oranjestad	NL	AWG	🇦🇼	nl-AW,pap,es,en	Aruba	533	.aw
+AX	ALA	358	Mariehamn	FI	EUR	🇦🇽	sv-AX	Åland Islands	248	.ax
+AZ	AZE	994	Baku	AZ	AZN	🇦🇿	az,ru,hy	Azerbaijan	031	.az
+BA	BIH	387	Sarajevo	BA	BAM	🇧🇦	bs,hr-BA,sr-BA	Bosnia	070	.ba
+BB	BRB	1-246	Bridgetown	BB	BBD	🇧🇧	en-BB	Barbados	052	.bb
+BD	BGD	880	Dhaka	BD	BDT	🇧🇩	bn-BD,en	Bangladesh	050	.bd
+BE	BEL	32	Brussels	BE	EUR	🇧🇪	nl-BE,fr-BE,de-BE	Belgium	056	.be
+BF	BFA	226	Ouagadougou	BF	XOF	🇧🇫	fr-BF,mos	Burkina Faso	854	.bf
+BG	BGR	359	Sofia	BG	EUR	🇧🇬	bg,tr-BG,rom	Bulgaria	100	.bg
+BH	BHR	973	Manama	BH	BHD	🇧🇭	ar-BH,en,fa,ur	Bahrain	048	.bh
+BI	BDI	257	Gitega	BI	BIF	🇧🇮	fr-BI,rn	Burundi	108	.bi
+BJ	BEN	229	Porto-Novo	BJ	XOF	🇧🇯	fr-BJ	Benin	204	.bj
+BL	BLM	590	Gustavia	FR	EUR	🇧🇱	fr	St. Barthélemy	652	.gp
+BM	BMU	1-441	Hamilton	GB	BMD	🇧🇲	en-BM,pt	Bermuda	060	.bm
+BN	BRN	673	Bandar Seri Begawan	BN	BND	🇧🇳	ms-BN,en-BN	Brunei	096	.bn
+BO	BOL	591	Sucre	BO	BOB	🇧🇴	es-BO,qu,ay	Bolivia	068	.bo
+BR	BRA	55	Brasilia	BR	BRL	🇧🇷	pt-BR,es,en,fr	Brazil	076	.br
+BS	BHS	1-242	Nassau	BS	BSD	🇧🇸	en-BS	Bahamas	044	.bs
+BT	BTN	975	Thimphu	BT	INR	🇧🇹	dz	Bhutan	064	.bt
+BW	BWA	267	Gaborone	BW	BWP	🇧🇼	en-BW,tn-BW	Botswana	072	.bw
+BY	BLR	375	Minsk	BY	BYN	🇧🇾	be,ru	Belarus	112	.by
+BZ	BLZ	501	Belmopan	BZ	BZD	🇧🇿	en-BZ,es	Belize	084	.bz
+CA	CAN	1	Ottawa	CA	CAD	🇨🇦	en-CA,fr-CA,iu	Canada	124	.ca
+CC	CCK	61	West Island	AU	AUD	🇨🇨	ms-CC,en	Cocos Islands	166	.cc
+CD	COD	243	Kinshasa	CD	CDF	🇨🇩	fr-CD,ln,ktu,kg,sw,lua	Congo - Kinshasa	180	.cd
+CF	CAF	236	Bangui	CF	XAF	🇨🇫	fr-CF,sg,ln,kg	Central African Republic	140	.cf
+CG	COG	242	Brazzaville	CG	XAF	🇨🇬	fr-CG,kg,ln-CG	Congo - Brazzaville	178	.cg
+CH	CHE	41	Bern	CH	CHF	🇨🇭	de-CH,fr-CH,it-CH,rm	Switzerland	756	.ch
+CI	CIV	225	Yamoussoukro	CI	XOF	🇨🇮	fr-CI	Côte d’Ivoire	384	.ci
+CK	COK	682	Avarua	NZ	NZD	🇨🇰	en-CK,mi	Cook Islands	184	.ck
+CL	CHL	56	Santiago	CL	CLP	🇨🇱	es-CL	Chile	152	.cl
+CM	CMR	237	Yaounde	CM	XAF	🇨🇲	en-CM,fr-CM	Cameroon	120	.cm
+CN	CHN	86	Beijing	CN	CNY	🇨🇳	zh-CN,yue,wuu,dta,ug,za	China	156	.cn
+CO	COL	57	Bogota	CO	COP	🇨🇴	es-CO	Colombia	170	.co
+CR	CRI	506	San Jose	CR	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
+CU	CUB	53	Havana	CU	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
+CV	CPV	238	Praia	CV	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
+CW	CUW	599	Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
+CX	CXR	61	Flying Fish Cove	AU	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
+CY	CYP	357	Nicosia	CY	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
+CZ	CZE	420	Prague	CZ	CZK	🇨🇿	cs,sk	Czechia	203	.cz
+DE	DEU	49	Berlin	DE	EUR	🇩🇪	de	Germany	276	.de
+DJ	DJI	253	Djibouti	DJ	DJF	🇩🇯	fr-DJ,ar,so-DJ,aa	Djibouti	262	.dj
+DK	DNK	45	Copenhagen	DK	DKK	🇩🇰	da-DK,en,fo,de-DK	Denmark	208	.dk
+DM	DMA	1-767	Roseau	DM	XCD	🇩🇲	en-DM	Dominica	212	.dm
+DO	DOM	1-809	Santo Domingo	DO	DOP	🇩🇴	es-DO	Dominican Republic	214	.do
+DZ	DZA	213	Algiers	DZ	DZD	🇩🇿	ar-DZ	Algeria	012	.dz
+EC	ECU	593	Quito	EC	USD	🇪🇨	es-EC	Ecuador	218	.ec
+EE	EST	372	Tallinn	EE	EUR	🇪🇪	et,ru	Estonia	233	.ee
+EG	EGY	20	Cairo	EG	EGP	🇪🇬	ar-EG,en,fr	Egypt	818	.eg
+EH	ESH	212	El-Aaiun	EH	MAD	🇪🇭	ar,mey	Western Sahara	732	.eh
+ER	ERI	291	Asmara	ER	ERN	🇪🇷	aa-ER,ar,tig,kun,ti-ER	Eritrea	232	.er
+ES	ESP	34	Madrid	ES	EUR	🇪🇸	es-ES,ca,gl,eu,oc	Spain	724	.es
+ET	ETH	251	Addis Ababa	ET	ETB	🇪🇹	am,en-ET,om-ET,ti-ET,so-ET,sid	Ethiopia	231	.et
+FI	FIN	358	Helsinki	FI	EUR	🇫🇮	fi-FI,sv-FI,smn	Finland	246	.fi
+FJ	FJI	679	Suva	FJ	FJD	🇫🇯	en-FJ,fj	Fiji	242	.fj
+FK	FLK	500	Stanley	GB	FKP	🇫🇰	en-FK	Falkland Islands	238	.fk
+FM	FSM	691	Palikir	FM	USD	🇫🇲	en-FM,chk,pon,yap,kos,uli,woe,nkr,kpg	Micronesia	583	.fm
+FO	FRO	298	Torshavn	DK	DKK	🇫🇴	fo,da-FO	Faroe Islands	234	.fo
+FR	FRA	33	Paris	FR	EUR	🇫🇷	fr-FR,frp,br,co,ca,eu,oc	France	250	.fr
+GA	GAB	241	Libreville	GA	XAF	🇬🇦	fr-GA	Gabon	266	.ga
+GB	GBR	44	London	GB	GBP	🇬🇧	en-GB,cy-GB,gd	UK	826	.uk
+GD	GRD	1-473	St. George's	GD	XCD	🇬🇩	en-GD	Grenada	308	.gd
+GE	GEO	995	Tbilisi	GE	GEL	🇬🇪	ka,ru,hy,az	Georgia	268	.ge
+GF	GUF	594	Cayenne	FR	EUR	🇬🇫	fr-GF	French Guiana	254	.gf
+GG	GGY	44	St Peter Port	GB	GBP	🇬🇬	en,nrf	Guernsey	831	.gg
+GH	GHA	233	Accra	GH	GHS	🇬🇭	en-GH,ak,ee,tw	Ghana	288	.gh
+GI	GIB	350	Gibraltar	GB	GIP	🇬🇮	en-GI,es,it,pt	Gibraltar	292	.gi
+GL	GRL	299	Nuuk	DK	DKK	🇬🇱	kl,da-GL,en	Greenland	304	.gl
+GM	GMB	220	Banjul	GM	GMD	🇬🇲	en-GM,mnk,wof,wo,ff	Gambia	270	.gm
+GN	GIN	224	Conakry	GN	GNF	🇬🇳	fr-GN	Guinea	324	.gn
+GP	GLP	590	Basse-Terre	FR	EUR	🇬🇵	fr-GP	Guadeloupe	312	.gp
+GQ	GNQ	240	Ciudad de la Paz	GQ	XAF	🇬🇶	es-GQ,fr,pt	Equatorial Guinea	226	.gq
+GR	GRC	30	Athens	GR	EUR	🇬🇷	el-GR,en,fr	Greece	300	.gr
+GT	GTM	502	Guatemala City	GT	GTQ	🇬🇹	es-GT	Guatemala	320	.gt
+GU	GUM	1-671	Hagatna	US	USD	🇬🇺	en-GU,ch-GU	Guam	316	.gu
+GW	GNB	245	Bissau	GW	XOF	🇬🇼	pt-GW,pov	Guinea-Bissau	624	.gw
+GY	GUY	592	Georgetown	GY	GYD	🇬🇾	en-GY	Guyana	328	.gy
+HK	HKG	852	Hong Kong	CN	HKD	🇭🇰	zh-HK,yue,zh,en	Hong Kong	344	.hk
+HN	HND	504	Tegucigalpa	HN	HNL	🇭🇳	es-HN,cab,miq	Honduras	340	.hn
+HR	HRV	385	Zagreb	HR	EUR	🇭🇷	hr-HR,sr	Croatia	191	.hr
+HT	HTI	509	Port-au-Prince	HT	HTG	🇭🇹	ht,fr-HT	Haiti	332	.ht
+HU	HUN	36	Budapest	HU	HUF	🇭🇺	hu-HU	Hungary	348	.hu
+ID	IDN	62	Jakarta	ID	IDR	🇮🇩	id,en,nl,jv	Indonesia	360	.id
+IE	IRL	353	Dublin	IE	EUR	🇮🇪	en-IE,ga-IE	Ireland	372	.ie
+IL	ISR	972	Jerusalem	IL	ILS	🇮🇱	he,ar-IL,en-IL	Israel	376	.il
+IM	IMN	44	Douglas	GB	GBP	🇮🇲	en,gv	Isle of Man	833	.im
+IN	IND	91	New Delhi	IN	INR	🇮🇳	en-IN,hi,bn,te,mr,ta,ur,gu,kn,ml,or,pa,as,bh,sat,ks,ne,sd,kok,doi,mni,sit,sa,fr,lus,inc	India	356	.in
+IO	IOT	246	Diego Garcia	GB	USD	🇮🇴	en-IO	British Indian Ocean Territory	086	.io
+IQ	IRQ	964	Baghdad	IQ	IQD	🇮🇶	ar-IQ,ku,hy	Iraq	368	.iq
+IR	IRN	98	Tehran	IR	IRR	🇮🇷	fa-IR,ku	Iran	364	.ir
+IS	ISL	354	Reykjavik	IS	ISK	🇮🇸	is,en,de,da,sv,no	Iceland	352	.is
+IT	ITA	39	Rome	IT	EUR	🇮🇹	it-IT,de-IT,fr-IT,sc,ca,co,sl	Italy	380	.it
+JE	JEY	44	Saint Helier	GB	GBP	🇯🇪	en,fr,nrf	Jersey	832	.je
+JM	JAM	1-876	Kingston	JM	JMD	🇯🇲	en-JM	Jamaica	388	.jm
+JO	JOR	962	Amman	JO	JOD	🇯🇴	ar-JO,en	Jordan	400	.jo
+JP	JPN	81	Tokyo	JP	JPY	🇯🇵	ja	Japan	392	.jp
+KE	KEN	254	Nairobi	KE	KES	🇰🇪	en-KE,sw-KE	Kenya	404	.ke
+KG	KGZ	996	Bishkek	KG	KGS	🇰🇬	ky,uz,ru	Kyrgyzstan	417	.kg
+KH	KHM	855	Phnom Penh	KH	KHR	🇰🇭	km,fr,en	Cambodia	116	.kh
+KI	KIR	686	Tarawa	KI	AUD	🇰🇮	en-KI,gil	Kiribati	296	.ki
+KM	COM	269	Moroni	KM	KMF	🇰🇲	ar,fr-KM	Comoros	174	.km
+KN	KNA	1-869	Basseterre	KN	XCD	🇰🇳	en-KN	St. Kitts & Nevis	659	.kn
+KP	PRK	850	Pyongyang	KP	KPW	🇰🇵	ko-KP	North Korea	408	.kp
+KR	KOR	82	Seoul	KR	KRW	🇰🇷	ko-KR,en	South Korea	410	.kr
+KW	KWT	965	Kuwait City	KW	KWD	🇰🇼	ar-KW,en	Kuwait	414	.kw
+KY	CYM	1-345	George Town	GB	KYD	🇰🇾	en-KY	Cayman Islands	136	.ky
+KZ	KAZ	7	Nur-Sultan	KZ	KZT	🇰🇿	kk,ru	Kazakhstan	398	.kz
+LA	LAO	856	Vientiane	LA	LAK	🇱🇦	lo,fr,en	Laos	418	.la
+LB	LBN	961	Beirut	LB	LBP	🇱🇧	ar-LB,fr-LB,en,hy	Lebanon	422	.lb
+LC	LCA	1-758	Castries	LC	XCD	🇱🇨	en-LC	St. Lucia	662	.lc
+LI	LIE	423	Vaduz	LI	CHF	🇱🇮	de-LI	Liechtenstein	438	.li
+LK	LKA	94	Colombo	LK	LKR	🇱🇰	si,ta,en	Sri Lanka	144	.lk
+LR	LBR	231	Monrovia	LR	LRD	🇱🇷	en-LR	Liberia	430	.lr
+LS	LSO	266	Maseru	LS	LSL	🇱🇸	en-LS,st,zu,xh	Lesotho	426	.ls
+LT	LTU	370	Vilnius	LT	EUR	🇱🇹	lt,ru,pl	Lithuania	440	.lt
+LU	LUX	352	Luxembourg	LU	EUR	🇱🇺	lb,de-LU,fr-LU	Luxembourg	442	.lu
+LV	LVA	371	Riga	LV	EUR	🇱🇻	lv,ru,lt	Latvia	428	.lv
+LY	LBY	218	Tripoli	LY	LYD	🇱🇾	ar-LY,it,en	Libya	434	.ly
+MA	MAR	212	Rabat	MA	MAD	🇲🇦	ar-MA,ber,fr	Morocco	504	.ma
+MC	MCO	377	Monaco	MC	EUR	🇲🇨	fr-MC,en,it	Monaco	492	.mc
+MD	MDA	373	Chisinau	MD	MDL	🇲🇩	ro,ru,gag,tr	Moldova	498	.md
+ME	MNE	382	Podgorica	ME	EUR	🇲🇪	sr,hu,bs,sq,hr,rom	Montenegro	499	.me
+MF	MAF	590	Marigot	FR	EUR	🇲🇫	fr	St. Martin	663	.gp
+MG	MDG	261	Antananarivo	MG	MGA	🇲🇬	fr-MG,mg	Madagascar	450	.mg
+MH	MHL	692	Majuro	MH	USD	🇲🇭	mh,en-MH	Marshall Islands	584	.mh
+MK	MKD	389	Skopje	MK	MKD	🇲🇰	mk,sq,tr,rmm,sr	North Macedonia	807	.mk
+ML	MLI	223	Bamako	ML	XOF	🇲🇱	fr-ML,bm	Mali	466	.ml
+MM	MMR	95	Nay Pyi Taw	MM	MMK	🇲🇲	my	Myanmar	104	.mm
+MN	MNG	976	Ulaanbaatar	MN	MNT	🇲🇳	mn,ru	Mongolia	496	.mn
+MO	MAC	853	Macao	CN	MOP	🇲🇴	zh,zh-MO,pt	Macao	446	.mo
+MP	MNP	1-670	Saipan	US	USD	🇲🇵	fil,tl,zh,ch-MP,en-MP	Northern Mariana Islands	580	.mp
+MQ	MTQ	596	Fort-de-France	FR	EUR	🇲🇶	fr-MQ	Martinique	474	.mq
+MR	MRT	222	Nouakchott	MR	MRU	🇲🇷	ar-MR,fuc,snk,fr,mey,wo	Mauritania	478	.mr
+MS	MSR	1-664	Plymouth	GB	XCD	🇲🇸	en-MS	Montserrat	500	.ms
+MT	MLT	356	Valletta	MT	EUR	🇲🇹	mt,en-MT	Malta	470	.mt
+MU	MUS	230	Port Louis	MU	MUR	🇲🇺	en-MU,bho,fr	Mauritius	480	.mu
+MV	MDV	960	Male	MV	MVR	🇲🇻	dv,en	Maldives	462	.mv
+MW	MWI	265	Lilongwe	MW	MWK	🇲🇼	ny,yao,tum,swk	Malawi	454	.mw
+MX	MEX	52	Mexico City	MX	MXN	🇲🇽	es-MX	Mexico	484	.mx
+MY	MYS	60	Kuala Lumpur	MY	MYR	🇲🇾	ms-MY,en,zh,ta,te,ml,pa,th	Malaysia	458	.my
+MZ	MOZ	258	Maputo	MZ	MZN	🇲🇿	pt-MZ,vmw	Mozambique	508	.mz
+NA	NAM	264	Windhoek	NA	NAD	🇳🇦	en-NA,af,de,hz,naq	Namibia	516	.na
+NC	NCL	687	Noumea	FR	XPF	🇳🇨	fr-NC	New Caledonia	540	.nc
+NE	NER	227	Niamey	NE	XOF	🇳🇪	fr-NE,ha,kr,dje	Niger	562	.ne
+NF	NFK	672	Kingston	AU	AUD	🇳🇫	en-NF	Norfolk Island	574	.nf
+NG	NGA	234	Abuja	NG	NGN	🇳🇬	en-NG,ha,yo,ig,ff	Nigeria	566	.ng
+NI	NIC	505	Managua	NI	NIO	🇳🇮	es-NI,en	Nicaragua	558	.ni
+NL	NLD	31	Amsterdam	NL	EUR	🇳🇱	nl-NL,fy-NL	Netherlands	528	.nl
+NO	NOR	47	Oslo	NO	NOK	🇳🇴	no,nb,nn,se,fi	Norway	578	.no
+NP	NPL	977	Kathmandu	NP	NPR	🇳🇵	ne,en	Nepal	524	.np
+NR	NRU	674	Yaren	NR	AUD	🇳🇷	na,en-NR	Nauru	520	.nr
+NU	NIU	683	Alofi	NZ	NZD	🇳🇺	niu,en-NU	Niue	570	.nu
+NZ	NZL	64	Wellington	NZ	NZD	🇳🇿	en-NZ,mi	New Zealand	554	.nz
+OM	OMN	968	Muscat	OM	OMR	🇴🇲	ar-OM,en,bal,ur	Oman	512	.om
+PA	PAN	507	Panama City	PA	PAB	🇵🇦	es-PA,en	Panama	591	.pa
+PE	PER	51	Lima	PE	PEN	🇵🇪	es-PE,qu,ay	Peru	604	.pe
+PF	PYF	689	Papeete	FR	XPF	🇵🇫	fr-PF,ty	French Polynesia	258	.pf
+PG	PNG	675	Port Moresby	PG	PGK	🇵🇬	en-PG,ho,meu,tpi	Papua New Guinea	598	.pg
+PH	PHL	63	Manila	PH	PHP	🇵🇭	tl,en-PH,fil,ceb,ilo,hil,war,pam,bik,bcl,pag,mrw,tsg,mdh,cbk,krj,sgd,msb,akl,ibg,yka,mta,abx	Philippines	608	.ph
+PK	PAK	92	Islamabad	PK	PKR	🇵🇰	ur-PK,en-PK,pa,sd,ps,brh	Pakistan	586	.pk
+PL	POL	48	Warsaw	PL	PLN	🇵🇱	pl	Poland	616	.pl
+PM	SPM	508	Saint-Pierre	FR	EUR	🇵🇲	fr-PM	St. Pierre & Miquelon	666	.pm
+PN	PCN	870	Adamstown	GB	NZD	🇵🇳	en-PN	Pitcairn	612	.pn
+PR	PRI	1	San Juan	US	USD	🇵🇷	en-PR,es-PR	Puerto Rico	630	.pr
+PT	PRT	351	Lisbon	PT	EUR	🇵🇹	pt-PT,mwl	Portugal	620	.pt
+PW	PLW	680	Melekeok	PW	USD	🇵🇼	pau,sov,en-PW,tox,ja,fil,zh	Palau	585	.pw
+PY	PRY	595	Asuncion	PY	PYG	🇵🇾	es-PY,gn	Paraguay	600	.py
+QA	QAT	974	Doha	QA	QAR	🇶🇦	ar-QA,es	Qatar	634	.qa
+RE	REU	262	Saint-Denis	FR	EUR	🇷🇪	fr-RE	Réunion	638	.re
+RO	ROU	40	Bucharest	RO	RON	🇷🇴	ro,hu,rom	Romania	642	.ro
+RS	SRB	381	Belgrade	RS	RSD	🇷🇸	sr,hu,bs,rom	Serbia	688	.rs
+RU	RUS	7	Moscow	RU	RUB	🇷🇺	ru,tt,xal,cau,ady,kv,ce,tyv,cv,udm,tut,mns,bua,myv,mdf,chm,ba,inh,kbd,krc,av,sah,nog	Russia	643	.ru
+RW	RWA	250	Kigali	RW	RWF	🇷🇼	rw,en-RW,fr-RW,sw	Rwanda	646	.rw
+SA	SAU	966	Riyadh	SA	SAR	🇸🇦	ar-SA	Saudi Arabia	682	.sa
+SB	SLB	677	Honiara	SB	SBD	🇸🇧	en-SB,tpi	Solomon Islands	090	.sb
+SC	SYC	248	Victoria	SC	SCR	🇸🇨	en-SC,fr-SC	Seychelles	690	.sc
+SD	SDN	249	Khartoum	SD	SDG	🇸🇩	ar-SD,en,fia	Sudan	729	.sd
+SE	SWE	46	Stockholm	SE	SEK	🇸🇪	sv-SE,se,sma,fi-SE	Sweden	752	.se
+SG	SGP	65	Singapore	SG	SGD	🇸🇬	cmn,en-SG,ms-SG,ta-SG,zh-SG	Singapore	702	.sg
+SH	SHN	290	Jamestown	GB	SHP	🇸🇭	en-SH	St. Helena	654	.sh
+SI	SVN	386	Ljubljana	SI	EUR	🇸🇮	sl,sh	Slovenia	705	.si
+SJ	SJM	47	Longyearbyen	NO	NOK	🇸🇯	no,ru	Svalbard & Jan Mayen	744	.sj
+SK	SVK	421	Bratislava	SK	EUR	🇸🇰	sk,hu	Slovakia	703	.sk
+SL	SLE	232	Freetown	SL	SLE	🇸🇱	en-SL,men,tem	Sierra Leone	694	.sl
+SM	SMR	378	San Marino	SM	EUR	🇸🇲	it-SM	San Marino	674	.sm
+SN	SEN	221	Dakar	SN	XOF	🇸🇳	fr-SN,wo,fuc,mnk	Senegal	686	.sn
+SO	SOM	252	Mogadishu	SO	SOS	🇸🇴	so-SO,ar-SO,it,en-SO	Somalia	706	.so
+SR	SUR	597	Paramaribo	SR	SRD	🇸🇷	nl-SR,en,srn,hns,jv	Suriname	740	.sr
+SS	SSD	211	Juba	SS	SSP	🇸🇸	en	South Sudan	728	.ss
+ST	STP	239	Sao Tome	ST	STN	🇸🇹	pt-ST	São Tomé & Príncipe	678	.st
+SV	SLV	503	San Salvador	SV	SVC	🇸🇻	es-SV	El Salvador	222	.sv
+SX	SXM	1-721	Philipsburg	NL	XCG	🇸🇽	nl,en	Sint Maarten	534	.sx
+SY	SYR	963	Damascus	SY	SYP	🇸🇾	ar-SY,ku,hy,arc,fr,en	Syria	760	.sy
+SZ	SWZ	268	Mbabane	SZ	SZL	🇸🇿	en-SZ,ss-SZ	Eswatini	748	.sz
+TC	TCA	1-649	Cockburn Town	GB	USD	🇹🇨	en-TC	Turks & Caicos Islands	796	.tc
+TD	TCD	235	N'Djamena	TD	XAF	🇹🇩	fr-TD,ar-TD,sre	Chad	148	.td
+TF	ATF	262	Port-aux-Francais	FR	EUR	🇹🇫	fr	French Southern Territories	260	.tf
+TG	TGO	228	Lome	TG	XOF	🇹🇬	fr-TG,ee,hna,kbp,dag,ha	Togo	768	.tg
+TH	THA	66	Bangkok	TH	THB	🇹🇭	th,en	Thailand	764	.th
+TJ	TJK	992	Dushanbe	TJ	TJS	🇹🇯	tg,ru	Tajikistan	762	.tj
+TL	TLS	670	Dili	TL	USD	🇹🇱	tet,pt-TL,id,en	Timor-Leste	626	.tl
+TM	TKM	993	Ashgabat	TM	TMT	🇹🇲	tk,ru,uz	Turkmenistan	795	.tm
+TN	TUN	216	Tunis	TN	TND	🇹🇳	ar-TN,fr	Tunisia	788	.tn
+TO	TON	676	Nuku'alofa	TO	TOP	🇹🇴	to,en-TO	Tonga	776	.to
+TR	TUR	90	Ankara	TR	TRY	🇹🇷	tr-TR,ku,diq,az,av	Türkiye	792	.tr
+TT	TTO	1-868	Port of Spain	TT	TTD	🇹🇹	en-TT,hns,fr,es,zh	Trinidad & Tobago	780	.tt
+TV	TUV	688	Funafuti	TV	AUD	🇹🇻	tvl,en,sm,gil	Tuvalu	798	.tv
+TW	TWN	886	Taipei	TW	TWD	🇹🇼	zh-TW,zh,nan,hak	Taiwan	158	.tw
+TZ	TZA	255	Dodoma	TZ	TZS	🇹🇿	sw-TZ,en,ar	Tanzania	834	.tz
+UA	UKR	380	Kyiv	UA	UAH	🇺🇦	uk,ru-UA,rom,pl,hu	Ukraine	804	.ua
+UG	UGA	256	Kampala	UG	UGX	🇺🇬	en-UG,lg,sw,ar	Uganda	800	.ug
+US	USA	1	Washington	US	USD	🇺🇸	en-US,es-US,haw,fr	US	840	.us
+UY	URY	598	Montevideo	UY	UYU	🇺🇾	es-UY	Uruguay	858	.uy
+UZ	UZB	998	Tashkent	UZ	UZS	🇺🇿	uz,ru,tg	Uzbekistan	860	.uz
+VA	VAT	39	Vatican City	VA	EUR	🇻🇦	la,it,fr	Vatican City	336	.va
+VC	VCT	1-784	Kingstown	VC	XCD	🇻🇨	en-VC,fr	St. Vincent & Grenadines	670	.vc
+VE	VEN	58	Caracas	VE	VES	🇻🇪	es-VE	Venezuela	862	.ve
+VG	VGB	1-284	Road Town	GB	USD	🇻🇬	en-VG	British Virgin Islands	092	.vg
+VI	VIR	1-340	Charlotte Amalie	US	USD	🇻🇮	en-VI	U.S. Virgin Islands	850	.vi
+VN	VNM	84	Hanoi	VN	VND	🇻🇳	vi,en,fr,zh,km	Vietnam	704	.vn
+VU	VUT	678	Port Vila	VU	VUV	🇻🇺	bi,en-VU,fr-VU	Vanuatu	548	.vu
+WF	WLF	681	Mata Utu	FR	XPF	🇼🇫	wls,fud,fr-WF	Wallis & Futuna	876	.wf
+WS	WSM	685	Apia	WS	WST	🇼🇸	sm,en-WS	Samoa	882	.ws
+YE	YEM	967	Sanaa	YE	YER	🇾🇪	ar-YE	Yemen	887	.ye
+YT	MYT	262	Mamoudzou	FR	EUR	🇾🇹	fr-YT	Mayotte	175	.yt
+ZA	ZAF	27	Pretoria	ZA	ZAR	🇿🇦	zu,xh,af,nso,en-ZA,tn,st,ts,ss,ve,nr	South Africa	710	.za
+ZM	ZMB	260	Lusaka	ZM	ZMW	🇿🇲	en-ZM,bem,loz,lun,lue,ny,toi	Zambia	894	.zm
+ZW	ZWE	263	Harare	ZW	ZWG	🇿🇼	en-ZW,sn,nr,nd	Zimbabwe	716	.zw

data/misc/timezone.json

@@ -1 +1 @@
-["UTC", "Africa/Cairo", "Africa/Johannesburg", "America/Chicago", "America/Denver", "America/Los_Angeles", "America/Mexico_City", "America/New_York", "America/Sao_Paulo", "America/Toronto", "Asia/Dubai", "Asia/Hong_Kong", "Asia/Kolkata", "Asia/Shanghai", "Asia/Singapore", "Asia/Tokyo", "Australia/Sydney", "Europe/Amsterdam", "Europe/Berlin", "Europe/London", "Europe/Madrid", "Europe/Moscow", "Europe/Oslo", "Europe/Paris", "Europe/Stockholm", "Europe/Warsaw", "Pacific/Auckland"]
+{ "format": "{zone}", "rows": "timezone.tsv", "key": "zone", "parent": "territory", "weight": "weight" }

data/misc/timezone.tsv

@@ -0,0 +1,402 @@
+offset	territory	weight	zone
++00:00	CI	17496811	Africa/Abidjan
++00:00	GH	11277813	Africa/Accra
++03:00	ET	18690487	Africa/Addis_Ababa
++01:00	DZ	20111539	Africa/Algiers
++03:00	ER	798575	Africa/Asmara
++00:00	ML	7673493	Africa/Bamako
++01:00	CF	2582076	Africa/Bangui
++00:00	GM	1305981	Africa/Banjul
++00:00	GW	904776	Africa/Bissau
++02:00	MW	3030905	Africa/Blantyre
++01:00	CG	3748808	Africa/Brazzaville
++02:00	BI	1251675	Africa/Bujumbura
++02:00	EG	47242427	Africa/Cairo
++00:00	MA	22808531	Africa/Casablanca
++01:00	ES	85985	Africa/Ceuta
++00:00	GN	6679241	Africa/Conakry
++00:00	SN	10055356	Africa/Dakar
++03:00	TZ	21350525	Africa/Dar_es_Salaam
++03:00	DJ	1316108	Africa/Djibouti
++01:00	CM	8330801	Africa/Douala
++00:00	EH	374107	Africa/El_Aaiun
++00:00	SL	1752243	Africa/Freetown
++02:00	BW	1262880	Africa/Gaborone
++02:00	ZW	4508424	Africa/Harare
++02:00	ZA	47939803	Africa/Johannesburg
++02:00	SS	1679674	Africa/Juba
++03:00	UG	9664318	Africa/Kampala
++02:00	SD	11781926	Africa/Khartoum
++02:00	RW	2910190	Africa/Kigali
++01:00	CD	20651243	Africa/Kinshasa
++01:00	NG	75725576	Africa/Lagos
++01:00	GA	1731692	Africa/Libreville
++00:00	TG	3089219	Africa/Lome
++01:00	AO	23567341	Africa/Luanda
++02:00	CD	16660579	Africa/Lubumbashi
++02:00	ZM	8422414	Africa/Lusaka
++01:00	GQ	355840	Africa/Malabo
++02:00	MZ	8598836	Africa/Maputo
++02:00	LS	739682	Africa/Maseru
++02:00	SZ	191312	Africa/Mbabane
++03:00	SO	6165052	Africa/Mogadishu
++00:00	LR	2127307	Africa/Monrovia
++03:00	KE	16920989	Africa/Nairobi
++01:00	TD	3363418	Africa/Ndjamena
++01:00	NE	3689982	Africa/Niamey
++00:00	MR	2157288	Africa/Nouakchott
++00:00	BF	6125427	Africa/Ouagadougou
++01:00	BJ	4001576	Africa/Porto-Novo
++00:00	ST	53300	Africa/Sao_Tome
++02:00	LY	6105792	Africa/Tripoli
++01:00	TN	7314557	Africa/Tunis
++02:00	NA	983097	Africa/Windhoek
+-10:00	US	15000	America/Adak
+-09:00	US	366200	America/Anchorage
+-04:00	AI	15000	America/Anguilla
+-04:00	AG	51737	America/Antigua
+-03:00	BR	858988	America/Araguaina
+-03:00	AR	11817784	America/Argentina/Buenos_Aires
+-03:00	AR	580969	America/Argentina/Catamarca
+-03:00	AR	8998034	America/Argentina/Cordoba
+-03:00	AR	413866	America/Argentina/Jujuy
+-03:00	AR	237745	America/Argentina/La_Rioja
+-03:00	AR	695484	America/Argentina/Mendoza
+-03:00	AR	241619	America/Argentina/Rio_Gallegos
+-03:00	AR	2120781	America/Argentina/Salta
+-03:00	AR	518624	America/Argentina/San_Juan
+-03:00	AR	266728	America/Argentina/San_Luis
+-03:00	AR	788916	America/Argentina/Tucuman
+-03:00	AR	109506	America/Argentina/Ushuaia
+-04:00	AW	90927	America/Aruba
+-03:00	PY	3653883	America/Asuncion
+-05:00	CA	15000	America/Atikokan
+-03:00	BR	13131907	America/Bahia
+-06:00	MX	36253	America/Bahia_Banderas
+-04:00	BB	98511	America/Barbados
+-03:00	BR	7977349	America/Belem
+-06:00	BZ	132559	America/Belize
+-04:00	CA	15000	America/Blanc-Sablon
+-04:00	BR	550434	America/Boa_Vista
+-05:00	CO	37243567	America/Bogota
+-07:00	US	748802	America/Boise
+-07:00	CA	15000	America/Cambridge_Bay
+-04:00	BR	2429053	America/Campo_Grande
+-05:00	MX	1328961	America/Cancun
+-04:00	VE	28324666	America/Caracas
+-03:00	GF	155245	America/Cayenne
+-05:00	KY	44705	America/Cayman
+-06:00	US	55510713	America/Chicago
+-06:00	MX	1533843	America/Chihuahua
+-07:00	MX	1512450	America/Ciudad_Juarez
+-06:00	CR	1458629	America/Costa_Rica
+-03:00	CL	62723	America/Coyhaique
+-07:00	CA	15000	America/Creston
+-04:00	BR	2960376	America/Cuiaba
+-04:00	CW	145838	America/Curacao
++00:00	GL	15000	America/Danmarkshavn
+-07:00	CA	15000	America/Dawson
+-07:00	CA	21465	America/Dawson_Creek
+-07:00	US	9069711	America/Denver
+-05:00	US	4344725	America/Detroit
+-04:00	DM	16571	America/Dominica
+-06:00	CA	3518059	America/Edmonton
+-05:00	BR	142900	America/Eirunepe
+-06:00	SV	2571611	America/El_Salvador
+-07:00	CA	15000	America/Fort_Nelson
+-03:00	BR	21824546	America/Fortaleza
+-04:00	CA	122883	America/Glace_Bay
+-04:00	CA	15000	America/Goose_Bay
+-05:00	TC	27489	America/Grand_Turk
+-04:00	GD	15000	America/Grenada
+-04:00	GP	250260	America/Guadeloupe
+-06:00	GT	7302205	America/Guatemala
+-05:00	EC	11564603	America/Guayaquil
+-04:00	GY	314746	America/Guyana
+-04:00	CA	843888	America/Halifax
+-05:00	CU	11972822	America/Havana
+-07:00	MX	2215048	America/Hermosillo
+-05:00	US	2672398	America/Indiana/Indianapolis
+-06:00	US	15000	America/Indiana/Knox
+-05:00	US	15000	America/Indiana/Marengo
+-05:00	US	15000	America/Indiana/Petersburg
+-06:00	US	15000	America/Indiana/Tell_City
+-05:00	US	15000	America/Indiana/Vevay
+-05:00	US	15451	America/Indiana/Vincennes
+-05:00	US	15000	America/Indiana/Winamac
+-06:00	CA	15000	America/Inuvik
+-05:00	CA	15000	America/Iqaluit
+-05:00	JM	2069735	America/Jamaica
+-09:00	US	31555	America/Juneau
+-05:00	US	909379	America/Kentucky/Louisville
+-05:00	US	15000	America/Kentucky/Monticello
+-04:00	BO	6958595	America/La_Paz
+-05:00	PE	21406340	America/Lima
+-08:00	US	46301242	America/Los_Angeles
+-04:00	SX	15000	America/Lower_Princes
+-03:00	BR	4805317	America/Maceio
+-06:00	NI	2621115	America/Managua
+-04:00	BR	3918958	America/Manaus
+-04:00	MF	15000	America/Marigot
+-04:00	MQ	243860	America/Martinique
+-06:00	MX	1899165	America/Matamoros
+-07:00	MX	2951216	America/Mazatlan
+-06:00	US	15000	America/Menominee
+-06:00	MX	4154924	America/Merida
+-09:00	US	15000	America/Metlakatla
+-06:00	MX	61603325	America/Mexico_City
+-03:00	PM	15000	America/Miquelon
+-04:00	CA	381536	America/Moncton
+-06:00	MX	9714766	America/Monterrey
+-03:00	UY	2457462	America/Montevideo
+-04:00	MS	15000	America/Montserrat
+-05:00	BS	319054	America/Nassau
+-05:00	US	88988896	America/New_York
+-09:00	US	15000	America/Nome
+-02:00	BR	15000	America/Noronha
+-06:00	US	15000	America/North_Dakota/Beulah
+-06:00	US	15000	America/North_Dakota/Center
+-06:00	US	21382	America/North_Dakota/New_Salem
+-02:00	GL	15000	America/Nuuk
+-06:00	MX	50784	America/Ojinaga
+-05:00	PA	1692726	America/Panama
+-03:00	SR	454064	America/Paramaribo
+-07:00	US	6541994	America/Phoenix
+-05:00	HT	5328623	America/Port-au-Prince
+-04:00	TT	456832	America/Port_of_Spain
+-04:00	BR	1551818	America/Porto_Velho
+-04:00	PR	1605968	America/Puerto_Rico
+-03:00	CL	135935	America/Punta_Arenas
+-06:00	CA	15000	America/Rankin_Inlet
+-03:00	BR	8832691	America/Recife
+-06:00	CA	982810	America/Regina
+-06:00	CA	15000	America/Resolute
+-05:00	BR	794527	America/Rio_Branco
+-03:00	BR	1403419	America/Santarem
+-04:00	CL	15912038	America/Santiago
+-04:00	DO	7521391	America/Santo_Domingo
+-03:00	BR	129186469	America/Sao_Paulo
+-02:00	GL	15000	America/Scoresbysund
+-09:00	US	15000	America/Sitka
+-04:00	BL	15000	America/St_Barthelemy
+-03:30	CA	202443	America/St_Johns
+-04:00	KN	15000	America/St_Kitts
+-04:00	LC	45210	America/St_Lucia
+-04:00	VI	70601	America/St_Thomas
+-04:00	VC	48723	America/St_Vincent
+-06:00	CA	16604	America/Swift_Current
+-06:00	HN	4808928	America/Tegucigalpa
+-04:00	GL	15000	America/Thule
+-08:00	MX	3797218	America/Tijuana
+-05:00	CA	24763654	America/Toronto
+-04:00	VG	15000	America/Tortola
+-07:00	CA	6340886	America/Vancouver
+-07:00	CA	28201	America/Whitehorse
+-06:00	CA	922566	America/Winnipeg
+-09:00	US	15000	America/Yakutat
++10:00	AU	15000	Antarctica/Macquarie
++01:00	SJ	15000	Arctic/Longyearbyen
++03:00	YE	6904010	Asia/Aden
++05:00	KZ	9055499	Asia/Almaty
++03:00	JO	4950859	Asia/Amman
++12:00	RU	15604	Asia/Anadyr
++05:00	KZ	282409	Asia/Aqtau
++05:00	KZ	598827	Asia/Aqtobe
++05:00	TM	2706037	Asia/Ashgabat
++05:00	KZ	388248	Asia/Atyrau
++03:00	IQ	29681856	Asia/Baghdad
++03:00	BH	815447	Asia/Bahrain
++04:00	AZ	5491302	Asia/Baku
++07:00	TH	49896297	Asia/Bangkok
++07:00	RU	1413356	Asia/Barnaul
++02:00	LB	4467748	Asia/Beirut
++06:00	KG	2233412	Asia/Bishkek
++08:00	BN	230416	Asia/Brunei
++09:00	RU	486311	Asia/Chita
++05:30	LK	4297175	Asia/Colombo
++03:00	SY	10939175	Asia/Damascus
++06:00	BD	35955225	Asia/Dhaka
++09:00	TL	311685	Asia/Dili
++04:00	AE	12931222	Asia/Dubai
++05:00	TJ	2578933	Asia/Dushanbe
++02:00	CY	109959	Asia/Famagusta
++07:00	VN	42129036	Asia/Ho_Chi_Minh
++08:00	HK	24678261	Asia/Hong_Kong
++07:00	MN	122174	Asia/Hovd
++08:00	RU	2231790	Asia/Irkutsk
++07:00	ID	68794489	Asia/Jakarta
++09:00	ID	2512221	Asia/Jayapura
++02:00	IL	7901151	Asia/Jerusalem
++04:30	AF	8764027	Asia/Kabul
++12:00	RU	247112	Asia/Kamchatka
++05:00	PK	88744953	Asia/Karachi
++05:45	NP	5913313	Asia/Kathmandu
++09:00	RU	15000	Asia/Khandyga
++05:30	IN	365373326	Asia/Kolkata
++07:00	RU	2615869	Asia/Krasnoyarsk
++08:00	MY	33322164	Asia/Kuala_Lumpur
++08:00	MY	4013140	Asia/Kuching
++03:00	KW	1662180	Asia/Kuwait
++08:00	MO	1236695	Asia/Macau
++11:00	RU	92782	Asia/Magadan
++08:00	ID	12458359	Asia/Makassar
++08:00	PH	64227523	Asia/Manila
++04:00	OM	3290021	Asia/Muscat
++02:00	CY	679420	Asia/Nicosia
++07:00	RU	2355475	Asia/Novokuznetsk
++07:00	RU	2119272	Asia/Novosibirsk
++06:00	RU	1243165	Asia/Omsk
++05:00	KZ	364799	Asia/Oral
++07:00	KH	4111780	Asia/Phnom_Penh
++07:00	ID	1819584	Asia/Pontianak
++09:00	KP	12876883	Asia/Pyongyang
++03:00	QA	1566088	Asia/Qatar
++05:00	KZ	529104	Asia/Qostanay
++05:00	KZ	511391	Asia/Qyzylorda
++03:00	SA	23913997	Asia/Riyadh
++11:00	RU	326982	Asia/Sakhalin
++05:00	UZ	3864475	Asia/Samarkand
++09:00	KR	47400555	Asia/Seoul
++08:00	CN	861293141	Asia/Shanghai
++08:00	SG	11621053	Asia/Singapore
++11:00	RU	15000	Asia/Srednekolymsk
++08:00	TW	26394699	Asia/Taipei
++05:00	UZ	7452125	Asia/Tashkent
++04:00	GE	1827911	Asia/Tbilisi
++03:30	IR	50648371	Asia/Tehran
++06:00	BT	170210	Asia/Thimphu
++09:00	JP	141330654	Asia/Tokyo
++07:00	RU	783576	Asia/Tomsk
++08:00	MN	1438738	Asia/Ulaanbaatar
++06:00	CN	12667986	Asia/Urumqi
++10:00	RU	15000	Asia/Ust-Nera
++07:00	LA	1561716	Asia/Vientiane
++10:00	RU	2534711	Asia/Vladivostok
++09:00	RU	902983	Asia/Yakutsk
++06:30	MM	18836338	Asia/Yangon
++05:00	RU	15407217	Asia/Yekaterinburg
++04:00	AM	2880620	Asia/Yerevan
+-01:00	PT	20056	Atlantic/Azores
+-04:00	BM	15000	Atlantic/Bermuda
++00:00	ES	2054431	Atlantic/Canary
+-01:00	CV	252678	Atlantic/Cape_Verde
++00:00	FO	15000	Atlantic/Faroe
++00:00	PT	213794	Atlantic/Madeira
++00:00	IS	245356	Atlantic/Reykjavik
++00:00	SH	15000	Atlantic/St_Helena
+-03:00	FK	15000	Atlantic/Stanley
++09:30	AU	1706678	Australia/Adelaide
++10:00	AU	5807246	Australia/Brisbane
++09:30	AU	17456	Australia/Broken_Hill
++09:30	AU	199509	Australia/Darwin
++08:45	AU	15000	Australia/Eucla
++10:00	AU	392450	Australia/Hobart
++10:00	AU	15000	Australia/Lindeman
++10:30	AU	15000	Australia/Lord_Howe
++10:00	AU	9304158	Australia/Melbourne
++08:00	AU	3086441	Australia/Perth
++10:00	AU	9352313	Australia/Sydney
++01:00	NL	13892513	Europe/Amsterdam
++01:00	AD	36283	Europe/Andorra
++04:00	RU	644812	Europe/Astrakhan
++02:00	GR	6286690	Europe/Athens
++01:00	RS	5756549	Europe/Belgrade
++01:00	DE	62740682	Europe/Berlin
++01:00	SK	2548401	Europe/Bratislava
++01:00	BE	9038306	Europe/Brussels
++02:00	RO	11070040	Europe/Bucharest
++01:00	HU	9366333	Europe/Budapest
++01:00	DE	15000	Europe/Busingen
++02:00	MD	1460347	Europe/Chisinau
++01:00	DK	3885672	Europe/Copenhagen
++01:00	IE	2720758	Europe/Dublin
++01:00	GI	26544	Europe/Gibraltar
++00:00	GG	16488	Europe/Guernsey
++02:00	FI	5265184	Europe/Helsinki
++00:00	IM	26218	Europe/Isle_of_Man
++03:00	TR	77554557	Europe/Istanbul
++00:00	JE	28000	Europe/Jersey
++02:00	RU	664545	Europe/Kaliningrad
++03:00	RU	782604	Europe/Kirov
++02:00	UA	35633582	Europe/Kyiv
++00:00	PT	6372176	Europe/Lisbon
++01:00	SI	552127	Europe/Ljubljana
++00:00	GB	57817786	Europe/London
++01:00	LU	131322	Europe/Luxembourg
++01:00	ES	42918247	Europe/Madrid
++01:00	MT	141714	Europe/Malta
++02:00	AX	15000	Europe/Mariehamn
++03:00	BY	6105410	Europe/Minsk
++01:00	MC	48977	Europe/Monaco
++03:00	RU	69342062	Europe/Moscow
++01:00	NO	3296711	Europe/Oslo
++01:00	FR	33093827	Europe/Paris
++01:00	ME	398494	Europe/Podgorica
++01:00	CZ	5924565	Europe/Prague
++02:00	LV	1206422	Europe/Riga
++01:00	IT	34033666	Europe/Rome
++04:00	RU	3546813	Europe/Samara
++01:00	SM	15000	Europe/San_Marino
++01:00	BA	1905798	Europe/Sarajevo
++04:00	RU	1705529	Europe/Saratov
++03:00	UA	1620906	Europe/Simferopol
++01:00	MK	1754303	Europe/Skopje
++02:00	BG	4111057	Europe/Sofia
++01:00	SE	7149541	Europe/Stockholm
++02:00	EE	1106943	Europe/Tallinn
++01:00	AL	1629643	Europe/Tirane
++04:00	RU	814189	Europe/Ulyanovsk
++01:00	LI	15000	Europe/Vaduz
++01:00	VA	15000	Europe/Vatican
++01:00	AT	5274040	Europe/Vienna
++02:00	LT	2103133	Europe/Vilnius
++03:00	RU	1837053	Europe/Volgograd
++01:00	PL	22720110	Europe/Warsaw
++01:00	HR	1650334	Europe/Zagreb
++01:00	CH	3461664	Europe/Zurich
++03:00	MG	6115943	Indian/Antananarivo
++06:00	IO	15000	Indian/Chagos
++07:00	CX	15000	Indian/Christmas
++06:30	CC	15000	Indian/Cocos
++03:00	KM	118353	Indian/Comoro
++05:00	TF	15000	Indian/Kerguelen
++04:00	SC	22881	Indian/Mahe
++05:00	MV	103693	Indian/Maldives
++04:00	MU	701493	Indian/Mauritius
++03:00	YT	138967	Indian/Mayotte
++04:00	RE	804329	Indian/Reunion
++13:00	WS	40407	Pacific/Apia
++12:00	NZ	5203989	Pacific/Auckland
++11:00	PG	40266	Pacific/Bougainville
++12:45	NZ	15000	Pacific/Chatham
++10:00	FM	15000	Pacific/Chuuk
+-06:00	CL	15000	Pacific/Easter
++11:00	VU	35901	Pacific/Efate
++12:00	FJ	334403	Pacific/Fiji
++12:00	TV	15000	Pacific/Funafuti
+-06:00	EC	15000	Pacific/Galapagos
+-09:00	PF	15000	Pacific/Gambier
++11:00	SB	194707	Pacific/Guadalcanal
++10:00	GU	121094	Pacific/Guam
+-10:00	US	1569453	Pacific/Honolulu
++13:00	KI	15000	Pacific/Kanton
++14:00	KI	15000	Pacific/Kiritimati
++11:00	FM	15000	Pacific/Kosrae
++12:00	MH	15000	Pacific/Kwajalein
++12:00	MH	45701	Pacific/Majuro
+-09:30	PF	15000	Pacific/Marquesas
++12:00	NR	15000	Pacific/Nauru
+-11:00	NU	15000	Pacific/Niue
++11:00	NF	15000	Pacific/Norfolk
++11:00	NC	137086	Pacific/Noumea
+-11:00	AS	15000	Pacific/Pago_Pago
++09:00	PW	15000	Pacific/Palau
+-08:00	PN	15000	Pacific/Pitcairn
++11:00	FM	15000	Pacific/Pohnpei
++10:00	PG	588589	Pacific/Port_Moresby
+-10:00	CK	15000	Pacific/Rarotonga
++10:00	MP	48220	Pacific/Saipan
+-10:00	PF	81958	Pacific/Tahiti
++12:00	KI	40311	Pacific/Tarawa
++13:00	TO	22400	Pacific/Tongatapu
++12:00	WF	15000	Pacific/Wallis

data/misc/tld.json

@@ -0,0 +1 @@
+{ "format": "{tld}", "rows": "tld.tsv", "key": "tld", "name": "unicode" }

data/misc/useragent.json

@@ -1 +1 @@
-["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"]
+{ "format": "{ua}", "rows": "useragent.tsv" }

data/misc/useragent.tsv

@@ -0,0 +1,97 @@
+browser	device	os	ua
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Mobile Safari/537.36
+Chrome	mobile	Android	Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36
+Chrome	desktop	ChromeOS	Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Chrome	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 YaBrowser/26.8.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.34 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Chrome	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36
+Chrome	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 26_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/152.0.7977.64 Mobile/15E148 Safari/604.1
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Chrome	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36
+Edge	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36 Edg/151.0.0.0
+Edge	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Edg/152.0.0.0
+Edge	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36 Edg/153.0.0.0
+Edge	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Edg/152.0.0.0
+Firefox	desktop	Linux	Mozilla/5.0 (X11; Linux x86_64; rv:155.0) Gecko/20100101 Firefox/155.0
+Firefox	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0
+Firefox	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:154.0) Gecko/20100101 Firefox/154.0
+Firefox	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0
+Firefox	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:154.0) Gecko/20100101 Firefox/154.0
+Firefox	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:155.0) Gecko/20100101 Firefox/155.0
+Opera	desktop	Windows	Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36 OPR/135.0.0.0
+Opera	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 OPR/134.0.0.0
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7.5 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5.2 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.1 Mobile/15E148 Safari/604.1
+Safari	mobile	iOS	Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/27.0 Mobile/15E148 Safari/604.1
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5.2 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.1 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.2 Safari/605.1.15
+Safari	desktop	macOS	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/27.0 Safari/605.1.15
+Samsung Internet	mobile	Android	Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/30.0 Chrome/143.0.0.0 Mobile Safari/537.36

data/modules.go

@@ -0,0 +1,19 @@
+// Package data lists every module fejkdata ships.
+package data
+
+import (
+	"io/fs"
+
+	enus "github.com/larvit/fejkdata/data/en_US"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+	"github.com/larvit/fejkdata/data/misc"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
+
+// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS {
+	return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
+}

data/sv_SE/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
+		"sv_SE.color": {"paths":[""]},
+		"sv_SE.company": {"paths":["","base","suffix"]},
+		"sv_SE.date": {"paths":[""]},
+		"sv_SE.email": {"paths":["","domain","local","local.n"]},
+		"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"sv_SE.ip": {"paths":[""]},
+		"sv_SE.last-name": {"paths":["","count","name"]},
+		"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
+		"sv_SE.personnummer": {"paths":[""]},
+		"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
+		"sv_SE.price": {"paths":["","amt","ore"]},
+		"sv_SE.samordningsnummer": {"paths":[""]},
+		"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
+		"sv_SE.time": {"paths":[""]},
+		"sv_SE.title": {"paths":["","name","share"]},
+		"sv_SE.url": {"paths":["","host","path"]},
+		"sv_SE.username": {"paths":["","n"]},
+		"sv_SE.version": {"paths":["","n","pre","suffix"]},
+		"sv_SE.word": {"paths":[""]}
+	},
+	"namespace": "sv_SE"
+}

data/sv_SE/address.json

@@ -1,7 +1,7 @@
 {
-  "format": "{street} {street-number}\n{postal-code} {locality}",
+  "format": "{/geo.SE.address as a}{a}",
-  "street": "{/geo.SE.address.street}",
+  "street": "{a.street}",
-  "street-number": "{/geo.SE.address.street-number}",
+  "street-number": "{a.street-number}",
-  "postal-code": "{/geo.SE.address.postal-code}",
+  "postal-code": "{a.postal-code}",
-  "locality": "{/geo.SE.address.locality}"
+  "locality": "{a.locality}"
 }

data/sv_SE/fs.go

@@ -0,0 +1,19 @@
+// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
+// geo.SE, which package geose holds.
+package svse
+
+import (
+	"embed"
+	"io/fs"
+
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.SE.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }

data/sv_SE/fs_test.go

@@ -0,0 +1,22 @@
+package svse_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	svse "github.com/larvit/fejkdata/data/sv_SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/sv_SE/person.json

@@ -1,7 +1,7 @@
 {
   "format": "{prefix}{first} {last}",
-  "first": "{.first-name.name}",
+  "first": "{.first-name as n}{n.name}",
   "last": "{.last-name.name}",
   "prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
-  "sex": "{.sex.name}"
+  "sex": "{n..sex.name}"
 }

data_test.go

@@ -1,7 +1,10 @@
 package fejkdata
 
 import (
+	"io/fs"
+	"os"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
 			regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
 	}
 
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, c := range cases {
 		for i := 0; i < 200; i++ {
 			if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
 	}
 }
 
-// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper
+// TestShippedMiscCategories covers the locale-neutral misc module: a proper
 // v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
 // whose trailing {luhn()} check passes.
 func TestShippedMiscCategories(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
 	mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
 	objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
@@ -93,25 +96,46 @@ func TestShippedMiscCategories(t *testing.T) {
 // codes follow their standard shape, dotted sub-paths resolve, emoji is a real
 // glyph, and a coordinate parses to a point within geographic bounds.
 func TestShippedMiscReferenceData(t *testing.T) {
-	f := newGenerator(t, "data/misc", WithSeed(1))
+	f := newBareModule(t, "misc", WithSeed(1))
 	re := map[string]*regexp.Regexp{
 		"currency":              regexp.MustCompile(`^[A-Z]{3}$`),
 		"currency.name":         regexp.MustCompile(`\p{L}`),
 		"currency.symbol":       regexp.MustCompile(`^\S+$`),
-		"country":         regexp.MustCompile(`\p{L}`),
+		"territory":             regexp.MustCompile(`\p{L}`),
-		"country.alpha2":  regexp.MustCompile(`^[A-Z]{2}$`),
+		"territory.alpha2":      regexp.MustCompile(`^[A-Z]{2}$`),
-		"country.alpha3":  regexp.MustCompile(`^[A-Z]{3}$`),
+		"territory.alpha3":      regexp.MustCompile(`^[A-Z]{3}$`),
+		"territory.country":     regexp.MustCompile(`^[A-Z]{2}$`),
 		"language":              regexp.MustCompile(`\p{L}`),
 		"language.code":         regexp.MustCompile(`^[a-z]{2}$`),
 		"language.code3":        regexp.MustCompile(`^[a-z]{3}$`),
-		"timezone":        regexp.MustCompile(`^([A-Za-z_]+(/[A-Za-z_]+)+|UTC)$`),
+		"timezone":              regexp.MustCompile(`^[A-Za-z]+(/[A-Za-z0-9_+-]+)+$`),
+		"timezone.offset":       regexp.MustCompile(`^[+-](0\d|1[0-4]):[0-5]\d$`),
+		"timezone.territory":    regexp.MustCompile(`^[A-Z]{2}$`),
+		"timezone.weight":       regexp.MustCompile(`^[1-9]\d*$`),
 		"mimetype":              regexp.MustCompile(`^[a-z]+/[a-z0-9.+-]+$`),
 		"mimetype.ext":          regexp.MustCompile(`^\.[a-z0-9_-]+$`),
+		"httpmethod":            regexp.MustCompile(`^(CONNECT|DELETE|GET|HEAD|OPTIONS|PATCH|POST|PUT|TRACE)$`),
+		"httpmethod.safe":       regexp.MustCompile(`^(true|false)$`),
+		"httpmethod.idempotent": regexp.MustCompile(`^(true|false)$`),
+		"port":                  regexp.MustCompile(`^[1-9]\d{0,4}$`),
+		"port.service":          regexp.MustCompile(`^\S+$`),
+		"protocol":              regexp.MustCompile(`^\S+( \S+)*$`),
+		"protocol.name":         regexp.MustCompile(`\S`),
+		"protocol.number":       regexp.MustCompile(`^\d{1,3}$`),
+		"loglevel":              regexp.MustCompile(`^(alert|crit|debug|emerg|err|info|notice|warning)$`),
+		"loglevel.code":         regexp.MustCompile(`^[0-7]$`),
+		"loglevel.severity":     regexp.MustCompile(`^(Alert|Critical|Debug|Emergency|Error|Informational|Notice|Warning)$`),
+		"tld":                   regexp.MustCompile(`^\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$`),
+		"tld.type":              regexp.MustCompile(`^(country-code|generic|generic-restricted|infrastructure|sponsored)$`),
+		"tld.unicode":           regexp.MustCompile(`^\.\S+$`),
 		"httpstatus":            regexp.MustCompile(`^[1-5]\d{2} \S.*$`),
 		"httpstatus.code":       regexp.MustCompile(`^[1-5]\d{2}$`),
 		"useragent":             regexp.MustCompile(`^Mozilla/5\.0 .+`),
+		"useragent.browser":     regexp.MustCompile(`^(Chrome|Edge|Firefox|Opera|Safari|Samsung Internet)$`),
+		"useragent.device":      regexp.MustCompile(`^(desktop|mobile)$`),
+		"useragent.os":          regexp.MustCompile(`^(Android|ChromeOS|Linux|Windows|iOS|macOS)$`),
 		"car":                   regexp.MustCompile(`^\S.* \S`),
-		"car.maker":       regexp.MustCompile(`^\S`),
+		"car.make":              regexp.MustCompile(`^\S`),
 	}
 	for i := 0; i < 100; i++ {
 		for p, rx := range re {
@@ -135,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
 // TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
 // regexes miss: no generated name repeats a character three times over.
 func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
 		for i := 0; i < 20000; i++ {
 			name := fake(t, f, path)
@@ -154,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
 // male), the trailing digit is a Luhn checksum over the other nine, and a
 // samordningsnummer is the same with 60 added to the day.
 func TestSwedishPersonnummer(t *testing.T) {
-	sv := newGenerator(t, "data", WithSeed(1))
+	sv := shipped(t, WithSeed(1))
 	re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
 	sawLongMonthEnd, birth := false, map[string]bool{}
 	for i := 0; i < 2000; i++ {
@@ -187,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
 	if !birth["238"] || !birth["239"] {
 		t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
 	}
-	for i := 0; i < 200; i++ {
-		got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
-		sex, id, _ := strings.Cut(got, " ")
-		if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
-			t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
-		}
-	}
 }
 
 // TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
 // 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
 // in 50-65, 70-88, 90-92 or 94-99.
 func TestShippedUSTaxIds(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
 	itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
 	for i := 0; i < 2000; i++ {
@@ -224,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
 // TestShippedPersonNames pins the name tables: a sex pins its first names, a name
 // both sexes carry appears under both, and the record's sex agrees with its name.
 func TestShippedPersonNames(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(9))
+	f := shipped(t, WithSeed(9))
 	for path, want := range map[string]string{
 		"sv_SE.sex[f]":                        "kvinna",
 		"sv_SE.sex[man].code":                 "m",
@@ -252,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
 		seen := map[string]bool{}
 		for i := 0; i < 2000; i++ {
 			seen[fake(t, f, locale+".sex[f].first-name")] = true
-			first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex")
+			got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
-			if first == "" || sex == "" {
+			first, sex, _ := strings.Cut(got, "|")
-				t.Fatalf("%s.person lacks a first name or a sex", locale)
+			fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
-			}
 		}
 		if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
 			t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
 		}
-		got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`)
+	}
-		code, first, _ := strings.Cut(got, "|")
+}
-		if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
+
-			t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code)
+// TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
+// from one pick, and that two names bound to one person category are two people.
+func TestShippedPersonIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(9))
+	for _, locale := range []string{"sv_SE", "en_US"} {
+		apart := false
+		for i := 0; i < 300; i++ {
+			got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
+			parts := strings.Split(got, "|")
+			for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
+				fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
+			}
+			if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
+				fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
+			}
+			apart = apart || parts[0] != parts[3]
+		}
+		if !apart {
+			t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
 		}
 	}
 }
 
 func TestShippedSwedishPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
 	re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
 	for i := 0; i < 50; i++ {
 		if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
@@ -279,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
 }
 
 func TestShippedSwedishAddress(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(3))
+	f := shipped(t, WithSeed(3))
 	digit := regexp.MustCompile(`\d`)
 	for i := 0; i < 30; i++ {
 		a := fake(t, f, "sv_SE.address")
@@ -296,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
 	}
 }
 
+// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
+// locality and region are one place in the geo tables, read through a name in a template or
+// across a struct's tags, and that a reference beside the name is another address.
+func TestShippedAddressIsOneNamedPick(t *testing.T) {
+	f := shipped(t, WithSeed(3))
+	for _, c := range []struct{ locale, country, template, largest string }{
+		{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
+		{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
+	} {
+		places := geoPlaces(t, c.country)
+		apart, localities := false, map[string]int{}
+		for i := 0; i < 2000; i++ {
+			got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
+			parts := strings.Split(got, "|")
+			if !places[strings.Join(parts[:4], "|")] {
+				t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
+			}
+			apart = apart || parts[0] != parts[4]
+			localities[parts[2]]++
+		}
+		if !apart {
+			t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
+		}
+		if c.country == "SE" {
+			var row struct {
+				Street     string `fake:"{/sv_SE.address as a}{a.street}"`
+				PostalCode string `fake:"{a.postal-code}"`
+				Locality   string `fake:"{a.locality}"`
+			}
+			for i := 0; i < 200; i++ {
+				if err := f.FakeStruct(&row); err != nil {
+					t.Fatal(err)
+				}
+				if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
+					t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
+				}
+			}
+		}
+		for l, n := range localities {
+			if n > localities[c.largest] {
+				t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
+			}
+		}
+	}
+}
+
+// 0644000 is Los Angeles.
+func TestShippedPlacesSpreadAddresses(t *testing.T) {
+	for country, city := range map[string]struct {
+		key   string
+		least int
+	}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
+		streets := 0
+		for _, r := range geoRows(t, country, "street") {
+			if r[1] == city.key {
+				streets++
+			}
+		}
+		if streets < city.least {
+			t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
+		}
+	}
+	header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
+	population := slices.Index(strings.Split(header, "\t"), "population")
+	if population < 0 {
+		t.Fatal("geo.US locality.tsv has no population column")
+	}
+	for _, r := range geoRows(t, "US", "locality") {
+		n, err := strconv.Atoi(r[population])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n < 2500 {
+			return
+		}
+	}
+	t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
+}
+
+// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
+// draw by population leaves out.
+func TestShippedCapitalsAndBigPlaces(t *testing.T) {
+	state := map[string]string{}
+	for _, r := range geoRows(t, "US", "municipality") {
+		state[r[0]] = r[2]
+	}
+	shipped := map[string]bool{}
+	for _, r := range geoRows(t, "US", "locality") {
+		shipped[r[1]+", "+state[r[2]]] = true
+	}
+	for _, place := range []string{
+		"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
+		"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
+		"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
+		"Ventura, CA",
+	} {
+		if !shipped[place] {
+			t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
+		}
+	}
+}
+
+func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
+	refused := map[string]bool{
+		"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
+		"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
+		"Öresundsvägen, Bondevägen": true,
+	}
+	for _, r := range geoRows(t, "SE", "street") {
+		if refused[r[0]] {
+			t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
+		}
+	}
+}
+
+func geoTSV(t *testing.T, country, table string) string {
+	b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func geoRows(t *testing.T, country, table string) [][]string {
+	var out [][]string
+	for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
+		out = append(out, strings.Split(line, "\t"))
+	}
+	return out
+}
+
+// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
+// one locality holds, the region empty for SE, whose address carries none.
+func geoPlaces(t *testing.T, country string) map[string]bool {
+	rows := func(table string) [][]string { return geoRows(t, country, table) }
+	name, region := map[string]string{}, map[string]string{}
+	if country == "US" {
+		county := map[string]string{}
+		for _, r := range rows("municipality") {
+			county[r[0]] = r[2]
+		}
+		for _, r := range rows("locality") {
+			name[r[0]], region[r[0]] = r[1], county[r[2]]
+		}
+	} else {
+		for _, r := range rows("locality") {
+			name[r[0]] = r[0]
+		}
+	}
+	codes := map[string][]string{}
+	for _, r := range rows("postal-code") {
+		codes[r[1]] = append(codes[r[1]], r[0])
+	}
+	places := map[string]bool{}
+	for _, r := range rows("street") {
+		for _, code := range codes[r[1]] {
+			places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
+		}
+	}
+	return places
+}
+
 func TestShippedPersonHasParts(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(7))
+	f := shipped(t, WithSeed(7))
 	for _, path := range []string{"sv_SE.person", "en_US.person"} {
 		for i := 0; i < 30; i++ {
 			if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
@@ -307,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
 	}
 }
 
+// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
 func TestShippedUSPhone(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(11))
+	f := shipped(t, WithSeed(11))
-	re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`)
+	re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
-	for i := 0; i < 50; i++ {
+	unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
-		if n := fake(t, f, "en_US.phone"); !re.MatchString(n) {
+	for i := 0; i < 2000; i++ {
+		n := fake(t, f, "en_US.phone")
+		m := re.FindStringSubmatch(n)
+		if m == nil {
 			t.Fatalf("phone %q does not match %s", n, re)
 		}
+		if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
+			t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
+		}
 	}
 }
 
-// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and
+func TestShippedTableCellsAreTrimmed(t *testing.T) {
-// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person.
+	err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil || !strings.HasSuffix(path, ".tsv") {
+			return err
+		}
+		b, err := fs.ReadFile(os.DirFS("data"), path)
+		if err != nil {
+			return err
+		}
+		for i, line := range strings.Split(string(b), "\n") {
+			for _, cell := range strings.Split(line, "\t") {
+				if cell != strings.TrimSpace(cell) {
+					t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
+// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
 func TestShippedNamespacedTree(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(1))
+	f := shipped(t, WithSeed(1))
 	for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
 		if got := fake(t, f, path); got == "" {
 			t.Fatalf("Fake(%q) returned empty", path)
@@ -364,7 +590,7 @@ func luhnValid(s string) bool {
 var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
 
 func TestShippedStreetIsARegisteredName(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(5))
+	f := shipped(t, WithSeed(5))
 	street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
@@ -376,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
 func TestShippedLastNameComposition(t *testing.T) {
 	// last is a choice of patronymic {first}sson templates, compound
 	// {first}{last} templates and literal surnames.
-	f := newGenerator(t, "data", WithSeed(6))
+	f := shipped(t, WithSeed(6))
 	for i := 0; i < 300; i++ {
 		if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
 			t.Fatalf("last name %q is not a Swedish surname", s)
@@ -386,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
 
 func TestShippedStreetNumberFormats(t *testing.T) {
 	// Reachable via a hyphenated path; covers all five weighted number variants.
-	f := newGenerator(t, "data", WithSeed(8))
+	f := shipped(t, WithSeed(8))
 	re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
 	for i := 0; i < 300; i++ {
 		if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
@@ -398,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
 // TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
 // that the everyday titles are reachable.
 func TestShippedUSTitleAgreesWithSex(t *testing.T) {
-	f := newGenerator(t, "data", WithSeed(4))
+	f := shipped(t, WithSeed(4))
 	female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
 	seen := map[string]bool{}
 	for i := 0; i < 3000; i++ {
-		got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`)
+		got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
 		title, sex, _ := strings.Cut(got, "|")
 		title = strings.TrimSpace(title)
 		seen[title] = true

datatype.go

@@ -3,97 +3,83 @@ package fejkdata
 import (
 	"errors"
 	"fmt"
+
+	"github.com/larvit/fejkdata/internal/datatype"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+	"github.com/larvit/fejkdata/internal/jsonvalue"
 )
 
-// DataType is what a record column holds, which decides how a record writes its value.
+// DataType is what a record column holds, which decides how a record writes its value. It
-type DataType int
+// prints as data spells it: string, integer, number or boolean.
+type DataType = datatype.DataType
 
 // The datatypes a column declares with "datatype"; a column without one is a string.
 const (
-	DataTypeString DataType = iota
+	DataTypeString  DataType = datatype.String
-	DataTypeInteger
+	DataTypeInteger DataType = datatype.Integer
-	DataTypeNumber
+	DataTypeNumber  DataType = datatype.Number
-	DataTypeBoolean
+	DataTypeBoolean DataType = datatype.Boolean
 )
 
-var (
+// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
-	dataTypeNames = [...]string{"string", "integer", "number", "boolean"}
+func datatypeOf(m map[string]any, pos position) (*DataType, error) {
-	dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
-)
-
-// String is the datatype as data spells it.
-func (d DataType) String() string {
-	if d < 0 || int(d) >= len(dataTypeNames) {
-		return fmt.Sprintf("DataType(%d)", int(d))
-	}
-	return dataTypeNames[d]
-}
-
-// position is where a JSON value sits, which decides whether it may carry a datatype or
-// be null.
-type position int
-
-const (
-	inFormat position = iota // rendered by a format, so neither
-	atTop                    // a category or an inline template, whose fields may be columns
-	inColumn                 // a column, or a choice item standing in for one
-)
-
-// datatypeOf reads a template's "datatype" (default DataTypeString).
-func datatypeOf(m map[string]any, pos position) (DataType, error) {
 	v, ok := m["datatype"]
 	if !ok {
-		return DataTypeString, nil
+		return nil, nil
 	}
 	name, ok := v.(string)
 	if !ok {
-		return 0, fmt.Errorf("datatype must be a string, got %T", v)
+		return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
 	}
-	if name == DataTypeString.String() {
+	for d := DataTypeString; d < datatype.Count; d++ {
-		return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
-	}
-	for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
 		if name != d.String() {
 			continue
 		}
-		if pos != inColumn {
+		if !pos.column() {
-			return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here")
+			return nil, nil
 		}
-		return d, nil
+		return &d, nil
 	}
-	return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name)
+	return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
 }
 
 // checkColumns rejects a record column whose items hold different datatypes, checking a column
 // after the columns its items read, so a column read is named before its readers.
-func checkColumns(s nodeScope) error {
+func checkColumns(s nodeSet) error {
 	checked := map[node]bool{}
-	var check func(path, name string, column node) error
+	var check func(label, name string, column node) error
-	check = func(path, name string, column node) error {
+	check = func(label, name string, column node) error {
 		if checked[column] {
 			return nil
 		}
 		checked[column] = true
 		items, _ := columnItems(column)
 		for _, it := range items {
-			if r := it.readsColumn; r != nil {
+			if r := it.readsColumn; r != nil && r.category != "" {
-				if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil {
+				if err := check(r.category, r.field, r.column); err != nil {
 					return err
 				}
 			}
 		}
-		if _, err := columnDatatype(column); err != nil {
+		if len(items) == 0 {
-			return fmt.Errorf("%s: field %q: %w", path, name, err)
+			return nil
+		}
+		first := columnDatatype(column)
+		for _, it := range items[1:] {
+			if d := itemDatatype(it); d != first {
+				return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
+			}
 		}
 		return nil
 	}
-	return s(func(path string, n node) error {
+	return s(func(label string, n node) error {
 		t, ok := n.(*template)
-		if !ok || !t.record {
+		if !ok || !t.isRecord {
 			return nil
 		}
-		for _, name := range recordColumns(t) {
+		for _, name := range sortedNames(t.fields) {
-			if err := check(path, name, t.fields[name]); err != nil {
+			if err := check(label, name, t.fields[name]); err != nil {
 				return err
 			}
 		}
@@ -101,42 +87,26 @@ func checkColumns(s nodeScope) error {
 	})
 }
 
-// columnDatatype is the datatype a column's items hold. They must agree, since a
+// columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
-// column holds one; a column only ever null is a string.
+// a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
-func columnDatatype(n node) (DataType, error) {
+func columnDatatype(n node) DataType {
 	items, _ := columnItems(n)
 	if len(items) == 0 {
-		return DataTypeString, nil
+		return DataTypeString
 	}
-	first := itemDatatype(items[0])
+	return itemDatatype(items[0])
-	for _, t := range items[1:] {
-		if d := itemDatatype(t); d != first {
-			return first, disagreement(items[0], first, t, d)
-		}
-	}
-	return first, nil
 }
 
-// itemDatatype is the datatype a column item declares, else that of the column it is.
+// itemDatatype is the datatype a column item declares, else that of the column it reads.
 func itemDatatype(t *template) DataType {
-	if t.datatype != DataTypeString {
+	switch {
-		return t.datatype
+	case t.datatype != nil:
+		return *t.datatype
+	case t.readsColumn != nil:
+		return columnDatatype(t.readsColumn.column)
 	}
-	return readDatatype(t)
-}
-
-// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
-// when t reads none.
-func readDatatype(t *template) DataType {
-	if t.readsColumn == nil {
 	return DataTypeString
 }
-	items, _ := columnItems(t.readsColumn.column)
-	if len(items) == 0 {
-		return DataTypeString
-	}
-	return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
-}
 
 // columnItems is a column's template items, its choices unwrapped, and whether one is null.
 func columnItems(n node) (items []*template, nullable bool) {
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
 			}
 		case *template:
 			items = append(items, n)
-		case *null:
+		case *nullItem:
 			nullable = true
+		case *tableColumn:
+		default:
+			panic(invariant.Broken("columnItems has no case for node %T", n))
 		}
 	}
 	collect(n)
 	return items, nullable
 }
 
-// disagreement names the fix for two items of one column holding different datatypes.
+// itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
-func disagreement(a *template, da DataType, b *template, db DataType) error {
+type itemKind int
-	bare, typed, want := b, a, da
+
-	if da == DataTypeString {
+const (
-		bare, typed, want = a, b, db
+	kindText itemKind = iota
-	}
+	kindReads
+	kindDeclares
+)
+
+func kindOf(t *template) itemKind {
 	switch {
-	case da != DataTypeString && db != DataTypeString:
+	case t.datatype != nil:
-		return bothTyped(a, da, b, db)
+		return kindDeclares
-	case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "":
+	case itemDatatype(t) != DataTypeString:
-		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed))
+		return kindReads
-	case bare.fromString: // an object may carry a weight, which this spelling would drop
-		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
 	}
-	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)
+	return kindText
 }
 
-// bothTyped names the fix for two items holding different datatypes: the one taking its datatype
+// disagreement names the fix for items a and b of one column holding different datatypes, fixing
-// from the column it reads is typed as the other where its values prove it, else read as text.
+// the one that declares least, a when both declare alike.
-func bothTyped(a *template, da DataType, b *template, db DataType) error {
+func disagreement(a *template, da DataType, b *template, db DataType) error {
-	read, other := a, db
+	c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
-	if a.datatype != DataTypeString {
+	if kindOf(b) < kindOf(a) {
-		read, other = b, da
+		c.fix, c.other, c.want = b, a, da
 	}
-	switch {
+	message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
-	case read.datatype != DataTypeString:
+	if !ok {
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
+		panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
-	case (&valueProof{}).columnItem(read).not[other] == "":
-		return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
 	}
-	return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read))
+	return message(c)
+}
+
+// clash is two items of one column holding different datatypes: other holds want; held says
+// what each holds.
+type clash struct {
+	fix, other *template
+	want       DataType
+	held       string
+}
+
+// disagreements is a clash's message by the kinds of its items, fix's first.
+var disagreements = map[[2]itemKind]func(clash) error{
+	{kindDeclares, kindDeclares}: clash.bothDeclare,
+	{kindReads, kindReads}:       clash.retypeRead,
+	{kindReads, kindDeclares}:    clash.retypeRead,
+	{kindText, kindReads}:        clash.textBesideRead,
+	{kindText, kindDeclares}:     clash.declareText,
+}
+
+func (c clash) bothDeclare() error { return errors.New(c.held) }
+
+// fits reports whether fix may hold want.
+func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
+
+func (c clash) retypeRead() error {
+	if c.fits() {
+		return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
+	}
+	return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
+}
+
+func (c clash) textBesideRead() error {
+	if !c.fits() {
+		return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
+	}
+	return c.declareText()
+}
+
+func (c clash) declareText() error {
+	if c.fix.fromString { // an object may carry a weight, which this spelling would drop
+		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
+	}
+	return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
 }
 
 // typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
 	return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
 }
 
-// asText names the spelling that reads the column a column-read item reads as text, keeping the
+// columnRead is a record's column read by a format that only reads one reference or name,
-// other keys an object item carries.
+// which is the column: it takes the column's datatype and null. category and field name it;
-func asText(t *template) string {
+// category is "" for a column of the reading template's own record, which checkColumns reaches
-	if t.fromString {
+// on its own.
-		return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format)
+type columnRead struct {
+	a               arm
+	category, field string
+	column          node
 }
-	return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format)
+
+// columnReadOf is the record's column t's format reads, where the format only reads one reference
+// or name, and nil where it does not or t declares a string, which reads the column as text.
+func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
+	ops := t.compiled.ops
+	if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
+		return nil
+	}
+	a := ops[0].arms[0]
+	var start node
+	var category string
+	var tail []string
+	switch b := a.named; {
+	case a.kind == namedRead && b.bindsField():
+		// {f as n}{n}: column f of the record binding n, in t's own category.
+		start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
+	case a.kind == namedRead:
+		// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
+		start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
+		tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
+	case grammar.IsRef(a.head):
+		// {/c.x}: column x of category c.
+		start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
+	default:
+		return nil
+	}
+	if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
+		return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
+	}
+	return nil
 }

datatype_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 )
 
-func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
+func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
 	for _, src := range []string{
 		`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
 		`{"format":"","n":{"format":"42","datatype":"integer"}}`,
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 		`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
 		`{"format":"","pick":[[null,"a"],"b"]}`,
 	} {
-		if _, err := compile(parse(t, src)); err != nil {
+		if _, err := resolved(t, src); err != nil {
 			t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
 		}
 	}
 	for src, want := range map[string]string{
-		`{"format":"","n":{"format":"1","datatype":"int"}}`:                             `datatype takes "integer", "number" or "boolean", got "int"`,
+		`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
-		`{"format":"","n":{"format":"1","datatype":1}}`:                                 "datatype must be a string",
+		`{"format":"","n":{"format":"1","datatype":1}}`:     "datatype must be a string, not a number",
-		`{"format":"{int(1,9)}","datatype":"integer"}`:                                  "datatype only types a record column",
-		`[{"format":"1","datatype":"integer"},"x"]`:                                     "datatype only types a record column",
-		`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
-		`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`:           "datatype only types a record column",
-		`null`: `so write ""`,
-		`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
 		}
 	}
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
 func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 	tree := map[string]string{
 		"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
-		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
 	}
 	for _, c := range []struct{ name, column, want string }{
 		{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
 		{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
-		{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
 		{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
 		{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
-		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
-		{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
 		{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
-		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`},
+		{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
 		{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
-		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`},
+		{"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
+		{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
+		{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
+		{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
+		{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
 		{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
 		{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
 		{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
 		{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
 		{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
 		{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
-		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"},
+		{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
 		{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
 		{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
 		{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
-		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"},
+		{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
 		{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
 		{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 		{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
 		{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
 		{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
-		{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
-		{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
 	} {
 		row := `{"format":"","col":` + c.column + `}`
 		files := map[string]string{"row": row}
 		for name, body := range tree {
 			files[name] = body
 		}
-		_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
+		_, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
 		if err == nil || !strings.Contains(err.Error(), c.want) {
 			t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
 		}
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
 			t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
 		}
 	}
-	_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
+	for column, want := range map[string]string{
+		`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
+		`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`:   "its items hold integer and number; a column holds one datatype",
+	} {
+		row := `{"format":"","col":` + column + `}`
+		if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
+		}
+		if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
+			t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
+		}
+	}
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
 		"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
 		"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
 	})))
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
 	} {
 		row := `{"format":"","col":` + column + `}`
 		src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
-		f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
+		f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
 		if err != nil {
 			t.Errorf("%s: New = %v, want it loaded", column, err)
 			continue
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
 		t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
 	}
 }
+
+func TestDisagreementRefusesTwoTextItems(t *testing.T) {
+	mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
+}
+
+func TestASignedZeroIsZero(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
+	r, err := f.FakeRecord("row")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
+		t.Errorf("columns = %+v, want each written as its format", c)
+	}
+}
+
+func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
+	f := newGenerator(t, writeData(t, map[string]string{
+		"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
+		"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
+	}), WithSeed(1))
+	empty := 0
+	for i := 0; i < 50; i++ {
+		r, err := f.FakeRecord("row")
+		if err != nil {
+			t.Fatal(err)
+		}
+		same, text := r.Columns()[0], r.Columns()[1]
+		if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
+			t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
+		}
+		if text.Value == "" {
+			empty++
+		}
+	}
+	if empty == 0 {
+		t.Error(`50 records never read a null as "", want the string column to render one so`)
+	}
+}

doc.go

@@ -0,0 +1,46 @@
+// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
+//
+// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
+// or a folder. A generator loads the modules you name, such as data.Modules(), every
+// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
+// become a dot-path namespace, and you generate values by path:
+//
+//	f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
+//	f.Fake("sv_SE.address")          // "Trekungagatan 43\n442 42 Kungälv"
+//	f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
+//
+// Several sources merge in order, the last winning a name clash, so custom data
+// layers over the built-ins. The JSON template format is documented in the [README].
+//
+// [README]: https://github.com/larvit/fejkdata#readme
+package fejkdata
+
+// Vocabulary
+//
+//   - draw — one item taken from a choice, or one row from a table: `drawItem`,
+//     `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
+//   - expansion — one render of one format: `expand`.
+//   - render — one walk of the tree from an entry point, over one `frameStack`:
+//     `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
+//     start one each, and `FakeStruct` one per record.
+//   - env — what a render reads names and rows through: `renderEnv`.
+//   - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
+//   - named pick — the draw a token such as {/person as p} binds to a name, drawn
+//     on its first read and kept while its scope renders: `nameBinding`,
+//     `namedPick`, `pickFrame`.
+//   - level, pick key, addressed key — `pickKey`.
+//   - memo — what a named pick keeps its draws in, the variant drawn at each level
+//     and the value each path read: `drawMemo`.
+//   - pin — fixing which row of a table one path, or one named pick, uses:
+//     `pinSet`, `namedPick.pins`, `table.followStep`.
+//   - link — a table's tie to its parent table, each row to its parent row:
+//     `table.linkParent`.
+//   - family — a table and every table reaching it through a chain of parent
+//     links.
+//   - resolve — tying a reference or a name to what it names in the assembled tree:
+//     `resolveTemplates`, `resolveRefs`.
+//   - head — the string a read starts from: a field or a name as written, or a
+//     reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
+//     the read's start: `template.startOf`, `nameTarget.start`.
+//   - fence — a load-time check, run over each category, template or struct as it
+//     loads or compiles, so rendering a compiled tree cannot fail.

doc_test.go

@@ -0,0 +1,286 @@
+package fejkdata
+
+import (
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"os"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Every backtick in the Vocabulary is a symbol.
+var backticked = regexp.MustCompile("`([^`\n]+)`")
+
+// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
+func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
+	_, files := sourceFiles(t)
+	declared := declaredSymbols(files)
+	if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
+		t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
+	}
+}
+
+// namesDeclared is every symbol text backticks, failing for each one the package
+// declares nowhere.
+func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
+	t.Helper()
+	var named []string
+	for _, m := range backticked.FindAllStringSubmatch(text, -1) {
+		if !declared[m[1]] {
+			t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
+		}
+		named = append(named, m[1])
+	}
+	return named
+}
+
+// namesAPass fails unless one symbol text backticks is a function: a field names
+// what a pass reads or writes, never the pass.
+func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
+	t.Helper()
+	named := namesDeclared(t, where, text, declared)
+	if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
+		t.Errorf("%s names no function, so it names no pass", where)
+	}
+}
+
+func packageDir(t *testing.T) *build.Package {
+	t.Helper()
+	pkg, err := build.ImportDir(".", 0)
+	if err != nil {
+		t.Fatalf("read the package directory: %v", err)
+	}
+	return pkg
+}
+
+func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	return parseFiles(t, packageDir(t).GoFiles)
+}
+
+func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
+	t.Helper()
+	fset := token.NewFileSet()
+	var files []*ast.File
+	for _, name := range names {
+		f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
+		if err != nil {
+			t.Fatalf("parse %s: %v", name, err)
+		}
+		files = append(files, f)
+	}
+	return fset, files
+}
+
+func vocabulary(t *testing.T, files []*ast.File) string {
+	t.Helper()
+	var sections strings.Builder
+	for _, f := range files {
+		for _, c := range f.Comments {
+			section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
+			if found && c.Pos() > f.Package {
+				sections.WriteString(section)
+			}
+		}
+	}
+	if sections.Len() == 0 {
+		t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
+	}
+	return sections.String()
+}
+
+// declaredSymbols is every name the package declares at the top level, a method and
+// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
+func declaredSymbols(files []*ast.File) map[string]bool {
+	names := declaredFuncs(files)
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			if d, isGen := decl.(*ast.GenDecl); isGen {
+				addSpecs(names, d)
+			}
+		}
+	}
+	return names
+}
+
+// declaredFuncs is every function the package declares, a method keyed under its
+// type: template.fixedText.
+func declaredFuncs(files []*ast.File) map[string]bool {
+	names := map[string]bool{}
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isFunc := decl.(*ast.FuncDecl)
+			if !isFunc {
+				continue
+			}
+			names[d.Name.Name] = true
+			if recv := receiverType(d.Recv); recv != "" {
+				names[recv+"."+d.Name.Name] = true
+			}
+		}
+	}
+	return names
+}
+
+func addSpecs(names map[string]bool, d *ast.GenDecl) {
+	for _, spec := range d.Specs {
+		switch s := spec.(type) {
+		case *ast.TypeSpec:
+			names[s.Name.Name] = true
+			addFields(names, s)
+		case *ast.ValueSpec:
+			for _, id := range s.Names {
+				names[id.Name] = true
+			}
+		}
+	}
+}
+
+func addFields(names map[string]bool, s *ast.TypeSpec) {
+	st, isStruct := s.Type.(*ast.StructType)
+	if !isStruct {
+		return
+	}
+	for _, field := range st.Fields.List {
+		for _, id := range field.Names {
+			names[s.Name.Name+"."+id.Name] = true
+		}
+	}
+}
+
+func receiverType(recv *ast.FieldList) string {
+	if recv == nil || len(recv.List) == 0 {
+		return ""
+	}
+	e := recv.List[0].Type
+	if star, isPointer := e.(*ast.StarExpr); isPointer {
+		e = star.X
+	}
+	id, isIdent := e.(*ast.Ident)
+	if !isIdent {
+		return ""
+	}
+	return id.Name
+}
+
+const fillHeader = "Filled by "
+
+// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
+// fills, and to the template doc.
+func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
+	fset, files := sourceFiles(t)
+	declared, funcs := declaredSymbols(files), declaredFuncs(files)
+	doc, st, headers := templateStruct(t, fset, files)
+	namesAPass(t, "the template doc", doc, declared, funcs)
+	for _, field := range st.Fields.List {
+		if len(field.Names) == 0 {
+			t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
+		}
+		for _, id := range field.Names {
+			header := headerAbove(headers, id.Pos())
+			if header == "" {
+				t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
+				continue
+			}
+			namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
+		}
+	}
+}
+
+// templateStruct is the template struct, its doc, and the fill headers standing
+// between its fields, in source order.
+func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
+	t.Helper()
+	for _, f := range files {
+		for _, decl := range f.Decls {
+			d, isGen := decl.(*ast.GenDecl)
+			if !isGen {
+				continue
+			}
+			for _, spec := range d.Specs {
+				s, isType := spec.(*ast.TypeSpec)
+				if !isType || s.Name.Name != "template" {
+					continue
+				}
+				st, isStruct := s.Type.(*ast.StructType)
+				if !isStruct {
+					t.Fatal("template is not a struct")
+				}
+				return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
+			}
+		}
+	}
+	t.Fatal("the package declares no template struct")
+	return "", nil, nil
+}
+
+// fillHeaders skips a header trailing a field: it names that field, never the
+// fields below it.
+func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
+	trailing := map[int]bool{}
+	for _, field := range st.Fields.List {
+		trailing[fset.Position(field.End()).Line] = true
+	}
+	var found []*ast.CommentGroup
+	for _, c := range f.Comments {
+		if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
+			continue
+		}
+		if strings.HasPrefix(c.Text(), fillHeader) {
+			found = append(found, c)
+		}
+	}
+	return found
+}
+
+// headerAbove is the last fill header standing above pos.
+func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
+	text := ""
+	for _, c := range headers {
+		if c.Pos() < pos {
+			text = c.Text()
+		}
+	}
+	return text
+}
+
+var (
+	printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
+	printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
+)
+
+// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
+func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
+	for _, name := range []string{"doc.go", "README.md"} {
+		src, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines := printedCall.FindAllString(string(src), -1)
+		if lines == nil {
+			t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
+		}
+		f, err := New(withShipped(), WithSeed(42))
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, line := range lines {
+			c := printedFake.FindStringSubmatch(line)
+			if c == nil {
+				t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
+			}
+			want, err := strconv.Unquote(c[2])
+			if err != nil {
+				t.Fatalf("%s prints %s: %v", name, c[2], err)
+			}
+			if got := fake(t, f, c[1]); got != want {
+				t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
+			}
+		}
+	}
+}

docs/comprehension-history.md

@@ -0,0 +1,722 @@
+# Comprehension history
+
+## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
+- `datatype.go:217` `columnRead`: beside the code that reads it
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
+- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
+- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
+
+## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
+- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
+- `node.go:70` `template.readsColumn`: names the four readers that change together
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
+- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
+- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
+
+## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
+- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
+- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
+
+Inherited architect, decided by:
+
+- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
+- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
+- `path.go:119` `compileStep`: six parameters, the one place that reads harder
+
+## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
+- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
+- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
+- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
+- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
+
+## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
+- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
+- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
+
+Inherited architect, decided by:
+
+- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
+- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
+- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
+
+## 2026-10-07, scoring run at ad59967
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
+
+## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
+| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
+| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
+| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
+
+Junior A, hardest first:
+
+- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
+- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
+
+Mid A, hardest first:
+
+- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
+- `node.go:70` `readsColumn`: four functions in three files must change together
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
+- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
+
+Inherited architect, would restructure first:
+
+- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
+- `node.go:52` `template`: filled in four phases by four functions
+
+## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
+
+Ruling: better
+
+| Seat | Vote |
+|---|---|
+| Mid A | better |
+| Inherited architect | better |
+
+Mid A, decided by:
+
+- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
+- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
+- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
+
+Inherited architect, decided by:
+
+- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
+- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
+- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
+
+## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
+- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
+
+Inherited architect, decided by:
+
+- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
+- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
+- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
+- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
+- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
+
+## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | worse |
+| Inherited architect | same | worse | worse | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
+- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
+- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
+- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
+- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
+- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
+
+## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
+- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
+- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
+- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
+- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
+- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
+- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
+- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
+
+## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
+- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
+- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
+- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
+
+Inherited architect, decided by:
+
+- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
+- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
+- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
+
+## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | better | same | better | better |
+| Junior A | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
+- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
+- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
+- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
+- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
+
+Junior A, decided by:
+
+- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
+- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
+- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
+
+## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | worse | same | worse |
+| Inherited architect | same | worse | worse | same | worse |
+
+Mid A, decided by:
+
+- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
+- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
+- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
+- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
+- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
+- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
+- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
+- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
+- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
+- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
+
+## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | worse | worse | better | same | worse |
+| Maintainability senior | worse | same | worse | same | worse |
+| Junior A | same | worse | same | same | same |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
+- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
+- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
+- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
+
+Inherited architect, decided by:
+
+- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
+- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
+- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
+- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
+- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
+
+Maintainability senior, decided by:
+
+- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
+- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
+- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
+- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
+
+Junior A, decided by:
+
+- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
+- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
+
+## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | worse | same | worse |
+| Inherited architect | worse | same | worse | same | worse |
+
+Mid A, decided by:
+
+- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
+- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
+- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
+- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
+
+Inherited architect, decided by:
+
+- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
+- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
+- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
+- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
+- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
+
+## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | worse | worse | same | worse |
+| Inherited architect | worse | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
+- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
+- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
+
+Inherited architect, decided by:
+
+- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
+- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
+- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
+- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
+- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
+
+## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
+
+Ruling: worse
+
+Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | worse | better | same | same | same |
+| Inherited architect | worse | same | same | same | same |
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6 | 6 | 6 |
+| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
+| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
+
+Mid A, decided by:
+
+- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
+- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
+- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
+- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
+
+Inherited architect, decided by:
+
+- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
+- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
+- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
+
+## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | better | better | same | better | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
+- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
+
+Inherited architect, decided by:
+
+- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
+- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
+- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
+- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
+
+## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | worse | same | same |
+| Inherited architect | same | worse | worse | same | worse |
+| Maintainability senior | same | worse | same | same | worse |
+| Junior A | same | worse | worse | better | worse |
+
+Mid A, decided by:
+
+- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
+- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
+- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
+- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
+- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
+
+Inherited architect, decided by:
+
+- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
+- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
+- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
+- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
+
+Maintainability senior, decided by:
+
+- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
+- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
+
+Junior A, decided by:
+
+- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
+- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
+- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
+- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
+
+## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | same | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
+- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
+- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
+- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
+- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
+- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
+- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
+- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
+- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
+
+## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | same | better |
+| Inherited architect | same | better | better | same | better |
+
+Mid A, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
+- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
+- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
+- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
+
+Inherited architect, decided by:
+
+- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
+- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
+- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
+- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
+- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
+
+## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
+
+Ruling: better
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | better | better | better | better |
+| Inherited architect | same | better | same | same | better |
+
+Mid A, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
+- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
+- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
+- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
+- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
+- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
+- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
+
+Inherited architect, decided by:
+
+- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
+- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
+- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
+
+## 2026-10-12T16:30:06Z, PR #216 at 52e0304
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
+| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
+| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
+| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
+| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
+
+Junior A, hardest first:
+
+- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
+- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
+- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
+- `node.go:65` `template`: filled in four phases by different files, with a change-together set
+- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
+
+Mid A, hardest first:
+
+- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
+- `render.go:146` `readField`: four read paths, and the name undersells it
+- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
+- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
+- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
+
+Maintainability senior, hardest first:
+
+- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
+- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
+- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
+- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
+- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
+
+Inherited architect, would restructure first:
+
+- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
+- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
+- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
+- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
+- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
+
+## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | better | worse | same | same | better |
+| Inherited architect | better | worse | same | same | same |
+| Maintainability senior | better | worse | same | same | same |
+| Junior A | better | worse | same | better | better |
+
+Mid A, decided by:
+
+- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
+- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
+- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
+
+Inherited architect, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
+- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
+- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
+- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
+
+Maintainability senior, decided by:
+
+- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
+- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
+- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
+- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
+
+Junior A, decided by:
+
+- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
+- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
+- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
+- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
+- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
+- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
+
+## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
+
+Ruling: same
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | same | same | same | same |
+| Inherited architect | same | same | same | same | same |
+
+## 2026-10-14T11:34:34Z, PR #225 at f734c9e, against f90b4d6
+
+Ruling: worse
+
+| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
+|---|---|---|---|---|---|
+| Mid A | same | worse | same | same | same |
+| Inherited architect | same | worse | same | same | worse |
+| Maintainability senior | same | worse | same | worse | worse |
+| Junior A | same | worse | same | worse | worse |
+
+Mid A, decided by:
+
+- Locality: `Dockerfile:26` `vulnerability-filing`: the filer's test reads the real todo.md inside docker build ., so a plan edit can turn the Go gate red
+- Locality: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the filer splices todo.md as text, assuming its row shape, the Next ID marker and the Details heading
+- Locality: `.github/workflows/test.yml:106` `release`: a release now also waits on vuln.go.dev's answer, visible only in a needs list
+
+Inherited architect, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the plan is copied into the merge-gate build, so a planning edit that changes the table's shape fails docker build .
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: edits todo.md as text; a missing Next ID crashes with exit 1, which reads like a vulnerability already planned
+- Locality: `.github/workflows/test.yml:69` `vulnerabilities`: whether a release happens depends on vuln.go.dev that day, and every PR shows a red job while one is known
+
+Maintainability senior, decided by:
+
+- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: finds the release by an unnamed regex group, never checks the Next ID match, and splices Details by a replace that does nothing when the heading is missing
+- Locality, Self-sufficiency: `.github/workflows/test.yml:83` `File each one in todo.md`: one exit code holds the release and triggers the proposal, which must tell filed, planned and crashed apart by a git diff
+- Locality: `Dockerfile:26` `vulnerability-filing`: copies todo.md into the gate build, and the path filter lists todo.md
+- Self-sufficiency: `README.md:1213` "Development": says the build runs the release tooling's tests and leaves out the filer's
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:2` module docstring: builds on rank_todo.py, which the repository does not hold
+
+Junior A, decided by:
+
+- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the Go merge gate copies todo.md and files an item into it, so editing the plan can turn the build red
+- Locality: `.github/workflows/test.yml:106` `release`: whether a push to main releases also depends on vuln.go.dev's answer that day
+- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the regex and the Details splice encode todo.md's layout, which the script never shows
+- Self-sufficiency: `README.md:1213` "Development": the gate paragraph leaves out the filer's tests while AGENTS.md now defers to it

docs/reading-cost.md

@@ -0,0 +1,637 @@
+# Reading cost
+
+## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
+
+68314 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `compose.yaml` 3
+- `data-import/territory.py` 2
+- `data_test.go` 2
+- `shipped_data_test.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `data-import/tsv.py` 1
+- `data/en_US/phone.json` 1
+
+## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
+
+118930 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 8
+- `README.md` 6
+- `table_test.go` 4
+- `render.go` 3
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `fejkdata.go` 1
+
+## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
+
+93128 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `data_test.go` 4
+- `todo.md` 3
+- `format_stability_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data/en_US/email.json` 1
+- `data/en_US/first-name.tsv` 1
+- `data/en_US/sex.tsv` 1
+- `data/en_US/title.tsv` 1
+
+## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
+
+79628 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `todo.md` 5
+- `format.go` 4
+- `valueproof.go` 4
+- `calc_test.go` 3
+- `datatype_test.go` 2
+- `name.go` 2
+- `name_test.go` 2
+- `record_test.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
+
+366332 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `table_test.go` 17
+- `table.go` 9
+- `struct.go` 8
+- `README.md` 7
+- `docs/decisions.md` 7
+- `record.go` 5
+- `calc.go` 4
+- `data_test.go` 4
+- `reference.go` 4
+- `render.go` 4
+
+## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
+
+705270 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `docs/decisions.md` 16
+- `todo.md` 16
+- `README.md` 10
+- `render.go` 5
+- `table.go` 5
+- `format.go` 4
+- `AGENTS.md` 3
+- `calc.go` 3
+- `data.go` 3
+- `fejkdata.go` 3
+
+## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
+
+59846 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `CHANGELOG.md` 2
+- `README.md` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `fejkdata_test.go` 2
+- `go.mod` 2
+- `.dockerignore` 1
+- `builtins_test.go` 1
+- `cmd/fejkdata/main.go` 1
+
+## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
+
+98933 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `format_test.go` 5
+- `todo.md` 5
+- `calc.go` 4
+- `calc_test.go` 4
+- `format.go` 4
+- `path.go` 3
+- `reference.go` 3
+- `README.md` 2
+- `builtins_test.go` 2
+- `inline.go` 2
+
+## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
+
+129910 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `builtins_test.go` 7
+- `todo.md` 5
+- `format.go` 4
+- `calc_test.go` 2
+- `docs/decisions.md` 2
+- `fejkdata.go` 2
+- `reference.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `calc.go` 1
+
+## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
+
+91864 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `table_test.go` 4
+- `data.go` 3
+- `todo.md` 3
+- `graph.go` 2
+- `node.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+
+## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
+
+98023 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `path_test.go` 4
+- `README.md` 3
+- `docs/decisions.md` 3
+- `imports_test.go` 2
+- `path.go` 2
+- `reference_test.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
+
+115125 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `name_test.go` 5
+- `todo.md` 5
+- `shape_test.go` 3
+- `README.md` 2
+- `graph.go` 2
+- `inline.go` 2
+- `node.go` 2
+- `record_test.go` 2
+- `struct.go` 2
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
+
+53085 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `AGENTS.md` 3
+- `README.md` 3
+- `todo.md` 3
+- `.gitignore` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
+
+67696 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 3
+- `pick_test.go` 2
+- `render.go` 2
+- `table_test.go` 2
+- `CHANGELOG.md` 1
+- `README.md` 1
+- `data/geo/SE/address.json` 1
+- `data/sv_SE/address.json` 1
+- `docs/decisions.md` 1
+- `pick.go` 1
+
+## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
+
+75959 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 5
+- `README.md` 2
+- `inline.go` 2
+- `namefence.go` 2
+- `path.go` 2
+- `pick_test.go` 2
+- `record.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
+
+129293 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `README.md` 6
+- `format.go` 6
+- `docs/decisions.md` 5
+- `record.go` 5
+- `inline.go` 4
+- `render.go` 4
+- `shipped.go` 4
+- `CHANGELOG.md` 3
+- `data.go` 3
+
+## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
+
+79101 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 5
+- `format.go` 5
+- `todo.md` 5
+- `path.go` 3
+- `docs/decisions.md` 2
+- `env.go` 2
+- `pick.go` 2
+- `table.go` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
+
+2986167 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 24
+- `AGENTS.md` 18
+- `CHANGELOG.md` 18
+- `data-import/geo-se.py` 17
+- `record.go` 17
+- `cmd/fejkdata/main.go` 16
+- `struct.go` 16
+- `testdata/shipped_shape.txt` 16
+- `.github/workflows/test.yml` 15
+- `arm.go` 15
+
+## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
+
+96842 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 6
+- `CHANGELOG.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 2
+- `doc.go` 2
+- `docs/decisions.md` 2
+- `path.go` 2
+- `resolve.go` 2
+- `AGENTS.md` 1
+- `arm.go` 1
+
+## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
+
+152553 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 8
+- `data-import/geo-us.py` 5
+- `AGENTS.md` 4
+- `fejkdata.go` 4
+- `graph.go` 4
+- `docs/decisions.md` 3
+- `name.go` 3
+- `pick.go` 3
+- `resolve.go` 3
+
+## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
+
+2852445 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 23
+- `cmd/fejkdata/main.go` 23
+- `pick.go` 18
+- `AGENTS.md` 17
+- `CHANGELOG.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+- `arm.go` 15
+- `fejkdata.go` 15
+- `path.go` 15
+
+## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
+
+93957 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 9
+- `name.go` 6
+- `README.md` 5
+- `graph.go` 4
+- `env.go` 3
+- `path.go` 3
+- `resolve.go` 3
+- `arm.go` 2
+- `compose.yaml` 2
+- `docs/decisions.md` 2
+
+## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
+
+68500 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `render_test.go` 6
+- `todo.md` 3
+- `README.md` 2
+- `docs/decisions.md` 2
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+- `doc.go` 1
+- `doc_test.go` 1
+- `node.go` 1
+- `render.go` 1
+
+## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
+
+69756 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 3
+- `docs/decisions.md` 3
+- `CHANGELOG.md` 2
+- `AGENTS.md` 1
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/tsv.py` 1
+
+## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
+
+60588 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 2
+- `docs/decisions.md` 2
+- `record.go` 2
+- `render.go` 2
+- `CHANGELOG.md` 1
+- `docs/comprehension-history.md` 1
+- `inline.go` 1
+- `reference.go` 1
+- `shipped.go` 1
+
+## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
+
+2855972 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 25
+- `CHANGELOG.md` 22
+- `cmd/fejkdata/main.go` 20
+- `node.go` 19
+- `testdata/shipped_shape.txt` 19
+- `AGENTS.md` 18
+- `arm.go` 17
+- `DATA-LICENSES.md` 16
+- `compose.yaml` 16
+- `env.go` 16
+
+## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
+
+340530 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 15
+- `docs/decisions.md` 14
+- `README.md` 4
+- `.github/workflows/test.yml` 2
+- `compose.yaml` 2
+- `fejkdata.go` 2
+- `go.mod` 2
+- `release-tooling/publish_release.py` 2
+- `shipped.go` 2
+- `shippedindex.go` 2
+
+## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
+
+91452 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `todo.md` 5
+- `cmd/fejkdata/main_test.go` 3
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `imports_test.go` 2
+- `.dockerignore` 1
+- `.github/workflows/test.yml` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+
+## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
+
+112728 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 9
+- `todo.md` 6
+- `docs/decisions.md` 5
+- `shipped_test.go` 2
+- `CHANGELOG.md` 1
+- `compose.yaml` 1
+- `data.go` 1
+- `fejkdata.go` 1
+- `inline.go` 1
+- `internal/datafiles/datafiles.go` 1
+
+## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
+
+200600 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 13
+- `todo.md` 8
+- `CHANGELOG.md` 6
+- `cmd/fejkdata/main.go` 6
+- `docs/decisions.md` 6
+- `shipped_test.go` 6
+- `DATA-LICENSES.md` 5
+- `bench_test.go` 5
+- `data_test.go` 5
+- `data.go` 4
+
+## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
+
+87725 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 6
+- `README.md` 4
+- `data_test.go` 4
+- `CHANGELOG.md` 2
+- `docs/decisions.md` 2
+- `compose.yaml` 1
+- `data-import/geo-se.py` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+- `data-import/source.py` 1
+
+## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
+
+90566 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `data-import/geo-us.py` 6
+- `README.md` 4
+- `todo.md` 4
+- `data-import/geo-se.py` 3
+- `compose.yaml` 2
+- `data/geo/US/locality.tsv` 2
+- `data_test.go` 2
+- `.gitignore` 1
+- `AGENTS.md` 1
+- `CHANGELOG.md` 1
+
+## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
+
+76079 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 10
+- `docs/decisions.md` 6
+- `README.md` 4
+- `AGENTS.md` 2
+- `data-import/geo-se.py` 2
+- `CHANGELOG.md` 1
+- `data-import/geo-us.py` 1
+- `data-import/geo.py` 1
+
+## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
+
+102267 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 7
+- `CHANGELOG.md` 6
+- `todo.md` 6
+- `cli/cli_test.go` 4
+- `docs/decisions.md` 3
+- `cmd/fejkdata/main_test.go` 2
+- `AGENTS.md` 1
+- `cmd/fejkdata/main.go` 1
+- `compose.yaml` 1
+- `data/modules.go` 1
+
+## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
+
+372546 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 30
+- `todo.md` 10
+- `cli/cli.go` 5
+- `cli/cli_test.go` 3
+- `CHANGELOG.md` 2
+- `data/sv_SE/.fejkdata.json` 2
+- `docs/decisions.md` 2
+- `internal/datafiles/datafiles.go` 2
+- `record.go` 2
+- `compose.yaml` 1
+
+## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
+
+78327 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `todo.md` 8
+- `README.md` 5
+- `CHANGELOG.md` 1
+- `DATA-LICENSES.md` 1
+- `data/misc/car.tsv` 1
+- `docs/decisions.md` 1
+- `docs/research/research-sources-se.md` 1
+- `docs/research/research-sources-world.md` 1
+
+## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
+
+60817 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 4
+- `todo.md` 3
+- `CHANGELOG.md` 2
+- `shipped_data_test.go` 2
+- `.dockerignore` 1
+- `.gitignore` 1
+- `data_test.go` 1
+- `docs/decisions.md` 1
+- `release-tooling/publish_release.py` 1
+- `shipped_test.go` 1
+
+## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
+
+78241 new input tokens before the first commit.
+
+Read most (opens and searches):
+
+- `README.md` 10
+- `todo.md` 7
+- `CHANGELOG.md` 3
+- `docs/decisions.md` 3
+- `compose.yaml` 1
+- `data/en_US/itin.json` 1
+- `data/misc/timezone.json` 1
+- `data/misc/timezone.tsv` 1
+- `go.mod` 1

docs/research/part1-names-finance-company.md

@@ -0,0 +1,148 @@
+# Part 1 — Names, finance, company: sources and rules
+
+Researched 2026-09-17. "Embed" = ship as a data file in an MIT repo. Everything not marked UNVERIFIED was read from the cited page or file this session.
+
+## 1. US person names and identifiers
+
+### SSA baby names (given names by year and sex)
+- Page: https://www.ssa.gov/oact/babynames/limits.html — national zip https://www.ssa.gov/oact/babynames/names.zip (7 MB), state zip https://www.ssa.gov/oact/babynames/state/namesbystate.zip (23 MB), territories 228 kB.
+- Format (per SSA page + readme as mirrored by https://github.com/hackerb9/ssa-baby-names): one `yobYYYY.txt` per year 1880–2025, CSV `name,sex,count`, name 2–15 chars, sex `M`/`F`, sorted by sex then count desc. Names with < 5 occurrences in a year are excluded.
+- Counts: ~100,364 distinct names, ~2.02 M rows across all years (hackerb9 snapshot; year of snapshot UNVERIFIED). Top 1000 names cover 71.5 % of 2025 births (SSA page).
+- Licence: data.gov catalog entry https://catalog.data.gov/dataset/baby-names-from-social-security-card-applications-national-data lists licence **CC0 1.0** (`https://creativecommons.org/publicdomain/zero/1.0/`), publisher SSA. US federal work, no attribution required; embeddable.
+- Note: ssa.gov returns 403 to non-browser user agents (curl/WebFetch); download with a browser UA or via the data.gov link.
+
+### Census 2010 surnames
+- Page: https://www.census.gov/topics/population/genealogy/data/2010_surnames.html
+- Files: full set https://www2.census.gov/topics/genealogy/2010surnames/names.zip (CSV + XLSX, `Names_2010Census.csv`), top 1000 `Names_2010Census_Top1000.xlsx`, docs https://www2.census.gov/topics/genealogy/2010surnames/surnames.pdf.
+- Fields (from surnames.pdf): `name, rank, count, prop100k, cum_prop100k, pctwhite, pctblack, pctapi, pctaian, pct2prace, pcthispanic`. `(S)` = suppressed percentage. Names are UPPERCASE.
+- Count: 162,253 surnames occurring ≥ 100 times (covers 90.1 % of people with a recorded surname); the CSV also has a trailing `ALL OTHER NAMES` row (UNVERIFIED — www2.census.gov returned 403 to curl this session; download with a browser).
+- Licence: US Census Bureau work → US federal government work, public domain in the US (17 U.S.C. §105). Census only asks for citation (https://www.census.gov/about/policies/citation.html: "U.S. Census Bureau, [Table], [Product], [Vintage], [URL], accessed on [date]"). Embeddable; cite source in the data file header.
+
+### Name prefixes / suffixes
+- No authoritative enumerated list exists. USPS Publication 28 §33 (https://pe.usps.com/text/pub28/28c3_011.htm) defines the fields: Name Prefix, First Name, Middle Name or Initial, Surname, **Suffix Title** = "maturity (e.g., JR, SR) and professional (e.g., PHD, DDS) suffixes"; example "MR. WALTER W. WITHERSPOON JR.".
+- Recommendation: hand-curate. Prefixes `Mr., Mrs., Ms., Mx., Dr., Rev.`; suffixes `Jr., Sr., II, III, IV, PhD, MD, DDS, Esq.`. Rule of thumb: generational suffix (Jr/Sr/II–IV) only with a male name in traditional US usage; never combine Jr. with Sr.; at most one generational suffix.
+
+### SSN
+- Structure: `AAA-GG-SSSS` (area 3, group 2, serial 4). No check digit (Wikipedia).
+- Randomization since **2011-06-25** (https://www.ssa.gov/employer/randomization.html): "Previously unassigned area numbers were introduced for assignment excluding area numbers 000, 666 and 900-999." Area no longer geographic.
+- Invalid per SSA POMS RM 10201.035 (https://secure.ssa.gov/apps10/poms.nsf/lnx/0110201035): area `000`, `666`, `900–999`; group `00`; serial `0000`.
+- Generator rule: area ∈ [001,899] \ {666}; group ∈ [01,99]; serial ∈ [0001,9999].
+- Advertising SSNs (https://www.ssa.gov/history/ssn/misused.html): `078-05-1120` (Woolworth wallet card, 1938; > 40,000 people claimed it) and `219-09-9999` (1940 Social Security Board pamphlet). Both are structurally valid — exclude from output.
+- `987-65-4320`–`987-65-4329` "reserved for advertising": repeated by searchbug.com, ssn-check.org, a Missouri state standard (https://oa.mo.gov/sites/default/files/CC-SocialSecurityNumberNamingStandardV050305.pdf, which even misprints it as 987-54-4329). **UNVERIFIED** — not found on any ssa.gov page or POMS this session; Wikipedia no longer carries it. Safe to use these ten as the library's "obviously fake" examples, but do not cite SSA for it.
+
+### ITIN (IRS)
+- IRS Publication 4757 (Rev. 6-2026), p. 5 (https://www.irs.gov/pub/irs-pdf/p4757.pdf): "All valid ITINs are nine-digit numbers in the same format as the SSN (9XX-8X-XXXX), beginning with a "9" and the 4th and 5th digits ranging from 50 to 65, 70 to 88, 90 to 92, and 94 to 99."
+- Group `93` = ATIN (adoption TIN); `89` reserved (Intuit/TaxSlayer support pages, not IRS — UNVERIFIED).
+- Generator rule: `9` + 2 digits + group ∈ {50–65, 70–88, 90–92, 94–99} + 4 digits (serial `0000` not documented as invalid for ITIN; avoid it anyway).
+
+### EIN (IRS)
+- Format `NN-NNNNNNN`. Page: https://www.irs.gov/businesses/small-businesses-self-employed/how-eins-are-assigned-and-valid-ein-prefixes ("The first two digits of your EIN show the IRS campus that assigned it or if you applied online").
+- Valid prefixes (union of the table, 2026-09-17):
+  `01 02 03 04 05 06 10 11 12 13 14 15 16 20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 71 72 73 74 75 76 77 80 81 82 83 84 85 86 87 88 90 91 92 93 94 95 98 99`
+  By campus: Andover 10,12 · Atlanta 60,67 · Austin 50,53 · Brookhaven 01–06,11,13,14,16,21–23,25,34,51,52,54–59,65 · Cincinnati 30,32,35–38,61 · Fresno 15,24 · Kansas City 40,44 · Memphis 94,95 · Ogden 80,90 · Philadelphia 33,39,41–43,46,48,62–64,66,68,71–77,85–88,91–93,98,99 · Internet 20,26,27,33,39,41,42,45–47,81–88,92,93,99 · SBA 31.
+- Not valid: 00, 07, 08, 09, 17, 18, 19, 28, 29, 49, 69, 70, 78, 79, 89, 96, 97. No check digit.
+
+## 2. Finance
+
+### ABA routing number (RTN)
+- Wikipedia https://en.wikipedia.org/wiki/ABA_routing_transit_number: 9 digits; check: `(3(d1+d4+d7) + 7(d2+d5+d8) + (d3+d6+d9)) mod 10 = 0`.
+- First two digits: `00` US Government; `01–12` Federal Reserve districts (normal banks); `21–32` thrifts (historic, still valid); `61–72` non-bank payment processors/clearinghouses; `80` traveler's checks. Other prefixes unused (Wikipedia does not say so explicitly — inferred).
+- Generator rule: prefix ∈ {01–12, 21–32, 61–72}, 6 random digits, compute 9th as `(10 − (3d1+7d2+d3+3d4+7d5+d6+3d7+7d8) mod 10) mod 10`.
+- Open list of real RTNs: the Federal Reserve E-Payments Routing Directory (https://www.frbservices.org/resources/routing-number-directory/index.html) says "may not be sold, re-licensed, or otherwise used for commercial gain" → **not embeddable**. Generate synthetically.
+
+### Payment card numbers (Wikipedia "Payment card number", https://en.wikipedia.org/wiki/Payment_card_number; all Luhn-checked)
+| Network | IIN ranges | Length | CVV |
+|---|---|---|---|
+| Visa | 4 | 13, 16, 19 | 3 |
+| Mastercard | 51–55, 2221–2720 | 16 | 3 |
+| American Express | 34, 37 | 15 | 4 (front) |
+| Discover | 6011, 644–649, 65, 622126–622925 (UnionPay co-brand) | 16–19 | 3 |
+| JCB | 3528–3589 | 16–19 | 3 |
+| Diners Club International | 30 (covers 300–305, 3095), 36, 38, 39 | 14–19 | 3 |
+| Diners Club US & Canada | 55 (Mastercard co-brand) | 16 | 3 |
+| UnionPay | 62 | 16–19 | 3 |
+| Maestro | 5018, 5020, 5038, 5893, 6304, 6759, 6761, 6762, 6763 | 12–19 | 3 |
+| Maestro UK | 6759, 676770, 676774 | 12–19 | 3 |
+| Dankort 5019 · Mir 2200–2204 · RuPay 60,65,81,82,508 · Troy 65,9792 · UATP 1 (15) · Verve 506099–506198, 507865–507964, 650002–650027 (16/18/19) | | | |
+- CVV lengths: https://en.wikipedia.org/wiki/Card_security_code ("three-digit ... Visa, Mastercard, and Discover"; "American Express is a four-digit code on the front").
+- The requested "Maestro 50/56–58/6xxx" is a looser legacy rule; use Wikipedia's explicit IINs.
+- Wikipedia text is CC BY-SA 4.0 — copy the *facts* (not prose) into your table; facts are not copyrightable.
+
+### Published test cards
+- Stripe: https://docs.stripe.com/testing — Visa 4242424242424242, Mastercard 5555555555554444, Amex 378282246310005, Discover 6011111111111117, Diners 3056930009020004 (14), JCB 3566002020360505, UnionPay 6200000000000005; any future expiry, any CVC (4 for Amex).
+- PayPal: https://developer.paypal.com/tools/sandbox/card-testing/ — Visa 4005519200000004, 4012000033330026, 4012000077777777, 4012888888881881, 4217651111111119, 4500600000000061, 4772129056533503, 4915805038587737; Mastercard 2223000048400011; Amex 371449635398431, 376680816376961; Diners 36461510000039, 36461510000013; Maestro 6304000000000000, 5063516945005047; JCB 3636500000000260, 3636500000000989; CUP 6200680000000004, 6200680000000038.
+- Reuse: neither page carries an open licence (Stripe docs are under Stripe's site terms). The numbers themselves are Luhn-valid facts, widely reproduced; embedding the numbers with a "from Stripe/PayPal test docs" note is low-risk but **UNVERIFIED** as an explicit grant. Alternative: generate Luhn-valid numbers from the IIN table above — no dependency on either vendor.
+
+### BIC / SWIFT (ISO 9362)
+- https://en.wikipedia.org/wiki/ISO_9362: 4 letters bank code + 2 letters ISO 3166-1 country + 2 alphanumeric location + optional 3 alphanumeric branch (8 or 11 chars; `XXX` = primary office). Location second char `0` = test BIC, `1` = passive participant, `2` = reverse billing.
+- Generator rule: `[A-Z]{4}[A-Z]{2}[A-Z2-9][A-NP-Z0-9]([A-Z0-9]{3})?` — avoid `0`/`1` as the 8th char for a "live" BIC.
+- SWIFT's BIC Directory is proprietary (SwiftRef licence, redistribution needs a separate "SwiftRef Redistribution License": https://www.swift.com/myswift/ordering/order-products-services/swiftref-redistribution-license) → **not embeddable**.
+- Open alternative: GLEIF BIC-to-LEI mapping, https://www.gleif.org/en/lei-data/lei-mapping/download-bic-to-lei-relationship-files, files at https://mapping.gleif.org/api/v2/bic-lei/ (monthly zip; `LEI-BIC-20260828.zip` → `lei-bic-20260828T000000.csv`, columns `LEI,BIC`, 39,347 rows). Licence: "BIC/LEI Mapping Table License Agreement" (https://www.gleif.org/lei-data/lei-mapping/download-bic-to-lei-relationship-files/2017-12-21_annex-2_bic-to-lei-mapping-table-license-agreement_final.pdf) — a CC0-style grant "for any purpose whatsoever, including ... commercial" but **conditional on reproducing the notice** "SWIFT © and database rights [month year]. All rights reserved. This Mapping Table has been developed by SWIFT. Any use of the Mapping Table ... is subject to the BIC/LEI Mapping Table License Agreement ... For the latest BIC information and updates, always refer to www.swift.com/bic." Embeddable in an MIT repo with that notice in the data file; it is a list of real live BICs (bank names not included — join to GLEIF LEI data if names are wanted).
+
+### ISIN (https://en.wikipedia.org/wiki/International_Securities_Identification_Number)
+- 12 chars: 2-letter country + 9 alphanumeric NSIN + 1 check digit.
+- Check: convert letters A=10…Z=35 (ASCII − 55) to produce a digit string, then Luhn over that string (double every second digit from the right, sum digits, check makes total ≡ 0 mod 10). Pitfall: work on the *expanded* digit string; a transposed letter pair can pass.
+
+### CUSIP (https://en.wikipedia.org/wiki/CUSIP)
+- 9 chars: 6 issuer + 2 issue + 1 check. Values: digits as-is, A=10…Z=35, `*`=36, `@`=37, `#`=38.
+- Check: for i in 1..8, v = value; if i even, v ×= 2; sum += v div 10 + v mod 10; check = (10 − sum mod 10) mod 10.
+- Real CUSIPs are proprietary (CGS/FactSet); generate synthetic ones only.
+
+### IBAN (ISO 13616)
+- Validation (https://en.wikipedia.org/wiki/International_Bank_Account_Number): move first 4 chars to end, letters → 10–35, integer mod 97 must equal 1; max 34 chars; check digits 02–98.
+- SWIFT registry: https://www.swift.com/standards/data-standards/iban-international-bank-account-number, PDF release 101 https://www.swift.com/sites/default/files/files/iban-registry-v101.pdf (release 100 was Oct 2025); TXT also published (URL **UNVERIFIED** — swift.com returns 403/errors to automation this session). Terms: free of charge; no explicit licence text found (**UNVERIFIED**). Country count: 89 countries per Wikipedia (Dec 2024).
+- Embeddable per-country length tables already extracted from the registry:
+  - php-iban `registry.txt` (https://github.com/globalcitizen/php-iban, **LGPL-3.0**): pipe-separated, 121 rows (116 official + unofficial), columns `country_code|country_name|domestic_example|bban_example|bban_format_swift|bban_format_regex|bban_length|iban_example|iban_format_swift|iban_format_regex|iban_length|bban_bankid_start_offset|...|country_sepa|swift_official|...|currency_iso4217|central_bank_url|central_bank_name|membership`. LGPL data in an MIT repo is awkward; use it as a *reference* to build your own table (formats are facts) rather than copying the file.
+  - ibankit-js (https://github.com/koblas/ibankit-js, **Apache-2.0**, registry v95) — same caveat.
+  - Best route: own table `{country, length, bban_format}` derived from the SWIFT PDF (facts), cite SWIFT.
+
+### Currencies (ISO 4217)
+- ISO says use is free: https://www.iso.org/iso-4217-currency-codes.html — "ISO allows free-of-charge use of its country, currency and language codes from ISO 3166, ISO 4217 and ISO 639". Official list from SIX: https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml (+ `.xls`, `list-three` historic). Fields `CtryNm, CcyNm, Ccy, CcyNbr, CcyMnrUnts`; published 2026-01-01; 280 entity rows, 178 distinct codes. No symbols. No licence text inside the XML.
+- Alternatives:
+  - datasets/currency-codes (https://github.com/datasets/currency-codes): **PDDL** (public domain); `data/codes-all.csv` fields `Entity, Currency, AlphabeticCode, NumericCode, MinorUnit, WithdrawalDate`; built from SIX list one + three; no symbols. Embeddable.
+  - npm `currency-codes` v2.2.0 (https://github.com/freeall/currency-codes): **MIT**; fields `code, number, digits, currency, countries[]`; generated from the SIX XML; no symbols. Embeddable.
+  - umpirsky/currency-list (https://github.com/umpirsky/currency-list): **MIT**; code → localised name only (311 entries in `data/en_US/currency.json`, includes historic); no symbols, no numeric codes, no decimals.
+  - Debian iso-codes (https://salsa.debian.org/iso-codes-team/iso-codes): **LGPL-2.1+**; `data/iso_4217.json` has `alpha_3, name, numeric` only (179 entries); no symbols/minor units. LGPL → avoid embedding.
+  - Symbols: none of the above carry them. Unicode CLDR (`common/main/en.xml` currency symbols, Unicode License, MIT-compatible with notice) is the usual source — **UNVERIFIED this session**.
+
+### Crypto addresses
+- Bitcoin (https://en.bitcoin.it/wiki/List_of_address_prefixes): P2PKH version 0x00 → leading `1`; P2SH 0x05 → leading `3`; Base58Check 25–34 chars (alphabet excludes `0OIl`; last 4 bytes = first 4 of double-SHA256 of version+payload). Testnet `m`/`n`, `2`, `tb1`.
+- Bech32 (BIP-173, https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki, BSD-2-Clause): HRP `bc`, separator `1`, charset `qpzry9x8gf2tvdw0s3jn54khce6mua7l`, all-lowercase (or all-uppercase), max 90 chars; P2WPKH `bc1q…` = 42 chars, P2WSH `bc1q…` = 62 chars.
+- Bech32m (BIP-350, https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): witness v1+ (P2TR `bc1p…`, 62 chars), checksum constant `0x2bc830a3` instead of 1.
+- Ethereum EIP-55 (https://eips.ethereum.org/EIPS/eip-55, CC0): `0x` + 40 hex; keccak256 of the lowercase hex (no 0x, as ASCII); for each hex letter, uppercase iff the corresponding hash nibble ≥ 8 (bit 4·i set). Digits unchanged.
+
+## 3. Company
+
+### NAICS 2022 (US Census)
+- Page https://www.census.gov/naics/ ; files: 6-digit list https://www.census.gov/naics/2022NAICS/6-digit_2022_Codes.xlsx (82 kB; **1,012** six-digit codes, 111110…928120, verified by parsing), 2–6 digit https://www.census.gov/naics/2022NAICS/2-6%20digit_2022_Codes.xlsx, structure https://www.census.gov/naics/2022NAICS/2022_NAICS_Structure.xlsx, descriptions https://www.census.gov/naics/2022NAICS/2022_NAICS_Descriptions.xlsx, manual PDF https://www.census.gov/naics/reference_files_tools/2022_NAICS_Manual.pdf.
+- Format: xlsx, column A code (numeric), column B title (trailing spaces in some titles — trim).
+- Licence: no statement on the page or in the manual front matter; US federal government work → public domain in the US (17 U.S.C. §105). Embeddable; cite Census/OMB.
+
+### SEC EDGAR company names
+- `https://www.sec.gov/files/company_tickers.json`: object keyed `"0".."N"` of `{cik_str, ticker, title}`; **10,422** entries (2026-09-17). `company_tickers_exchange.json`: `{"fields":["cik","name","ticker","exchange"],"data":[...]}`.
+- `https://www.sec.gov/Archives/edgar/cik-lookup-data.txt`: 40 MB, **1,059,372** lines, `NAME:CIK:` (`COMPANY NAME:0001234567:`), all filers incl. individuals — filter to the ticker list for company names.
+- Access: declare `User-Agent: Company Name email@domain`, ≤ 10 req/s (https://www.sec.gov/os/webmaster-faq). Licence: "All Government-created content on sec.gov and EDGAR public filing content are free to access and reuse" (same FAQ). Embeddable; the ticker file (~800 kB) is the practical one.
+
+### DUNS
+- https://en.wikipedia.org/wiki/Data_Universal_Numbering_System: 9 digits, no significance; shown `NN-NNN-NNNN` or plain. Had a mod-10 check digit until ~Dec 2006; **dropped** since (expanded the pool by 800 M). So: any 9 digits are structurally valid today; no validation rule to enforce. Optional DUNS+4 suffix (4 alphanumerics) is user-assigned.
+
+### Legal entity suffixes per country
+- GLEIF ISO 20275 Entity Legal Forms code list v1.6 (2026-02-19): page https://www.gleif.org/en/lei-data/code-lists/iso-20275-entity-legal-forms-code-list ; CSV https://www.gleif.org/lei-data/code-lists/iso-20275-entity-legal-forms-code-list/2026-02-19-elf-code-list-v1.6.csv (758 kB; xlsx alongside).
+- Columns: `ELF Code, Country of formation, Country Code (ISO 3166-1), Jurisdiction of formation, Country sub-division code (ISO 3166-2), Entity Legal Form name Local name, Language, Language Code (ISO 639-1), Entity Legal Form name Transliterated name, Abbreviations Local language, Abbreviations transliterated, Date created, ELF Status ACTV/INAC, Modification, Modification date, Reason`.
+- Counts: 4,002 rows; 130 countries; 3,790 ACTV / 212 INAC; 1,518 rows have a local abbreviation (e.g. US `N.A.`, `FSA`; SE `AB`… multiple abbreviations separated by `;`). US has 737 rows (per-state forms). Abbreviations are the field you want for `Inc`, `LLC`, `GmbH`, `AB`, `SAS`, `Pty Ltd`; many rows have none, so filter on non-empty abbreviation and status ACTV.
+- Licence: GLEIF Open Data page (https://www.gleif.org/en/about/open-data): "The data on GLEIF's website is provided under a Creative Commons (CC0) license"; LEI Data Terms of Use: "provided under the CC0 licence, see CC0 1.0 Universal". The ELF page itself names no licence — that CC0 explicitly covers this code list is **UNVERIFIED**, but GLEIF treats all its published data this way. Embeddable.
+
+## Quick embed verdict
+| Item | Embed? | Licence / attribution |
+|---|---|---|
+| SSA baby names | yes | CC0 (data.gov) |
+| Census 2010 surnames | yes | US gov PD; cite Census |
+| EIN prefixes, ITIN/SSN rules | yes (rules) | facts from IRS/SSA |
+| NAICS 2022 | yes | US gov PD |
+| SEC company_tickers.json | yes | free to reuse (SEC) |
+| GLEIF ELF code list | yes | CC0 (GLEIF) |
+| GLEIF BIC-LEI (real BICs) | yes, with mandatory SWIFT notice | custom permissive licence |
+| ISO 4217 (SIX XML / datasets PDDL / npm MIT) | yes | free use (ISO), PDDL, MIT |
+| Fed routing directory, SwiftRef BIC directory, CUSIP lists | no | restricted/proprietary — generate synthetically |
+| php-iban / ibankit registries | copy facts only | LGPL-3.0 / Apache-2.0 |
+| Stripe / PayPal test cards | numbers only, note source | no explicit grant (UNVERIFIED) |

docs/research/part2-internet-vehicle-airline.md

@@ -0,0 +1,118 @@
+# Open data sources: Internet, Vehicle, Airline (researched 2026-09-17)
+
+Verdict key: **EMBED** = safe to vendor into an MIT repo; **EMBED+ATTR** = ok with a notice; **AVOID** = licence incompatible or unclear.
+
+## 4. Internet
+
+### IANA registries (all of them)
+- Licence: IANA/IETF statement at https://www.iana.org/help/licensing-terms — "IANA and IETF intend that the Protocol Registries may be freely used by any party for any purpose", data dedicated under **CC0 1.0**. No attribution required. **EMBED**.
+- All registries below share this licence; ship one `SOURCES`/NOTICE line pointing at that URL.
+
+| Registry | CSV URL | Rows (2026-09-16) | Notes |
+|---|---|---|---|
+| Media types | `https://www.iana.org/assignments/media-types/<type>.csv`, type ∈ application, audio, font, haptics, image, message, model, multipart, text, video (also `example`) | application 1800, audio 165, font 6, haptics 3, image 88, message 27, model 42, multipart 17, text 105, video 97 (header excluded) ≈ **2350** | Fields: `Name,Template,Reference`. Template is the full `type/subtype`; some rows are DEPRECATED/OBSOLETED in Name (filter). Format: RFC 6838 §4.2 — `type "/" subtype`, restricted chars, max 127 chars each, case-insensitive. |
+| HTTP status codes | https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv | 75 rows, **64 assigned** (306 and 418 are "(Unused)"; 104 is TEMPORARY) → 62 usable | Fields: `Value,Description,Reference`. Value is 3 digits 1xx–5xx. |
+| IPv4 special-purpose | https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry-1.csv | 26 blocks | Fields: `Address Block,Name,RFC,Allocation Date,Termination Date,Source,Destination,Forwardable,Globally Reachable,Reserved-by-Protocol`. |
+| IPv6 special-purpose | https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry-1.csv | 27 blocks | Same fields. Contains `2001:db8::/32` and `3fff::/20`. |
+| Service names / ports | https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv | 14,534 rows; 11,731 with both name and a single port; **6,102 distinct named ports**; 6,008 tcp-named; 707 tcp names < 1024 | Fields: `Service Name,Port Number,Transport Protocol,Description,Assignee,Contact,Registration Date,Modification Date,Reference,Service Code,Unauthorized Use Reported,Assignment Notes`. Port Number may be empty, single, or a range `a-b`. Well-known = 0–1023, registered 1024–49151, dynamic 49152–65535 (RFC 6335). ~1.1 MB; embed only the named-port subset. |
+| TLDs | https://data.iana.org/TLD/tlds-alpha-by-domain.txt | **1,401** TLDs (first line is a `# Version …` comment) | Upper-case ASCII; IDNs as `XN--…` punycode. Same IANA CC0 terms (UNVERIFIED that data.iana.org is explicitly covered by the licensing page; it is the same registry data). Lower-case on use. |
+
+### MIME extension mapping: jshttp/mime-db
+- https://github.com/jshttp/mime-db — **MIT** (GitHub API spdx MIT; npm `license: MIT`), version 1.54.0 on npm.
+- `db.json` (CDN: https://cdn.jsdelivr.net/npm/mime-db/db.json): object keyed by lower-case type → `{source, extensions[], compressible, charset}`.
+- Counts (1.54.0): **2,522 types**, **1,015 with extensions**; source = iana 2,136, apache 275, nginx 13, custom 98.
+- Sources: IANA registry (CC0), Apache httpd `mime.types` (Apache-2.0), nginx `mime.types` (BSD-2). **EMBED+ATTR** (keep the MIT copyright notice). Data updates are not semver-breaking per README.
+
+### IPv4 / IPv6 generation rules (RFCs, no data file needed)
+- RFC 5737 documentation: `192.0.2.0/24` (TEST-NET-1), `198.51.100.0/24` (TEST-NET-2), `203.0.113.0/24` (TEST-NET-3). https://www.rfc-editor.org/rfc/rfc5737.html
+- RFC 1918 private: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`. Also useful: `100.64.0.0/10` CGNAT (RFC 6598), `198.18.0.0/15` benchmarking (RFC 2544), `169.254.0.0/16` link-local.
+- IPv6 documentation: `2001:db8::/32` (RFC 3849) and `3fff::/20` (RFC 9637, "expands on the existing 2001:db8::/32 … with the reservation of an additional, larger prefix"). ULA `fc00::/7` (use `fd00::/8`), link-local `fe80::/10`.
+- Validation: generate inside those blocks; avoid network/broadcast (`.0`/`.255`) if the consumer expects host addresses. Format IPv6 per RFC 5952 (lower-case hex, `::` once, longest zero run).
+
+### MAC OUI (IEEE MA-L)
+- CSV: https://standards-oui.ieee.org/oui/oui.csv (3.8 MB, **40,161 rows**, fields `Registry,Assignment,Organization Name,Organization Address`; Assignment is 6 hex chars). Server returns HTTP 418 to non-browser user agents; fetch with a browser UA. Also MA-M `oui28.csv`, MA-S `oui36.csv`, CID `cid.csv`.
+- Licence: no licence on the IEEE pages themselves (site footer "© IEEE – All rights reserved" is generic). Debian's `ieee-data` copyright file (https://metadata.ftp-master.debian.org/changelogs/main/i/ieee-data/unstable_copyright) records IEEE's 2014 statement: *"IEEE does not assert any copyright in the OUI Public Listing or attempt to restrict distribution of the listing in any way. The IEEE Registration Authority does, however, strongly encourage those who use the list to obtain it directly from IEEE…"* — same text in the FSF directory. Debian ships it in `main` on that basis. **EMBED+ATTR** (cite the statement); the org-address column is not needed — keep `Assignment,Organization Name` only (~1 MB → few hundred KB). UNVERIFIED: no current IEEE page restates it (regauth FAQ and IPR pages do not mention it).
+- Alternative needing no data: locally administered unicast MAC — first octet's bit 1 (U/L) = 1, bit 0 (I/G) = 0 → second hex digit ∈ {2, 6, A, E}, e.g. `x2:xx:xx:xx:xx:xx`. Guaranteed never to collide with a vendor OUI. Notations: `01:23:45:67:89:ab`, `01-23-45-67-89-AB`, Cisco `0123.4567.89ab`.
+
+### User-agent strings
+| Source | Licence | Verdict |
+|---|---|---|
+| https://www.useragents.me/ | No licence stated anywhere on the page ("© 2022–2025 Useragents.me"); `/api` returns 404 now; JSON blobs embedded in page, weekly updates | **AVOID** (unlicensed = all rights reserved) |
+| https://explore.whatismybrowser.com/useragents/explore/ + API legal https://developers.whatismybrowser.com/api/about/legal/ | Proprietary: "You may NOT share the unique database download URL, or the downloaded file itself … EXCEPT for making it publicly available on the internet" | **AVOID** |
+| npm `user-agents` (intoli) https://github.com/intoli/user-agents | **BSD-2-Clause**, v2.1.185, daily-rebuilt dataset from Intoli traffic, weighted by frequency; fields `userAgent, platform, vendor, appName, deviceCategory, screenWidth/Height, viewportWidth/Height, connection{…}, oscpu, cpuClass, pluginsLength` | **EMBED+ATTR** (keep BSD notice). Count not stated on the page (UNVERIFIED; dataset is a few thousand rows). |
+| npm `top-user-agents` (microlinkhq) https://github.com/microlinkhq/top-user-agents | **MIT** (GitHub API). Top 100 UA strings from microlink.io traffic, weekly; JSON: https://cdn.jsdelivr.net/gh/microlinkhq/top-user-agents@master/src/index.json (+ desktop.json, mobile.json) | **EMBED+ATTR** — smallest clean option |
+- None of the above is CC0. Alternative: generate UAs from templates (Chrome/Firefox/Safari/Edge strings are fixed-form; only version numbers vary), no data needed.
+
+### Programming languages: GitHub Linguist
+- https://raw.githubusercontent.com/github-linguist/linguist/main/lib/linguist/languages.yml — **MIT** (repo LICENSE, "Copyright (c) 2017 GitHub, Inc."). **EMBED+ATTR**.
+- **835** top-level entries; **562** with `type: programming` (others: data, markup, prose). Per-entry fields: `type, color, extensions, aliases, tm_scope, ace_mode, language_id, group, interpreters, …`.
+
+### Operating systems
+- No open dataset found beyond awesome-lists. OS names/versions are facts — hand-author a small table (Windows 10/11, Windows Server, macOS versions+names, Ubuntu/Debian/Fedora/Arch/Alpine, iOS, Android, FreeBSD…). `user-agents` (BSD-2) carries `platform`/`oscpu` values if a data-driven list is wanted.
+
+### Database engines / collations
+- Engine names: hand-author (facts).
+- MySQL manual (https://dev.mysql.com/doc/refman/8.4/en/preface.html): Oracle terms — "you may not … copy, reproduce … distribute … any part" except distributing the docs with the software. **AVOID copying tables from MySQL docs.** Instead dump `SHOW COLLATION` from a `mysql:8.4.x` container (server output is factual, not the manual) — ~286 collations / 41 charsets in 8.x (UNVERIFIED count). PostgreSQL docs are under the PostgreSQL licence (permissive, keep notice) — `pg_collation` dump or docs both fine. MariaDB KB is CC BY-SA — avoid.
+
+## 5. Vehicle
+
+### NHTSA vPIC
+- API base `https://vpic.nhtsa.dot.gov/api/vehicles/`, `?format=json|xml|csv`. Endpoints verified: `GetAllMakes` (**12,363 makes**, fields `Make_ID,Make_Name`; 612 KB JSON, mostly small trailer/custom builders — filter with `GetMakesForVehicleType/car` → 195 makes, `/truck` → 207), `GetModelsForMake/honda` (362 models, `Make_ID,Make_Name,Model_ID,Model_Name`), `DecodeWMI/1HG`, `GetWMIsForManufacturer/honda` (45 WMIs, fields `WMI,Name,Country,VehicleType,Id,…`), `GetVehicleVariableList`, `GetVehicleVariableValuesList/<display name>` (must be the display name, URL-encoded, e.g. `Fuel%20Type%20-%20Primary`; the camel-case name returns 0 rows).
+- Value lists (counts): **Fuel Type - Primary 14**, **Body Class 71**, **Transmission Style 12**, **Drive Type 23**, **Vehicle Type 9**. Fields `Id,Name`.
+- Bulk: https://vpic.nhtsa.dot.gov/downloads/ — `vPICList_lite_2026_08.bak.zip` (MS SQL Server 2019+ backup, 190 MB, updated 2026-08-14), also `.plain.zip` / `.custom.zip`. Page says standalone DB "limited to VIN decoding"; makes/models/variables still via API. Contains `Wmi`, `Make`, `Model`, `WMIYearValidChars` tables (UNVERIFIED table names).
+- Licence: FAQ https://vpic.nhtsa.dot.gov/api/home/index/faq — "No, NHTSA is a government agency and the services provided on the API are free for use by the public as an offering as a part of our Open Data initiatives"; no rate-limit number stated, batch jobs asked to run at night. Work of the US federal government → public domain under 17 USC §105; data.gov entry lists licence as "unknown". **EMBED** (note source; no attribution legally required).
+
+### VIN (ISO 3779 / 49 CFR 565, North America)
+- 17 chars from `0-9 A-H J-N P R-Z` (**I, O, Q never**). Positions: 1–3 WMI, 4–8 VDS, 9 check digit, 10 model year, 11 plant, 12–17 serial (last 4 must be digits for US, positions 12–14 alphanumeric — 49 CFR 565.15).
+- Check digit: transliterate `A=1 B=2 C=3 D=4 E=5 F=6 G=7 H=8 J=1 K=2 L=3 M=4 N=5 P=7 R=9 S=2 T=3 U=4 V=5 W=6 X=7 Y=8 Z=9`, digits as-is; weights `8 7 6 5 4 3 2 10 0 9 8 7 6 5 4 3 2`; sum mod 11 → digit, remainder 10 → `X`.
+- Model year (pos 10): excludes `I O Q U Z 0`; `A=2010 … H=2017 J=2018 K=2019 L=2020 M=2021 N=2022 P=2023 R=2024 S=2025 T=2026 V=2027 W=2028 X=2029 Y=2030`, `1=2031…9=2039` (30-year cycle; `1–9` also = 2001–2009). Source: https://en.wikipedia.org/wiki/Vehicle_identification_number (rules are facts; do not copy prose).
+
+### WMI list
+- SAE J1044 / SAE WMI database is paid (https://www.sae.org/standards/j1044_202501-world-manufacturer-identifier) — **AVOID**.
+- Wikipedia "List of WMIs" — CC BY-SA — **AVOID**.
+- Use vPIC: `GetWMIsForManufacturer/<name or id>` per manufacturer, or the `Wmi` table in the standalone `.bak`. Public domain. Country prefix rule (first char): `1,4,5` USA, `2` Canada, `3` Mexico, `J` Japan, `K` Korea, `S` UK, `W` Germany, `Y` Sweden/Finland, `Z` Italy, `L` China (ISO 3780 facts).
+
+### US licence plate formats
+| Source | Licence | Verdict |
+|---|---|---|
+| openalpr `runtime_data/postprocess/us.patterns` (https://github.com/openalpr/openalpr) | **AGPL-3.0** | **AVOID** (291 lines, `@`=letter `#`=digit — useful only as a reference to check hand-authored patterns) |
+| Wikipedia "United States license plate designs and serial formats" | CC BY-SA | **AVOID copying**; formats themselves are facts |
+| jonnii/platekit | MIT, but SVG rendering, no serial patterns | not useful |
+- No MIT/CC0 per-state pattern list found. Recommendation: hand-author a per-state table (facts); current standard passenger formats as listed by Wikipedia (verify against DMV pages before use): AL `0AXXXXX`/`00AXXXX`, AK `ABC 123`, AZ `XXX 1XX`, AR `ABC 12D`, CA `1ABC123` (digit-3 letters-3 digits), CO `ABC-D12`, CT `AB·12345`, DE `123456`, DC `AB-1234`, FL `ABC D12`, GA `ABC1234`, HI `ABC 123`, ID `A 1234U`, IL `AB 12345`, IN `123ABC`, IA `ABC 123`, KS `1234ABC`, KY `ABC123`, LA `123 ABC`, ME `123·ABC`, MD `1AB2345`, MA `1ABC 23`, MI `ABC 1234`, MN `ABC-123`, MS `ABC 123`, MO `AB1 C2D`, MT `0-AB1234`, NE `ABC 123`, NV `123·A45`, NH `123 4567`, NJ `D12-ABC`, NM `123-ABC`, NY `ABC-1234`, NC `ABC-1234`, ND `123 ABC`, OH `ABC 1234`, OK `ABC-123`, OR `123 ABC`, PA `ABC1234`, RI `1AB 234`, SC `123ABC`, SD `0A1 234`, TN `ABC 1234`, TX `ABC-1234`, UT `A12 3BC`, VT `ABC 123`, VA `ABC-1234`, WA `ABC1234`, WV `X1A 2345`, WI `ABC-1234`, WY `1A-123A`. Most states omit I/O/Q from serials (state-specific, UNVERIFIED per state).
+
+## 6. Airline
+
+### OurAirports
+- https://ourairports.com/data/ — "All data is released to the Public Domain"; mirror repo https://github.com/davidmegginson/ourairports-data is **Unlicense** (GitHub API). Nightly updates. **EMBED**.
+- Raw URLs: `https://davidmegginson.github.io/ourairports-data/{airports,airport-frequencies,runways,navaids,countries,regions}.csv`.
+
+| File | Rows | Fields |
+|---|---|---|
+| airports.csv (12.7 MB) | **86,089**; by type: small_airport 42,734, heliport 23,216, closed 13,524, medium 4,106, seaplane_base 1,273, large 1,174, balloonport 62 | `id,ident,type,name,latitude_deg,longitude_deg,elevation_ft,continent,iso_country,iso_region,municipality,scheduled_service,icao_code,iata_code,gps_code,local_code,home_link,wikipedia_link,keywords` |
+| — with `iata_code` | **9,055** (large 1,171, medium 3,398, small 4,233, seaplane 153, heliport 100); `scheduled_service=yes` 4,335; US with IATA 2,036 (873 large/medium) | Suggested subset: `iata_code != '' AND type IN (large_airport, medium_airport)` → 4,569 rows |
+| countries.csv | **249** | `id,code,name,continent,wikipedia_link,keywords` |
+| regions.csv | **3,987** | `id,code,local_code,name,continent,iso_country,wikipedia_link,keywords` (code = ISO 3166-2, e.g. `US-KS`) |
+- No airline file. Derived: datasets/airport-codes on datahub (PDDL) — no need, use upstream.
+
+### OpenFlights
+- https://openflights.org/data.php — airports/airlines/routes/planes under **ODbL 1.0** (+ DbCL); airline/plane data partly from Wikipedia (GFDL/CC BY-SA). 5,888 airlines (2012). **AVOID** (share-alike). npm `airline-codes` (npow, ISC) is a straight OpenFlights sync → inherits ODbL → **AVOID**.
+
+### Airline IATA/ICAO codes — open options
+| Source | Licence | Count | Verdict |
+|---|---|---|---|
+| Wikidata SPARQL (https://query.wikidata.org/sparql) | **CC0** (https://www.wikidata.org/wiki/Wikidata:Licensing) | **2,928** items with IATA code (P229), **3,644** with ICAO code (P230) | **EMBED**. Query: `SELECT ?item ?itemLabel ?iata ?icao ?callsign WHERE { ?item wdt:P229 ?iata . OPTIONAL { ?item wdt:P230 ?icao } OPTIONAL { ?item wdt:P432 ?callsign } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }`. Includes defunct airlines — filter `MINUS { ?item wdt:P576 ?dissolved }` and/or P31 = Q46970 (airline). |
+| OpenTravelData `optd_airlines.csv` (https://github.com/opentraveldata/opentraveldata) | **CC BY 4.0** | 1,620 rows; `^`-separated; fields `pk^env_id^validity_from^validity_to^3char_code^2char_code^num_code^name^name2^alliance_code^alliance_status^type^wiki_link^flt_freq^alt_names^bases^key^version^parent_pk_list^successor_pk_list` | **EMBED+ATTR** (attribution notice required) |
+- Format rules: IATA designator = 2 alphanumeric chars (`[A-Z0-9]{2}`, at least one letter in practice; a third optional char exists in the standard but is unused); "controlled duplicates" share a code between non-overlapping regional carriers. ICAO designator = 3 letters `[A-Z]{3}`, unique. Accounting/prefix code = 3 digits (ticket number prefix, e.g. `016` United).
+
+### Aircraft types
+| Source | Licence | Verdict |
+|---|---|---|
+| ICAO Doc 8643 (https://www.icao.int/operational-safety/doc-8643-aircraft-type-designators) | ICAO copyright notice: "None of the materials … may be used, reproduced or transmitted … without permission in writing from ICAO"; API Data Service is paid (25 free trial calls) | **AVOID** — including GitHub mirrors such as ColtJD45/icao-aircraft-designator-list (MIT-labelled, 7,388 rows, fields `manufacturer,model,type_designator,description,engine_type,engine_count,wtc`) because the underlying data is scraped ICAO content (UNVERIFIED provenance) |
+| OpenSky aircraft database (https://opensky-network.org/data/aircraft; CSV https://s3.opensky-network.org/data-samples/metadata/aircraftDatabase.csv) | "The aircraft database is unlicensed and does not fall under our terms of use … offered as is"; built from registries, openflights.org and ICAO Doc 8643; citation requested; page says it is no longer up to date | **AVOID** ("unlicensed" ≠ open; upstream includes ODbL/ICAO). Fields for reference: `icao24,registration,manufacturericao,manufacturername,model,typecode,serialnumber,linenumber,icaoaircrafttype,operator,operatorcallsign,operatoricao,operatoriata,owner,…,engines,…,categoryDescription` |
+| Wikidata | **CC0** | **550** items with ICAO aircraft type designator (P8305); also IATA aircraft code P9040 (UNVERIFIED property id), manufacturer P176 | **EMBED**. Query: `SELECT ?item ?itemLabel ?icao ?mfrLabel WHERE { ?item wdt:P8305 ?icao . OPTIONAL { ?item wdt:P176 ?mfr } SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } }` |
+- Format: ICAO type designator 2–4 alphanumerics (`[A-Z0-9]{2,4}`, e.g. `B738`, `A20N`, `E190`); IATA aircraft code 3 alphanumerics (`738`, `32N`).
+
+### Airport / flight / seat format rules
+- IATA airport code `[A-Z]{3}`; ICAO location indicator `[A-Z0-9]{4}` (US `K…`, Canada `C…`, UK `EG…`, Sweden `ES…`). Use OurAirports rows so codes are real.
+- Flight number: 2-char IATA designator + 1–4 digits, no leading zeros in display (`QF9`, `AA1234`); systems cap at 0001–9999. Conventions (not rules): 1–999 mainline, 3000–5999 regional affiliates, ≥6000 codeshares, 8xxx charters, ≥9000 ferry/positioning. ICAO form: 3-letter designator + 1–4 alphanumerics (`AFR1`). Source: https://en.wikipedia.org/wiki/Flight_number.
+- Seat: `row + letter`. Rows typically 1–~60 (A380 up to ~90); many airlines skip 13 (and 14/17 on some carriers). Letters `A–K` **skipping I** (looks like 1/l); narrow-body 3-3 = `ABC DEF` (A/F windows, C/D aisles); wide-body 3-4-3 = `ABC DEFG HJK`, 2-4-2 = `AB DEFG JK`. Regex: `^[1-9][0-9]?[A-HJK]$`.

docs/research/part3-science-text-dates.md

@@ -0,0 +1,149 @@
+# Part 3 — Science/nature, Text, Dates: open data sources for an MIT fake-data library
+
+Researched 2026-09-17. "OK to embed" = redistributable inside an MIT repo with the stated notice. Anything not fetched first-hand is marked **UNVERIFIED**.
+
+## 7. Science / nature
+
+### Periodic table
+
+| Source | Licence | Embed in MIT repo | Fields | Rows | Format |
+|---|---|---|---|---|---|
+| PubChem periodic table — CSV https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/CSV, JSON https://pubchem.ncbi.nlm.nih.gov/rest/pug/periodictable/JSON (page: https://pubchem.ncbi.nlm.nih.gov/periodic-table/) | US-government work, public domain. NLM policy (https://www.ncbi.nlm.nih.gov/home/about/policies/): "Information that is created by or for the US government on this site is within the public domain … may be freely distributed and copied. However, it is requested that in any subsequent use of this work, NLM be given appropriate acknowledgment." | Yes; add an acknowledgment line ("Element data: PubChem/NLM"). | `AtomicNumber, Symbol, Name, AtomicMass, CPKHexColor, ElectronConfiguration, Electronegativity, AtomicRadius, IonizationEnergy, ElectronAffinity, OxidationStates, StandardState, MeltingPoint, BoilingPoint, Density, GroupBlock, YearDiscovered` (17) | 118 (Z=1..118, verified) | CSV; JSON is `{Table:{Columns:{Column:[...]},Row:[{Cell:[...]}]}}` |
+| NIST periodic table https://www.nist.gov/pml/periodic-table-elements | Public domain (17 USC 105); NIST asks for citation (https://www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications) | Yes, but | PDF only (no CSV/JSON) — not usable as a data source | 118 | PDF |
+| Bowserinator/Periodic-Table-JSON https://github.com/Bowserinator/Periodic-Table-JSON | **CC BY-SA 3.0** (LICENSE.md, verified; data derived from Wikipedia) | **No** — ShareAlike is incompatible with MIT redistribution | 33 keys incl. `name, symbol, number, atomic_mass, category, phase, period, group, block, shells, electron_configuration, summary, cpk-hex, …` | **119** (includes hypothetical Ununennium) | JSON, CSV |
+| faker-js `src/locales/en/science/chemical_element.ts` https://github.com/faker-js/faker (branch `next`) | MIT (LICENSE verified) | Yes, keep MIT notice | `{symbol, name, atomicNumber}` | 118 | TS |
+
+Validation rules: `AtomicNumber` 1..118 unique; `Symbol` 1–2 letters, first uppercase, unique; `Name` unique. PubChem leaves numeric fields empty for Z≥104 where unknown; `ElectronConfiguration` for Og is `"[Rn]7s2 7p6 5f14 6d10 (predicted)"` and `StandardState` is `"Expected to be a Gas"` — strip "(predicted)"/"Expected to be a" if you want an enum. `AtomicMass` for unstable elements is the mass number of the most stable isotope as a bare decimal (e.g. `295.216`), no brackets. `CPKHexColor` is 6 hex digits without `#`, empty for some.
+
+### SI units
+
+- Source: BIPM SI Brochure, 9th ed. (2019; current revision v4.01, 2026) https://www.bipm.org/en/publications/si-brochure ; PDF https://www.bipm.org/documents/20126/41483022/SI-Brochure-9-EN.pdf
+- Licence: **CC BY 4.0** — stated on the brochure page and on BIPM's copyright page (https://www.bipm.org/en/copyright): content may be adapted, copied, used in commercial products with "appropriate acknowledgement of the BIPM and its source"; BIPM name/logo must not imply endorsement. Verified via the HTML pages; the PDF's own front-matter licence line is **UNVERIFIED** (could not text-extract the PDF here).
+- Embed: Yes — the unit *names/symbols* are facts (not copyrightable); attribute "SI Brochure, BIPM, CC BY 4.0" anyway.
+- Data (cross-checked against https://en.wikipedia.org/wiki/International_System_of_Units):
+  - 7 base units: second s (time), metre m (length), kilogram kg (mass), ampere A (electric current), kelvin K (thermodynamic temperature), mole mol (amount of substance), candela cd (luminous intensity).
+  - 22 coherent derived units with special names: radian rad, steradian sr, hertz Hz, newton N, pascal Pa, joule J, watt W, coulomb C, volt V, farad F, ohm Ω, siemens S, weber Wb, tesla T, henry H, degree Celsius °C, lumen lm, lux lx, becquerel Bq, gray Gy, sievert Sv, katal kat.
+  - 24 prefixes: quetta Q 10^30, ronna R 10^27, yotta Y 10^24, zetta Z 10^21, exa E 10^18, peta P 10^15, tera T 10^12, giga G 10^9, mega M 10^6, kilo k 10^3, hecto h 10^2, deca da 10^1, deci d 10^-1, centi c 10^-2, milli m 10^-3, micro µ 10^-6, nano n 10^-9, pico p 10^-12, femto f 10^-15, atto a 10^-18, zepto z 10^-21, yocto y 10^-24, ronto r 10^-27, quecto q 10^-30.
+- Format rules for valid output: unit symbols are case-sensitive (`s` vs `S`, `m` vs `M`); prefix and unit symbol are joined without a space (`kJ`, `µs`); a space separates number and unit (`12 kg`), except the degree/minute/second of plane angle; `kg` already carries a prefix — never `µkg`; micro is Greek mu U+03BC (U+00B5 micro sign is the legacy compatibility character); ohm is U+03A9 (U+2126 OHM SIGN is deprecated); `°C` not `° C`.
+- MIT alternative: faker-js `en/science/unit.ts` — 29 `{name, symbol}` objects, MIT.
+
+### Animals by class
+
+- **Wikidata** (https://www.wikidata.org/wiki/Wikidata:Licensing): all structured data **CC0** — no attribution required. Embed: Yes.
+  - Feasibility, live SPARQL (https://query.wikidata.org/sparql), pattern `?t wdt:P31 wd:Q16521; wdt:P105 wd:Q7432; wdt:P171* wd:<class>; wdt:P1843 ?cn FILTER(LANG(?cn)='en')` (taxon, rank species, parent-taxon chain, English common name):
+    - Mammalia Q7377: **5,759** species with ≥1 English common name.
+    - Aves Q5113: **12,432**.
+    - Reptilia Q10811: **16,369** — suspiciously large; Wikidata's parent chain likely pulls birds/Sauropsida in. Use Squamata Q122422 + Testudines Q223044 + Crocodilia Q1387 (or check the chain) instead.
+    - Insecta Q1390 and Plantae Q756: **timed out (502/504) every attempt** — trees too big for the 60 s public endpoint. Options: paginate by order (`P171` one level at a time), use the weekly JSON dump, or take GBIF instead.
+  - Sample rows (mammals): `Notoryctes typhlops` → "Central Desert Marsupial Mole", "Itjaritjari", "Marsupial Mole", "Southern marsupial mole", "Southern Marsupial Mole" — i.e. **several P1843 values per taxon, inconsistent casing, and indigenous-language names tagged `en`**. Validation: pick one name per taxon (prefer the one matching the item's English `rdfs:label`), normalise case, drop names that equal the scientific name.
+- **GBIF Backbone Taxonomy** https://www.gbif.org/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c — licence **CC BY 4.0** (verified via https://api.gbif.org/v1/dataset/d7dddbf4-2cf0-4f39-9b2a-bb099caae36c, `license: http://creativecommons.org/licenses/by/4.0/legalcode`); required citation text: "GBIF Secretariat (2023). GBIF Backbone Taxonomy. Checklist dataset https://doi.org/10.15468/39omei". Embed: Yes with that citation. Vernacular names come via `/v1/species/{key}/vernacularNames`, each row from a *different* source checklist with its own licence (some CC BY-NC) — record and check per row. GBIF's site terms page (https://www.gbif.org/terms) was unreachable (timeout) — **UNVERIFIED** beyond the API field.
+- **faker-js** `src/locales/en/animal/*` (MIT; counts = quoted string lines on branch `next`): bird 821, snake 533, dog 493, cow 467, horse 342, rodent 161, insect 129, fish 95, cat 55, cetacean 52, rabbit 49, type 44 (generic: "bat, bear, bee, …, zebra"), pet_name 42, crocodilia 24, bear 8, lion 7. These are breeds/species mixed with common names, no class field; fine as MIT drop-ins. (A WebFetch summariser claimed 1,036 dogs; the regex count is 493 — treat the exact count as UNVERIFIED.)
+
+### Plants
+
+- Same Wikidata approach (CC0) — but the Plantae query timed out; paginate by order or family (`?t wdt:P171 ?family . ?family wdt:P105 wd:Q35409`) or use the dump.
+- GBIF Plantae (kingdomKey 6) vernacular names, CC BY 4.0, same per-row-licence caveat.
+- faker-js has **no** plant list (en locale dirs: airline, animal, app, book, color, commerce, company, database, date, finance, food, hacker, internet, location, lorem, medical, music, person, phone_number, science, team, vehicle, word).
+- USDA PLANTS database (public domain, has common names) — **UNVERIFIED**, not fetched.
+
+### Planets / moons
+
+- NASA NSSDCA Planetary Fact Sheet https://nssdc.gsfc.nasa.gov/planetary/factsheet/ — 10 columns (Mercury, Venus, Earth, Moon, Mars, Jupiter, Saturn, Uranus, Neptune, Pluto) × 20 rows (mass 10^24 kg, diameter km, density, gravity, escape velocity, rotation period, day length, distance from Sun, perihelion, aphelion, orbital period, orbital velocity, inclination, eccentricity, obliquity, mean temp °C, surface pressure, number of moons, rings Y/N, magnetic field Y/N). HTML table only. NASA content "generally not subject to copyright in the United States" (https://www.nasa.gov/nasa-brand-center/images-and-media/); NASA asks to be acknowledged; the NASA insignia must not be used. Embed: Yes with "Source: NASA NSSDCA".
+- Wikidata (CC0): planets = `?p wdt:P31/wdt:P279* wd:Q634; wdt:P397 wd:Q525` → Mercury Q308, Venus Q313, Earth Q2, Mars Q111, Jupiter Q319, Saturn Q193, Uranus Q324, Neptune Q332 (plus hypothetical Theia Q1053432 — filter out). Moons = `?m wdt:P397 <planet>; wdt:P31/wdt:P279* wd:Q2537` (natural satellite; note **Q2199 is "dwarf planet"**, not moon): Saturn 165, Jupiter 85, Uranus 29, Neptune 16, Earth 5, Mars 2. Caveats: Wikidata lags IAU counts (Saturn has 274 recognised moons as of 2025 — UNVERIFIED figure); Earth's 5 includes quasi-satellites; provisional designations (`S/2004 S 3`) are labels too — filter `^S/\d{4}` if you want proper names only.
+
+### Blood type distribution
+
+- **US** (Stanford Blood Center https://stanfordbloodcenter.org/donate-blood/blood-donation-facts/blood-types/, verified; cites AABB Technical Manual 18th ed.): O+ 37.4 %, A+ 35.7 %, B+ 8.5 %, AB+ 3.4 %, O− 6.6 %, A− 6.3 %, B− 1.5 %, AB− 0.6 % (sums to 100.0). Facts — freely embeddable; cite Stanford/AABB.
+- American Red Cross https://www.redcrossblood.org/donate-blood/blood-types.html — **UNVERIFIED**: site returns HTTP 403 (Akamai) to both WebFetch and the headless browser. Search snippets say the Red Cross gives type O by ethnicity (Latino 57 %, African American 51 %, Caucasian 45 %) rather than an 8-way national table.
+- **Global**: no authoritative figure found. Wikipedia "Blood type distribution by country" gives a population-weighted row (O+ 38.4, A+ 27.3, B+ 8.1, AB+ 2.0, O− 13.1, A− 8.1, B− 2.0, AB− 0.01) that its own editors flag "unreliable source"; WorldAtlas gives O+ 42, A+ 31, B+ 15, AB+ 5, O− 3, A− 2.5, B− 1, AB− 0.5 with no source. Recommend shipping only the US table, or per-country tables from Wikipedia (facts; Wikipedia prose is CC BY-SA but the numbers are cited to national blood services — cite those).
+- Validation: ABO ∈ {A, B, AB, O}; Rh ∈ {+, −}; weights must sum to 100 ± rounding.
+
+## 8. Text
+
+### English word lists with part of speech
+
+| Source | Licence | Embed | Counts | Format |
+|---|---|---|---|---|
+| Princeton WordNet 3.0 / 3.1 https://wordnet.princeton.edu/license-and-commercial-use ; download https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz | "WordNet License" (SPDX `WordNet`), BSD-style: permission to use/copy/modify/distribute "for any purpose and without fee or royalty … provided that … the following copyright notice and statements, including the disclaimer … appear on ALL copies … including modifications". Notice text: "WordNet 3.0 Copyright 2006 by Princeton University. All rights reserved." + AS-IS disclaimer; no use of Princeton's name in advertising. | Yes — ship the LICENSE text alongside the data | WordNet 3.0 (wnstats(7WN), verified): unique strings noun 117,798 / verb 11,529 / adj 21,479 / adv 4,481 (total 155,287); synsets 82,115 / 13,767 / 18,156 / 3,621 | WNDB: `index.noun|verb|adj|adv` (one lemma per line: `lemma pos synset_cnt p_cnt [ptr…] sense_cnt tagsense_cnt synset_offset…`), `data.<pos>` (synsets with glosses). Multi-word lemmas use `_` for spaces; lower-case; entries can contain digits/apostrophes/hyphens. |
+| Open English WordNet 2025 https://en-word.net/ ; https://github.com/globalwordnet/english-wordnet | **CC BY 4.0**; cite "Open English WordNet … derived from Princeton WordNet" | Yes, with attribution | 135,969 words, 107,519 synsets (README, verified) | `english-wordnet-2025.zip` (WNDB, 9.2 MB), `english-wordnet-2025.xml.gz` (WN-LMF, 10.8 MB), `english-wordnet-2025-json.zip` (9.5 MB), `.ttl.gz` (16.9 MB) |
+| SCOWL v2 / ESDB https://github.com/en-wl/wordlist (branch v2) ; classic http://wordlist.aspell.net/ | Custom MIT-like (Copyright 2000-2026 Kevin Atkinson: use/copy/modify/distribute/sell "provided that the above copyright notice appears in all copies and that both the above copyright notice and this notice appear in supporting documentation"). Sources 12dicts + ENABLE2K are public domain; lists above size 80 add the UKACD notice; **POS assignments partly from WordNet, so the WordNet notice "MIGHT apply"** (their words). Australian-English parts carry an extra notice. | Yes, ship the Copyright file | Sizes 35 (small), 50 (medium), 60 (spell-check default), 70 (large), 80 (incl. game words), 85 (archaic). No per-size counts published in the v2 README. Classic SCOWL sizes 10–95 — counts **UNVERIFIED** (README not reachable). | v2: `scowl.db` (SQLite), `scowl.txt`, Python `libscowl`. ESDB rows carry POS, spelling (A/B/Z/C/D) and region codes. |
+| 12dicts http://wordlist.aspell.net/12dicts-readme/ | Public domain ("I explicitly release them to the public domain, but request acknowledgment"), **except** `2of12inf` and the `2+2+3*` lists, which depend on AGID and inherit its terms | Yes (PD lists); acknowledge Alan Beale | 3esl ≈22k, 6of12 ≈32k, 2of12 ≈41k, 2of12inf ≈82k, 3of6game ≈65k, 5d+2a ≈68k, 3of6all ≈83k, 2+2+3lem ≈84k, 2of5core ≈4.7k, 6phrase ≈22k, neol2016 ≈600 | Plain text, one word per line; marker suffixes `+ ! ^ &` (not POS). **No POS tags** in any current 12dicts list. |
+| Moby Part-of-Speech II https://www.gutenberg.org/ebooks/3203 ; file https://www.gutenberg.org/files/3203/files/mobypos.txt | Public domain ("Public Domain material by grant from the author, January, 2001", in the package README) | Yes, no notice needed | **233,356** lines (verified) | One entry per line: `word\CODES`, CRLF line ends (e.g. `A-line\NA`, `a tempo\Avh`). Codes in priority order: `N` noun, `p` plural, `h` noun phrase, `V` verb (usu. participle), `t` transitive verb, `i` intransitive verb, `A` adjective, `v` adverb, `C` conjunction, `P` preposition, `!` interjection, `r` pronoun, `D` definite article, `I` indefinite article, `o` nominative. Many entries are phrases, proper nouns, or 1990s-era; non-ASCII entries use a legacy 8-bit encoding (e.g. `a bon march\v` lost its `é`) — restrict to ASCII `[a-z]+` for a clean list. |
+| faker-js `en/word/*` (MIT) | MIT | Yes | adjective 1000, noun 1000, verb 1000, adverb 325, preposition 109, conjunction 51, interjection 46 | TS string arrays |
+
+### Word frequency lists
+
+| Source | Licence | Verdict |
+|---|---|---|
+| Google Books Ngram data v2/v3 https://storage.googleapis.com/books/ngrams/books/datasetsv3.html | **CC BY 3.0** ("This compilation is licensed under a Creative Commons Attribution 3.0 Unported License") | Embeddable with attribution. Format `ngram TAB year TAB match_count TAB volume_count`; 1-gram files split by leading letter, GBs each — aggregate offline to a top-N list. |
+| wordfreq https://github.com/rspeer/wordfreq | Code **Apache 2.0** (not MIT). Data: "may be redistributed under a Creative Commons Attribution-ShareAlike 4.0 license" (mix of Google Books, Wikipedia, OpenSubtitles, SUBTLEX, Leeds, ParaCrawl, Twitter); maintainer notes the CSV export "does not follow the CC-By-SA license". Project in sunset mode (data snapshot ≈2021). | **Avoid** embedding the data (ShareAlike). |
+| Peter Norvig count_1w.txt https://norvig.com/ngrams/ | Page says "Code … under the MIT license"; the **data** comes from the Google Web 1T corpus distributed by LDC under LDC terms — no data licence stated. | **UNVERIFIED / avoid**. 333,333 words, `word TAB count`, lowercase. |
+| hermitdave/FrequencyWords https://github.com/hermitdave/FrequencyWords | "MIT License for code. CC-by-sa-4.0 for content." | **Avoid** (ShareAlike). |
+
+### Lorem ipsum
+
+- Canonical paragraph (Letraset, 1966; public domain — garbled Cicero): "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+- Source: Cicero, *De finibus bonorum et malorum* 1.10.32–33 (45 BC), public domain; the "dolorem ipsum" fragment begins "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit…". Reference: https://en.wikipedia.org/wiki/Lorem_ipsum
+- MIT word pool alternative: faker-js `en/lorem/word.ts` — 999 words.
+
+### Quote collections
+
+- Project Gutenberg #27889 https://www.gutenberg.org/ebooks/27889 — Bartlett, *Familiar Quotations*, **9th edition** (title page "NINTH EDITION", copyright 1875/1882/1891/1903) — not the 1919 10th. "Public domain in the USA." Plain text 3.1 MB, HTML zip 50 MB, EPUB.
+- Project Gutenberg #16732 https://www.gutenberg.org/ebooks/16732 — an early (Hurst & Co. reprint) edition, no edition number given. PD in USA.
+- 10th edition (1914/1919, ed. Nathan Haskell Dole) exists on archive.org (https://archive.org/details/bartlettsfamilia0000john_o4b4) — not found on Gutenberg; **UNVERIFIED** availability as clean text.
+- Either Bartlett text needs parsing (quote / author / source headings); no structured file. Wikiquote is CC BY-SA — avoid.
+- Validation: strip Gutenberg header/footer (`*** START OF THE PROJECT GUTENBERG EBOOK` … `*** END …`) before extraction; Gutenberg's trademark terms only bind if you keep the "Project Gutenberg" name on the output — strip it.
+
+### Colour names
+
+| Source | Licence | Embed | Count / format |
+|---|---|---|---|
+| CSS Color Module Level 4 §named colors https://www.w3.org/TR/css-color-4/#named-colors | W3C Document License (2023) https://www.w3.org/copyright/document-license-2023/ — copy/distribute allowed for any purpose; derivatives allowed only "to facilitate implementation of the technical specifications"; notice required: "Copyright © 2023 W3C®. This software or document includes material copied from or derived from [title and URI]." | Yes — the name→hex table is factual data and is copied into every browser; include the W3C notice line with the spec title + URI. | **148** names (verified: aliceblue … yellowgreen, incl. `rebeccapurple #663399`); all names lowercase ASCII, hex 6 lowercase digits. |
+| XKCD colour survey https://xkcd.com/color/rgb.txt | **CC0** (file header: `# License: https://creativecommons.org/publicdomain/zero/1.0/`) | Yes, no notice needed | **949** lines (verified). Format `name TAB #rrggbb TAB` (note the trailing tab). Max name length 26; all hex valid lowercase; no duplicate names. Contains crude names ("poo", "baby poop green", "diarrhea", "vomit") — apply a blocklist. |
+| faker-js `en/color/human.ts` | MIT | Yes | 31 names |
+| Pantone | Proprietary | **Avoid** | — |
+
+## 9. Dates
+
+### IANA tz database
+
+- Source https://www.iana.org/time-zones ; files https://data.iana.org/time-zones/tzdb/ (current version **2026d**, verified). LICENSE: "Unless specified below, all files in the tz code and data (including this LICENSE file) are in the public domain" (only `date.c`, `newstrftime.3`, `strftime.c` are BSD-3). Embed: Yes, no notice needed.
+- `zone1970.tab` — **312** data rows; tab-separated; UTF-8; `#` comments. Columns: (1) comma-separated ISO 3166-1 alpha-2 codes of countries overlapping the zone, most-populous first; (2) coordinates in ISO 6709 `±DDMM±DDDMM` or `±DDMMSS±DDDMMSS` (lat then lon, no separator); (3) TZ name (`Europe/Zurich`); (4) comment, present iff the country has several zones. One row per timezone where civil time has agreed since 1970.
+- `zone.tab` — **418** rows, older one-country-per-row table (column 1 is a single country code); kept for backward compatibility. Prefer `zone1970.tab`.
+- `backward` — **252** `Link TARGET LINK-NAME` lines mapping old/merged names (`US/Eastern`, `Asia/Calcutta`, `Europe/Kiev`) to current ones; also a few `Zone` entries. Use it to (a) accept aliases as input and (b) never emit them as output.
+- UTC offsets: do **not** embed — offsets depend on DST rules that change several times a year. Derive at runtime: `new Intl.DateTimeFormat('en-US', { timeZone, timeZoneName: 'longOffset' }).formatToParts(date)` → part `timeZoneName` = `GMT-05:00` (or `GMT` for zero) — strip `GMT`, treat bare `GMT` as `+00:00`. Validation of a generated zone name: `Intl.supportedValuesOf('timeZone')` (canonical names only; whether links like `US/Eastern` appear depends on the ICU build) or `try { new Intl.DateTimeFormat(undefined, { timeZone }) } catch { invalid }`, which accepts links too.
+
+### ISO 8601 layouts (https://en.wikipedia.org/wiki/ISO_8601)
+
+| Layout | Extended | Basic | Example |
+|---|---|---|---|
+| Calendar date | `YYYY-MM-DD` | `YYYYMMDD` | 2009-01-06 |
+| Ordinal date | `YYYY-DDD` | `YYYYDDD` | 1981-095 |
+| Week date | `YYYY-Www-D` | `YYYYWwwD` | 2009-W01-1 (weeks start Monday, week 1 contains 4 Jan) |
+| Time | `hh:mm:ss[.fff]` | `hhmmss` | 13:47:30 (24 h; `T` prefix optional standalone) |
+| UTC / offset | `Z` or `±hh:mm` | `±hhmm`, `±hh` | 2007-04-05T14:30Z, 2024-06-01T09:00-05:00 |
+| Date-time | date `T` time offset | | 2007-04-05T14:30:00+02:00 |
+| Duration | `PnYnMnDTnHnMnS` / `PnW` | | P3Y6M4DT12H30M5S |
+| Interval | start`/`end, start`/`duration, duration`/`end | | 2007-03-01T13:00:00Z/2008-05-11T15:30:00Z |
+| Recurring | `Rn/`interval | | R5/2008-03-01T13:00:00Z/P1Y2M10DT2H30M |
+
+RFC 3339 profile (what JSON/APIs usually mean): extended forms only, `T`/`Z` may be lowercase, space allowed instead of `T`, `-00:00` = unknown offset, no durations/intervals/week/ordinal dates. Generate RFC 3339 by default.
+
+### Unicode CLDR (locale month/weekday names, date patterns)
+
+- Licence: **Unicode License v3** (SPDX `Unicode-3.0`), OSI-approved 2023-11-17 (https://opensource.org/license/unicode-3-0). Text https://www.unicode.org/license.txt — MIT-style: use/copy/modify/sell "provided that either (a) this copyright and permission notice appear with all copies of the Data Files or Software, or (b) this copyright and permission notice appear in associated Documentation"; no use of the Unicode name in advertising. Embed: Yes; put the notice in LICENSE/NOTICE.
+- npm: `cldr-core`, `cldr-dates-full`, `cldr-numbers-full` (peer of dates), etc.; current **48.2.0** (CLDR 48, verified via npm registry, `license: Unicode-3.0`); repo https://github.com/unicode-org/cldr-json .
+- Path is **`cldr-dates-full/main/<locale>/ca-gregorian.json`** (not `dates/gregorian.json`), rooted at `main.<locale>.dates.calendars.gregorian`. Keys (verified for `en`): `months`, `days`, `quarters`, `dayPeriods`, `eras`, `dateFormats`, `dateSkeletons`, `timeFormats`, `timeSkeletons`, `dateTimeFormats`, `dateTimeFormats-atTime`, `dateTimeFormats-relative`.
+  - `months.{format|stand-alone}.{abbreviated|narrow|wide}` keyed `"1".."12"`; `days.{format|stand-alone}.{abbreviated|narrow|short|wide}` keyed `sun..sat`.
+  - `dateFormats`: en = full `EEEE, MMMM d, y`, long `MMMM d, y`, medium `MMM d, y`, short `M/d/yy`.
+  - `timeFormats`: en = full `h:mm:ss a zzzz`, long `h:mm:ss a z`, medium `h:mm:ss a`, short `h:mm a` — **the `a` is preceded by U+202F NARROW NO-BREAK SPACE**; `*-alt-ascii` variants use a plain space. Pick one spelling (ASCII) and ignore the other.
+  - `dateTimeFormats.{full|long|medium|short}` = glue like `{1}, {0}` ({1} date, {0} time); `availableFormats` = skeleton→pattern map (`yMMMd` → `MMM d, y`), plus `intervalFormats`, `appendItems`.
+  - `dayPeriods.format.abbreviated`: `am: AM`, `pm: PM`, plus `midnight`, `noon`, `morning1`… variants.
+- Pattern letters follow LDML (https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table): `y` year, `M`/`L` month (format/stand-alone), `d` day, `E`/`c` weekday, `h` 1–12, `H` 0–23, `a` AM/PM, `z`/`zzzz` zone name, `x`/`X` ISO offset; literal text in single quotes. Same patterns drive `Intl.DateTimeFormat` internally, so runtime `Intl` can replace embedding for month/weekday names if bundle size matters.
+
+## Summary of licence verdicts
+
+- Embed freely (PD/CC0): PubChem periodic table, NIST, NASA fact sheet, Wikidata, Moby POS, 12dicts (PD lists), Bartlett via Gutenberg, XKCD colours, IANA tz, Lorem ipsum/Cicero.
+- Embed with notice: WordNet 3.x (WordNet License), Open English WordNet (CC BY 4.0), SCOWL/ESDB (custom MIT-like), GBIF backbone (CC BY 4.0 + citation), BIPM SI (CC BY 4.0), CSS named colours (W3C notice), CLDR (Unicode-3.0), Google Books ngrams (CC BY 3.0), faker-js lists (MIT).
+- Avoid: Bowserinator periodic table (CC BY-SA 3.0), wordfreq data (CC BY-SA 4.0), hermitdave (CC BY-SA 4.0), Norvig count_1w (LDC-derived, unstated), Wikiquote (CC BY-SA), Pantone.
+- UNVERIFIED: American Red Cross figures (site blocks fetches), any global blood-type table, BIPM PDF front-matter licence line, classic SCOWL per-size counts, USDA PLANTS, IAU moon counts, exact faker dog count, Bartlett 10th-edition text availability.

docs/research/part4-iso-commerce-books.md

@@ -0,0 +1,139 @@
+# Part 4 — ISO lists, commerce, books/music/food
+
+Researched 2026-09-17. "Embed?" = can the data ship inside an MIT-licensed repo. Anything marked UNVERIFIED was not confirmed against the primary source.
+
+## 10. ISO lists
+
+### ISO 3166-1 countries
+
+| Source | Licence | Embed in MIT repo? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [datasets/country-codes](https://github.com/datasets/country-codes) | PDDL (README: "Public Domain Dedication and License"; GitHub API detects no licence file) | Yes, no attribution required. README caveat: ISO itself says its list is "for internal use and non-commercial purposes free of charge" — the repo argues no rights subsist in a list of facts | 249 | ISO3166-1-Alpha-2/-Alpha-3/-numeric, official_name_en, UNTERM names (ar/zh/en/fr/ru/es short+formal), Dial, TLD, Capital, Continent, Languages, ISO4217 currency code/name/numeric/minor_unit, Region/Sub-region/Intermediate names+codes (M49), FIPS, IOC, FIFA, MARC, ITU, WMO, GAUL, EDGAR, Geoname ID, wikidata_id, CLDR display name, is_independent, LDC/LLDC/SIDS flags | CSV `data/country-codes.csv` |
+| [annexare/Countries](https://github.com/annexare/Countries) (npm `countries-list` 3.4.1, 2026-07-14) | MIT | Yes; keep MIT notice | 252 countries, 185 languages (incl. non-ISO entries like AC, XK) | per country: name, native, phone[] (calling codes), continent, capital, currency[], languages[], optional alias[], partOf; separate ISO 4217 currency table (name, native, symbol, numeric, decimals); languages: name, native | TS source, exported JSON/CSV/SQL |
+| [Debian iso-codes](https://salsa.debian.org/iso-codes-team/iso-codes) | LGPL-2.1-or-later (Debian copyright file) | Not safely as embedded data in an MIT repo — LGPL applies to the data files; you would have to ship it as a separately licensed component with LGPL notice. Avoid unless accepting that | 3166-1: 249 | alpha_2, alpha_3, numeric, name, official_name, common_name, flag | JSON `data/iso_3166-1.json` (raw URL needs `?inline=false`) |
+| [mledoze/countries](https://github.com/mledoze/countries) | ODbL-1.0 | No — share-alike + attribution; derived databases must be ODbL. Avoid | ~250 | common/official names, cca2/cca3/ccn3/cioc, tld, currencies, idd (calling codes), capital, region/subregion, languages, translations, latlng, borders, area, demonyms, flag emoji, UN member | JSON/CSV/XML/YAML |
+| [restcountries](https://gitlab.com/restcountries/restcountries) | MPL-2.0 (source repo; hosted API v5 is proprietary/API-key) | File-level copyleft: a copied data file stays MPL and must carry the notice; OK to ship next to MIT code but not to relicense. Prefer PDDL/MIT sources | 250+ | name, tld, cca2, ccn3, cca3, cioc, fifa, independent, status, unMember, currencies, idd, capital, capitalInfo, altSpellings, region, subregion, continents, languages, translations, latlng, landlocked, borders, area, flag (emoji), demonyms, flags, coatOfArms, population, maps, gini, car, postalCode, startOfWeek, timezones | JSON `src/main/resources/countriesV3.1.json` |
+| [lukes/ISO-3166-Countries-with-Regional-Codes](https://github.com/lukes/ISO-3166-Countries-with-Regional-Codes) | CC BY-SA 4.0 (LICENSE.md) | No — share-alike. Avoid | 249 | name, alpha-2, alpha-3, country-code, iso_3166-2, region, sub-region, intermediate-region + codes | CSV/JSON/XML (all, slim-2, slim-3) |
+| [stefangabos/world_countries](https://github.com/stefangabos/world_countries) | Data: CC BY-SA 4.0 (README, GitHub licence detection); npm `world_countries_lists` package.json says LGPL-3.0-or-later — contradictory | No — share-alike either way. Avoid | 249 world / 193 UN; subdivisions 5046 | id (numeric), alpha2, alpha3, name (37 languages); subdivisions: country, code, name, name_en, type, parent | JSON/CSV/PHP/SQL/XML |
+| [i18n-iso-countries](https://www.npmjs.com/package/i18n-iso-countries) 7.14.0 | MIT | Yes | 78 language files | alpha2, alpha3, numeric, localized names | JSON per language |
+| Wikipedia ISO 3166-1 | CC BY-SA 4.0 | No (share-alike) | — | — | — |
+| [ISO OBP](https://www.iso.org/obp) | Proprietary; ISO grants free use only for "internal use and non-commercial purposes" (quoted in datasets/country-codes README) | No | — | — | — |
+
+- Flag emoji: derive, no data needed — for alpha-2 `XY`, emit U+1F1E6 + (X − 'A') and U+1F1E6 + (Y − 'A') (regional indicator symbols).
+- Recommendation: `datasets/country-codes` (PDDL) covers every requested field (name, alpha2, alpha3, numeric, calling code, TLD, capital, currency, languages) in one CSV; flag emoji derived. `annexare` (MIT) is the fallback for native names/calling codes if PDDL's ISO caveat worries you; it lacks TLD.
+- Validation rules: alpha-2 `^[A-Z]{2}$`, alpha-3 `^[A-Z]{3}$`, numeric `^\d{3}$` (zero-padded, keep as string); calling code 1–3 digits, NANP countries share `1`; TLD `^\.[a-z]{2}$` (ccTLD = lowercase alpha-2, exceptions: GB uses `.uk`).
+
+### ISO 639 languages
+
+| Source | Licence | Embed? | Rows | Fields | Format |
+|---|---|---|---|---|---|
+| [langs](https://www.npmjs.com/package/langs) 2.0.0 (2017, unmaintained) | MIT | Yes | 184 | name, local, 1 (639-1), 2, 2T, 2B, 3 | `data.js` array |
+| [iso-639-1](https://www.npmjs.com/package/iso-639-1) 3.1.6 (2026-07-02) | MIT | Yes | 183 | code, name, nativeName | `src/data.js` |
+| annexare/Countries languages | MIT | Yes | 185 | code (639-1), name, native | TS/JSON |
+| Debian iso-codes 639-2 / 639-3 / 639-5 | LGPL-2.1+ | See above (LGPL) | 639-2: 487; 639-3: 7923 | 639-2: alpha_2, alpha_3, bibliographic, name, common_name; 639-3: + inverted_name, scope, type | JSON |
+| [SIL ISO 639-3 tables](https://iso639-3.sil.org/code_tables/download_tables) | SIL terms: may incorporate into software (commercial or not) with attribution to iso639-3.sil.org, must not modify identifiers, and product "does not provide a means to redistribute the code set" | Risky — a git repo with the table IS a means to redistribute; avoid embedding the full table. Codes themselves are facts | 7927 | Id, Part2b, Part2t, Part1, Scope, Language_Type, Ref_Name, Comment | tab-delimited `iso-639-3.tab` |
+| [Library of Congress ISO 639-2](https://www.loc.gov/standards/iso639-2/ascii_8bits.html) | US federal government work → public domain in the US | Yes | ~487 (matches Debian 639-2 count) | pipe-delimited: alpha3-B, alpha3-T, alpha2, English name, French name — UNVERIFIED: loc.gov is behind a Cloudflare bot check (WebFetch, curl and Playwright all blocked); field layout from memory | `ISO-639-2_utf-8.txt` |
+
+- Recommendation: `langs` or `iso-639-1` (MIT) for 639-1 names; both are small and derived from public code tables. Anything 639-3-sized is LGPL or SIL-restricted.
+- Rules: 639-1 `^[a-z]{2}$`, 639-2/3 `^[a-z]{3}$`; 639-2 has B/T pairs (e.g. `fre`/`fra`, `ger`/`deu`) — pick one consistently (T codes = 639-3).
+
+### ISO 3166-2 subdivisions
+
+| Source | Licence | Embed? | Rows | Fields |
+|---|---|---|---|---|
+| Debian iso-codes 3166-2 | LGPL-2.1+ | LGPL caveat | 5046 | code, name, type, parent |
+| [olahol/iso-3166-2.json](https://github.com/olahol/iso-3166-2.json) | package.json says ISC; no LICENSE file; GitHub detects none; data scraped from eQuest xls (provenance unclear); last push 2021 | No — unclear licence and provenance | 237 countries / 3807 divisions | `{alpha2: {name, divisions: {code: name}}}` |
+| [esosedi/3166](https://github.com/esosedi/3166) (npm `iso3166-2-db` 2.3.11) | MIT | Yes; data mixes GeoNames (CC BY 4.0), Wikipedia (CC BY-SA), OSM (ODbL) — the MIT label may not survive scrutiny for the Wikipedia/OSM-derived parts. UNVERIFIED how much is derived from each | "all countries and regions" (count not stated) | ISO 3166-2 + FIPS codes, names in 12 languages, admin level, GeoNames/OSM/Wikipedia/WOF ids |
+| stefangabos subdivisions | CC BY-SA 4.0 | No | 5046 | country, code, name, name_en, type, parent |
+| [US Census state.txt](https://www2.census.gov/geo/docs/reference/state.txt) | US government work → public domain | Yes | 57 (50 states + DC + 6 territories) | STATE (FIPS), STUSAB, STATE_NAME, STATENS |
+
+- Recommendation: for an en_US library, ship only the US list (Census/USPS, public domain: 50 states + DC, optionally territories) — ISO 3166-2 code = `US-` + USPS abbreviation. Skip the ~5000-row world list; every open compilation is LGPL, CC BY-SA or unclear.
+- Rule: `^[A-Z]{2}-[A-Z0-9]{1,3}$`.
+
+## 11. Commerce
+
+### Word lists (product name / adjective / material)
+
+- [@faker-js/faker](https://github.com/faker-js/faker) 10.6.0 — MIT (LICENSE covers code and locale data; copyright Faker contributors 2022-2025 and Marak Squires 2011-2020; acknowledges Ruby Faker and Perl Data::Faker, both MIT). No separate data licence.
+- Provenance policy in [CONTRIBUTING.md](https://github.com/faker-js/faker/blob/next/CONTRIBUTING.md): "Faker must not contain copyrighted materials"; facts (finite known lists) are OK; compilations must not be copied from a single source (Wikipedia, "most popular" articles, government sites) because "a compilation of facts can be copyrighted" — contributors are told to draw on multiple sources and their own judgement.
+- Verdict: reusing faker's `en` commerce lists (`product_name.adjective/material/product`, `department`) is fine under MIT — keep the MIT notice with the copied data. The lists are short curated words (no external dataset). Nothing in the repo or docs restricts locale-data reuse beyond MIT.
+
+### Garment and shoe sizes
+
+- Letter sizes XS–XXL: EN 13402-3 letter codes with chest/bust ranges (Wikipedia [EN 13402](https://en.wikipedia.org/wiki/EN_13402), CC BY-SA — but the table is a handful of facts; the standard itself is paywalled). Men chest / women bust cm: XXS 70–78 / 66–74, XS 78–86 / 74–82, S 86–94 / 82–90, M 94–102 / 90–98, L 102–110 / 98–107, XL 110–118 / 107–119, XXL 118–129 / 119–131, 3XL 129–141 / 131–143.
+- US numeric women's sizes (0–20, even) come from ASTM D5585 (paywalled); no open dataset. Just generate the even-number series.
+- Shoe sizes: no open dataset found (search hits are ML demos and retailer charts). Use the formulas from Wikipedia [Shoe size](https://en.wikipedia.org/wiki/Shoe_size) (facts, not copyrightable): EU (Paris point) ≈ 1.5 × foot cm + 2 (i.e. 3/20 × mm + 2); UK adult ≈ 3 × foot in − 23; US men ≈ 3 × foot in − 22 = UK + 1; US women (common) = US men + 1.5 (FIA scale: 3 × foot in − 21); Mondopoint = foot length mm. ISO/TS 19407:2023 has the official conversion tables (paywalled). Generate from a foot length (men 24–31 cm, women 21–27 cm) in half-size steps so all systems agree.
+
+### Check-digit rules (all verified against Wikipedia articles; algorithms are facts)
+
+| Code | Rule |
+|---|---|
+| EAN-13 / UPC-A / GTIN-8/12/13/14 | Weights 3,1,3,1… starting from the rightmost data digit (the one next to the check digit); check = (10 − sum mod 10) mod 10. UPC-A = EAN-13 with a leading 0 (GS1 prefix of a 12-digit GTIN is `0` + first two digits). GTIN-14 first digit is an indicator 1–8 (packaging level) or 9 (variable measure); 0 is not valid there. |
+| UPC-A number system (first digit) | 0,1,6,7,8,9 regular; 2 variable-weight; 3 drugs (NDC); 4 in-store/local; 5 coupons. |
+| ISBN-10 | Σ(weights 10..2 × first 9 digits) + check ≡ 0 mod 11; check = (11 − sum mod 11) mod 11, 10 → `X`. |
+| ISBN-13 | EAN-13 with prefix 978 or 979. 979 groups in use: 979-8 USA, 979-10 France, 979-11 Korea, 979-12 Italy. 978 groups 0 and 1 = English-language. For an en_US generator: `978-0…`, `978-1…`, `979-8…`. The official range file (`https://www.isbn-international.org/export_rangemessage.xml`) is XML only and the site says "You must not republish material from our site … without our written permission" — do not embed it; group/registrant length only matters for hyphenation, not validity. |
+| ISSN | Format `NNNN-NNNC`; weights 8..2 over first 7 digits, sum mod 11; check = 0 if remainder 0 else 11 − remainder; 10 → `X`. EAN-13 form: `977` + 7 ISSN digits (no check) + 2 variant digits + EAN check. |
+| IMEI | 15 digits: TAC 8 (RBI 2 + 6) + serial 6 + Luhn check (ISO/IEC 7812; from the right, double every second digit, sum digits, total ≡ 0 mod 10). SVN never enters the check. IMEISV = 14 digits + 2-digit SVN, no check. RBI `00` = test IMEI (Wikipedia [Reporting Body Identifier](https://en.wikipedia.org/wiki/Reporting_Body_Identifier)); live RBIs: 01 CTIA/PTCRB (US), 35 TÜV SÜD/BABT (UK), 86 TAF (China), 99 GHA. For fake data use TAC `00xxxxxx` so it can never match a real handset — UNVERIFIED: third-party pages claim the GSMA TS.06 test-TAC shape is `00 44` + 4 digits; the TS.06 PDF could not be text-extracted here. |
+| ASIN | 10 chars `[A-Z0-9]{10}`; non-books start `B0` + 8 alphanumerics (`^B0[A-Z0-9]{8}$`); books reuse their ISBN-10 verbatim. No check digit (Wikipedia). |
+| SKU | Not standardised (Wikipedia). Convention only: 8–12 uppercase alphanumerics, e.g. `[A-Z]{3}-\d{4}-[A-Z]{2}`; avoid leading 0 and O/I if you want scanner-safe. |
+
+### GS1 prefixes
+
+- Official table: [gs1.org/standards/id-keys/company-prefix](https://www.gs1.org/standards/id-keys/company-prefix) (rendered via Playwright; WebFetch gets 403). Copyright notice ([gs1.org/terms-use](https://www.gs1.org/terms-use)): reproduction allowed only "in unaltered form … for your personal, non-commercial use or use within your organisation"; "You are not permitted to re-transmit, distribute or commercialise the information or material without seeking prior written approval from GS1." → Do not copy the GS1 page. The prefix-to-country mapping is factual and also on Wikipedia ([List of GS1 country codes](https://en.wikipedia.org/wiki/List_of_GS1_country_codes), CC BY-SA 4.0 — avoid verbatim copy). Safe path: embed only the US/special ranges you need, written from the facts below.
+- Ranges relevant to an en_US generator (from the GS1 page): 001–019, 030–039, 060–139 GS1 US (UPC-A compatible); 020–029 restricted circulation within a region; 040–049 restricted within a company; 050–059 GS1 US reserved; 200–299 restricted circulation (region); 952 "used for demonstrations and examples of the GS1 system"; 977 ISSN; 978–979 ISBN; 980 refund receipts; 981–983 coupons (common currency); 990–999 coupons. Prefixes do not identify country of origin. Selected others: 300–379 France, 400–440 Germany, 450–459 & 490–499 Japan, 500–509 UK, 690–699 China, 730–739 Sweden, 750 Mexico, 754–755 Canada, 760–769 Switzerland, 800–839 Italy, 840–849 Spain, 870–879 Netherlands, 880–881 South Korea, 890 India, 930–939 Australia, 940–949 New Zealand.
+- Generator rule: for fake UPC/EAN use prefix `952` (GS1 demo) or a `2xx` restricted-circulation prefix — never a real company prefix; for US-looking UPC-A use number system 0–1/6–8 with a random 5-digit manufacturer code and accept that it may collide with real products.
+
+## 12. Books, music, food
+
+### Project Gutenberg catalog
+
+- URL: `https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv` (20 MB) / `pg_catalog.csv.gz` (5.3 MB), regenerated weekly; also RDF (`rdf-files.tar.bz2`, 121 MB) and MARC.
+- Fields: `Text#, Type, Issued, Title, Language, Authors, Subjects, LoCC, Bookshelves` (comma CSV, quoted; Authors as `Last, First, birth-death`, `;`-separated).
+- Count (2026-09-17): 79 381 rows; Type: Text 78 130, Sound 1 114, Dataset 89, Image 33, other 12; Language: en 62 860, fr 4 190, fi 3 684, de 2 425, it 1 110.
+- Licence: each RDF record carries `<cc:Work><cc:license rdf:resource="https://creativecommons.org/publicdomain/zero/1.0/"/>` → catalog metadata is CC0 1.0 (verified on `cache/epub/1/pg1.rdf`). The CSV itself has no header notice; treat as CC0 by the same source. Terms of use: do not hammer their servers (bulk feeds are the sanctioned path), "Project Gutenberg" is a trademark — royalties for commercial use of the *name*; do not brand the data file with it beyond a source citation.
+- Embed: yes (CC0). For a fake-data library ship a filtered subset (Type=Text, Language=en, title + first author, maybe 5–10k rows) rather than 20 MB.
+
+### Music genres and instruments
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| [MusicBrainz genre list](https://musicbrainz.org/genres) | Genre is a core entity (schema doc lists 13 core entities incl. Genre) → core data dump `mbdump.tar.bz2` is CC0. The genre→entity *associations* come via user tags (CC BY-NC-SA 3.0) — do not use those | Yes, names only | 2 202 (`/ws/2/genre/all?fmt=json`, `genre-count`) | WS API needs a descriptive User-Agent; list is a JSON array of `{id, name, disambiguation}` |
+| [Discogs data dumps](https://data.discogs.com/) | CC0 | Yes | genre/style vocab is embedded in release XML (monthly dumps, GBs) — no standalone style list; UNVERIFIED count (~15 genres, ~600 styles from memory) | Impractical to extract; prefer MusicBrainz |
+| ID3v1 genres | Names 0–79 from the 1999 ID3v1 spec, 80–125 Winamp, 126–191 Winamp 5.6 (2010) — a de-facto spec table, treated as public domain facts | Yes | 192 (0–191) | Wikipedia [List of ID3v1 genres](https://en.wikipedia.org/wiki/List_of_ID3v1_genres) (CC BY-SA page; the list itself is a spec enumeration). id3.org returned HTTP 500 during research — UNVERIFIED against the original spec page |
+| Wikidata instruments | CC0 | Yes | 9 194 items `P31/P279* Q34379` with English labels; 2 372 with a Hornbostel–Sachs number (P1762) | Filter by P1762 for a clean orchestral/folk instrument list |
+| Wikidata music genres | CC0 | Yes | 6 619 (`Q188451`) | Noisier than MusicBrainz |
+
+### USDA FoodData Central
+
+- Downloads: [fdc.nal.usda.gov/download-datasets](https://fdc.nal.usda.gov/download-datasets/) (April 2026 release):
+  - Foundation Foods: JSON 459 KB zip / 6.5 MB; CSV 3.7 MB zip / 32 MB — 394 foods (API `totalHits`, dataType=Foundation).
+  - SR Legacy (final, April 2018): JSON 12.3 MB zip / 205 MB; CSV 6.7 MB zip / 54 MB — 7 793 foods.
+  - FNDDS 2021-2023: CSV 200 MB zip / 1.6 GB — 5 432 foods.
+  - Branded: CSV 428 MB zip / 2.9 GB — 433 403 foods.
+  - Full: CSV 460 MB zip / 3.1 GB.
+- Licence ([API guide](https://fdc.nal.usda.gov/api-guide/)): "USDA FoodData Central data are in the public domain and they are not copyrighted. They are published under CC0 1.0 Universal (CC0 1.0)". No permission needed; USDA *requests* listing FoodData Central as source and notifying them. Suggested citation: "U.S. Department of Agriculture, Agricultural Research Service. FoodData Central, 2019. fdc.nal.usda.gov."
+- Embed: yes (CC0). Ship a derived list (SR Legacy `description` + `food_category`) not the raw dump.
+- CSV structure (UNVERIFIED — the field-description PDF could not be text-extracted; from memory): `food.csv` (fdc_id, data_type, description, food_category_id, publication_date), `food_category.csv` (id, code, description), `nutrient.csv` (id, name, unit_name, nutrient_nbr, rank), `food_nutrient.csv` (id, fdc_id, nutrient_id, amount, …), `food_portion.csv`, `measure_unit.csv`, `sr_legacy_food.csv` (fdc_id, NDB_number), `foundation_food.csv`.
+
+### Dishes, cuisines, drinks
+
+| Source | Licence | Embed? | Count | Notes |
+|---|---|---|---|---|
+| Wikidata dishes (`Q746549`) | CC0 | Yes | 7 480 with English label | SPARQL `?i wdt:P31/wdt:P279* wd:Q746549` |
+| Wikidata cuisines (`Q1968435`) | CC0 | Yes | 219 | |
+| Wikidata cocktails (`Q134768`) | CC0 | Yes | 305 | |
+| Wikidata beer (`Q44` subclass tree) / wine (`Q282`) | CC0 | Yes | 420 / 3 025 | Wine tree is mostly appellations/brands; filter by P279 depth |
+| IBA official cocktails | IBA site "© IBA 2026 – All rights reserved"; Wikipedia list CC BY-SA 4.0 | Names only are facts (102 cocktails: 34 Unforgettables, 34 Contemporary Classics, 34 New Era, 2024 revision); do not copy recipes/descriptions | 102 | Safest: take the names from Wikidata (`P31 Q134768`) rather than the IBA page |
+| [Open Brewery DB](https://github.com/openbrewerydb/openbrewerydb) | MIT (LICENSE, © 2025 Open Brewery DB) | Yes | 11 931 breweries (US 8 308, DE 1 445, AU 514, BE 478, CA 283, NZ 243) | `breweries.csv`: id (UUID), name, brewery_type (micro 5 906, brewpub 3 947, closed 644, planning 635, regional 240, contract 208, large 137, proprietor, taproom, bar, nano, cidery, beergarden), address_1..3, city, state_province, postal_code, country, phone, website_url, longitude, latitude. Real businesses — fine for names, but generating "fake" data with real addresses/phones may be undesirable |
+| Wikipedia lists (cuisines, dishes, IBA) | CC BY-SA 4.0 | No verbatim copies | | |
+
+## Summary of safe picks
+
+- Countries: `datasets/country-codes` (PDDL) or `annexare/Countries` (MIT); flag emoji derived from alpha-2.
+- Languages: `langs` / `iso-639-1` (MIT, 639-1 only).
+- Subdivisions: US-only list from Census (public domain); skip world list.
+- Commerce words: faker `en` lists (MIT). Check digits: implement from the rules above; GS1 prefix table: hand-write the few ranges needed, use `952`/`2xx` for generated barcodes; ISBN range file: do not embed.
+- Books: Gutenberg `pg_catalog.csv` (CC0), filtered.
+- Music: MusicBrainz genre names (CC0, 2 202), ID3v1 list (192), Wikidata instruments (CC0).
+- Food: USDA FDC SR Legacy/Foundation descriptions (CC0, cite USDA), Wikidata dishes/cuisines (CC0), Open Brewery DB (MIT).
+- Avoid: mledoze (ODbL), lukes (CC BY-SA), stefangabos (CC BY-SA/LGPL), Debian iso-codes (LGPL), SIL 639-3 table (no-redistribution clause), GS1/ISBN-International pages (all rights reserved), Wikipedia tables verbatim (CC BY-SA), MusicBrainz tag associations (CC BY-NC-SA).

docs/research/research-age-bands.md

@@ -0,0 +1,10 @@
+# Population by age band — research 2026-10-04
+
+Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
+
+| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
+|---|---|---|---|---|---|---|---|
+| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
+| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
+
+At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.

docs/research/research-geo-se.md

@@ -0,0 +1,114 @@
+# Open data sources for real Swedish geographic and postal-address data
+
+Researched 2026-09-16. Goal: Sweden → län → kommun → tätort/postort → street, with real postal codes, embeddable in an MIT-licensed library.
+
+## TL;DR
+
+| Need | Best source | Licence | Embed in MIT repo? |
+|---|---|---|---|
+| Län + kommun names/codes | SCB `kommunlankod-2026.xlsx` | CC0 1.0 | Yes, no attribution needed |
+| Population per kommun (weighting) | SCB PxWeb API `BE0101A/BefolkningNy` | CC0 1.0 | Yes |
+| Tätorter (2,017) with code, kommun, län, population | SCB open geodata WFS/GeoPackage `stat:Tatorter_2023` + PxWeb `MI0810A/LandarealTatortN` | CC0 1.0 | Yes |
+| Streets + house numbers + postnummer + postort + kommun + coordinates (~3.9M points) | Lantmäteriet *Belägenhetsadress Nedladdning, vektor* (STAC, GeoPackage per kommun) | CC BY 4.0 **plus** personal-data terms, purpose review, Basic-auth download | **Not as-is** — raw redistribution is bound by the personal-data terms; a derived aggregate (street ↔ postnummer ↔ postort, no house numbers) is the realistic path but needs a stated purpose approved by Lantmäteriet. Legal review needed. |
+| postnummer → postort (list only) | GeoNames `SE.zip` (18,887 rows, 1,780 postorter) | CC BY 4.0 | Yes with attribution — but source is an undated third-party file, likely stale |
+| Street names per kommun/postort without Lantmäteriet | Trafikverket NVDB *Gatunamn* (Lastkajen) | CC0 1.0 | Yes, but no postnummer; needs account to download |
+| Street names | OpenStreetMap | ODbL 1.0 | Effectively no (share-alike on the extracted database) |
+| Municipal address files | OpenAddresses `sources/se/*` (16 kommuner) | Mixed: CC0 (Göteborg, Malmö), CC BY 4.0 (Helsingborg), unspecified (Stockholm 2016 dump, Uppsala) | Per source |
+| Postal code register (authoritative) | PostNord → Postnummerservice Norden AB / Geposit AB | Commercial; "resale/sublicensing not permitted" | No |
+
+## 1. Lantmäteriet
+
+Since 2025-02-03 the EU High Value Datasets (HVD) products are fee-free but Lantmäteriet explicitly says they are *not* "öppna data" because conditions attach ([Avgiftsfria produkter](https://www.lantmateriet.se/sv/geodata/vara-produkter/avgiftsfria-produkter/)). Two tiers exist:
+
+- **Öppna data** — licence "Creative Commons, CC0" ([Öppna data](https://www.lantmateriet.se/sv/geodata/vara-produkter/avgiftsfria-produkter/oppna-data/)). Verified on Geotorget product pages for *Topografi 50/100/250/1M Nedladdning, vektor* ("Villkor: Creative Commons, CC0", "Juridisk prövning: Nej", GeoPackage, download in Geotorget or via the *Geotorget Nedladdning* API).
+- **Värdefulla datamängder (HVD)** — licence **CC BY 4.0** per the terms document *Användningsvillkor för värdefulla datamängder* (DNR LM2025/009266 v1.0, 2025-02-01, [PDF](https://www.lantmateriet.se/globalassets/geodata/geodataprodukter/anvandningsvillkor_for_vardefulla_datamangder.pdf)). Products with personal data use *Användningsvillkor för värdefulla datamängder som innehåller personuppgifter* (DNR LM2025/009269 v1.1, 2025-03-04, [PDF](https://www.lantmateriet.se/globalassets/geodata/geodataprodukter/anvandningsvillkor_for_vardefulla_datamangder_pu.pdf)).
+
+Per-product status (Geotorget product pages, rendered 2026-09-16):
+
+| Product | Fee | Terms | Legal review | Access | Format |
+|---|---|---|---|---|---|
+| [Belägenhetsadress Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/belagenhetsadress-nedladdning-vektor-api) | 0 | HVD terms **with personal data** | **Yes** | STAC API `https://api.lantmateriet.se/stac-vektor/v1`, collection `belagenhetsadresser` | GeoPackage, one zip per kommun |
+| [Belägenhetsadress Direkt](https://geotorget.lantmateriet.se/geodataprodukter/belagenhetsadress-direkt-api) | 0 | HVD terms with personal data | Yes | REST `https://api.lantmateriet.se/distribution/produkter/belagenhetsadress/v4.2` (OAuth2 via API portal) | JSON / XML |
+| [Belägenhetsadress Nedladdning, Inspire](https://www.lantmateriet.se/sv/geodata/vara-produkter/produktlista/belagenhetsadress-nedladdning-inspire/) | 0 | HVD terms with personal data | Yes (3–5 working days extra for security review) | Atom feed, login | GML; SWEREF 99 TM / ETRS89; semi-annual |
+| [Ortnamn Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/ortnamn-nedladdning-vektor-api) | 0 | HVD terms (no personal data) | No | STAC collection `ortnamn`, one national zip (58.5 MB) | GeoPackage |
+| [Ortnamn Direkt](https://geotorget.lantmateriet.se/geodataprodukter/ortnamn-direkt-api) | 0 | HVD terms | No | REST `…/distribution/produkter/ortnamn/v2.2` | JSON / XML |
+| Kommun, län och rike (administrativ indelning) | 0 | STAC says `CC-BY-4.0`; a Lantmäteriet HVD presentation (GISS, 2025-03-04) tables it as **CC0** ("CC0 används som licens för tjänster som är gemensamma för HVD och NGP … Kommun, Län och Rike") — **conflicting, unverified which prevails** | No | STAC collection `kommun-lan-rike` (zip ~6 MB, yearly + "aktuell"); OGC API Features `https://api.lantmateriet.se/ogc-features/v1/administrativ-indelning` (collections `kommuner`, `lan`, `rike`, plus `-2025`/`-2026`; items endpoint returns 401) | GeoPackage / GeoJSON |
+| [Topografi 10 Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/topografi-10-nedladdning-vektor) | 0 | HVD terms with personal data | **Yes** | Geotorget download / API | GeoPackage |
+| Topografi 50 / 100 / 250 / 1M Nedladdning, vektor | 0 | **CC0** | No | Geotorget download / *Geotorget Nedladdning* API | GeoPackage |
+
+Notes:
+- *Administrativ indelning* is not a standalone Geotorget product; it is the STAC collection `kommun-lan-rike`, the OGC API above, an [Inspire download](https://www.lantmateriet.se/sv/geodata/vara-produkter/produktlista/administrativ-indelning-nedladdning-inspire/) (Atom/WCS), or a layer in Topografi 100/250/1M (CC0). Topografi 1M also carries "tätortsgränser och tätortssymboler".
+- The STAC catalog root and collection/item listings are public (HTTP 200, no auth); it self-describes: "avgiftsfria och får användas enligt creative commons licens CC BY 4.0. För vissa informationsmängder kommer din användning att prövas juridiskt … och du behöver då godkänna särskilda användningsvillkor." Asset downloads on `dl1.lantmateriet.se` return **401 Basic** — credentials are the Geotorget account (private person) or a system account (organisation). Private accounts are "endast för privat bruk" ([konto-privatperson](https://geotorget.lantmateriet.se/konto-privatperson)); an MIT library is not private use, so apply as an organisation.
+- **Record count.** 290 STAC items (one per kommun), total 324 MB zipped. Smallest Bjurholm 143 KB, largest Stockholm 9.1 MB, Göteborg 8.9 MB, Malmö 4.9 MB. Fastighetsregistret held 3,882,361 valid addresses in 2022 (2,886,796 street addresses + 964,737 rural) per [SCB/LM presentation](https://kartografiska.se/wp-content/uploads/5C_Foretagsadresser_SCB_LM.pdf); the OSM import thread cites 3.7 M. So ~3.9 M address points, not 4–5 M.
+- **Fields** (JSON schema `belagenhetsadress-4.2.2.json`, matches *Nationell specifikation Adress* v1.0.1): `kommunkod`, `kommunnamn`, `kommundel` (geografisk kommundel), `adressomrade` + `adressomradestyp` (gatuadressområde / metertalsadressområde / byadressområde), `gardsadressomrade`, `adressplatsnummer`, `bokstavstillagg`, `lagestillagg`/`lagestillaggsnummer`, `adressplatstyp`, `adressplatsbeteckning`, `postnummer`, `postort` (both "beslutas av PostNord AB", only on current addresses), `popularnamn`, `distriktskod`/`distriktsnamn`, `adressattAnlaggning`, `anmarkningstyp`/`anmarkningstext`, `registerenhetsreferens` (property link), `objektidentitet`, `objektstatus`, `versionGiltigFran`, `geometri` (SWEREF 99 TM point). No län field — derive from kommunkod (first two digits).
+- Lantmäteriet treats addresses as personal data via Fastighetsregisterlagen § 6: usage only for the purpose approved in the application, EU/EEA-only storage, spot checks, revocation (terms PU §§ 3.3–4.2). The OSM community reports Lantmäteriet refused ODbL compatibility and OpenAddresses ([#7657](https://github.com/openaddresses/openaddresses/issues/7657), [#7608](https://github.com/openaddresses/openaddresses/issues/7608)) has not been able to include it; the OSM thread ([127427](https://community.openstreetmap.org/t/lantmateriet-belagenhetsadress/127427), 91 posts, last 2026-07-25) says one mapper got approval for a full 3.7 M import but no formal import has happened.
+- **Attribution text required (CC BY 4.0 + terms § 3.1):** product name as data source, "©Lantmäteriet", whether the data was processed, and that CC BY 4.0 applies — may be placed in accompanying documentation/metadata.
+- Ortnamn: name types include `Tätort` ("minst 200 invånare"), Bebyggelse, Trakt, Kyrka, Anläggning, nature names; Ortnamnsregistret holds ~980,000 names (Lantmäteriet, figure not re-verified). Not delivered as CSV.
+
+## 2. SCB (Statistics Sweden)
+
+- **Licence.** Statistics and geodata published as open data: "Creative Commons 0 1.0 Universal, CC0" ([användningsvillkor](https://www.scb.se/om-scb/om-scb.se-och-anvandningsvillkor)); since 2021-07-01. Optional credit "Källa: SCB" / "Source: Statistics Sweden"; do not cite SCB as source for data you have processed. API limit 10 requests / 10 s per IP.
+- **Län and kommuner with codes.** 21 län, 290 kommuner ([lan-och-kommuner](https://www.scb.se/hitta-statistik/regional-statistik-och-kartor/regionala-indelningar/lan-och-kommuner/)). Download: `https://www.scb.se/contentassets/7a89e48960f741e08918e489ea36354a/kommunlankod-2026.xlsx` (18 KB, HTTP 200). Codes: län 2 digits (01 Stockholm … 25 Norrbotten), kommun 4 digits = län + 2 (0180 Stockholm).
+- **Population per kommun.** PxWeb API `https://api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0101/BE0101A/BefolkningNy` — region dimension has 312 values (riket, 21 län, 290 kommuner), years 1968–2024 (verified 2026-09-16). Also `BE0101C/BefArealTathetKon` (density, 1991–2025).
+- **Tätorter.** 2,017 tätorter in 2023 (WFS `resultType=hits` on `stat:Tatorter_2023` → `numberOfFeatures="2017"`; SCB also states 2,017). Open geodata: `https://geodata.scb.se/geoserver/stat/wfs` (WFS/WMS) and GeoPackage download ([statistiska tätorter](https://www.scb.se/vara-tjanster/oppna-data/oppna-geodata/statistiska-tatorter/)). Attributes: `tatortskod`, `tatort`, `kommun`, `kommunnamn`, `lan`, `lannamn`, `area_ha`, `bef` (population), `ar`. Population per tätort also in PxWeb `MI0810/MI0810A/LandarealTatortN` (2,827 region codes, every 5 years to 2023). Also `Smaorter_2023`, `DeSO_2025`, `RegSO_2025`.
+
+## 3. Postal codes
+
+- **Owner.** PostNord Sverige owns and operates the postnummer system; PTS is supervisory authority; a Postnummerråd approves changes 3–4 times a year (sv.wikipedia). Register management is delegated to **Postnummerservice Norden AB**, machine access via **Geposit AB**. Postnummerservice's [terms](https://postnummerservice.se/en/information/kopvillkor): "free use within your own organization", "resale of data is not permitted", "sublicensing to third parties is not permitted" — unusable for an open library.
+- **No open list from PostNord, Lantmäteriet or Digg.** dataportal.se search for "postnummer" (2026-09-16) yields only Postnummerservice's own commercial catalogue entry ("Svenska postnummer och postorter", published 2012), Helsingborg's municipal postnummer polygons, and Lantmäteriet's Inspire address entry. The old dataportal community called the postnummer system "slarvats bort till ett privat bolag". GitHub CSVs (`zegl/sweden-zipcode`, `lapplandi/sveriges-postnummer`, `beshrkayali/sverige_postnummer`) state no licence or provenance — do not use.
+- **Lantmäteriet addresses carry `postnummer` + `postort`** on every current address point (set by PostNord), so the address product is the only lawful open-ish route to a *complete, current* postnummer ↔ postort ↔ street mapping.
+- **GeoNames `SE.zip`** ([readme](https://download.geonames.org/export/zip/readme.txt)): CC BY 4.0 ("a link on your website to www.geonames.org is ok"). Downloaded and counted: **18,887 rows, 18,887 distinct postnummer, 1,780 postorter, 22 admin1 values (21 län + blank), 285 admin2 (kommun code) values; 3,055 rows lack kommun; accuracy: 16,115 rows = 4 (centroid of postal code area), 389 = 1, 2,383 blank**. Fields: country, postal code (`111 64` with space), place name, admin1 name/code (län), admin2 name/code (kommun 4-digit), admin3, lat, lon, accuracy. The GeoNames sources page lists Sweden's source as `www.pellesoft.se/upload/program/prg00745.zip` with **no date** (URL now dead) — a hobbyist upload, so freshness is unknown; the row count (18.9 k) exceeds today's ~17 k total codes, indicating retired codes are still present. The zip is rebuilt daily (Last-Modified 2026-09-16) but that says nothing about the underlying data. 96 KB zipped.
+- **Counts.** Postnummerservice: "~17 000 postal codes", "10 500 deliverable postal codes", "1 743 postal cities"; Kartanalys: 10,839 geographic postnummer, 1,740 postorter; Wikipedia: >16,100 areas (2008). Changes quarterly.
+- **Structure** (sv/en Wikipedia): 5 digits written `NNN NN`. Lower numbers further south, except 1xx xx = Stockholm. First digit/tens: 10–19 Stockholm, 20–29 Skåne (Malmö 20–21), 30–39 southern Sweden, 40–49 Göteborg area (Göteborg 40–41), 50–59, 60–69, 70–79, 80–89, 90–99 progressively north to Norrbotten (98x). Postorter come in 2-, 3- or 5-position sizes; in two-position places the 3rd digit (in three-position the 4th) marks delivery type: 0 = boxes/postal, 1 = boxes/business, 2–4 and 6–7 = street delivery, 5 = rural (lantbrevbäring), 8 = reply mail, 9 = competitions/temporary — with exceptions in big cities. Box codes are therefore not street-deliverable and should be excluded when generating street addresses.
+
+## 4. Streets
+
+- **Lantmäteriet** — see § 1; `adressomrade` with `adressomradestyp = gatuadressområde` is the street name.
+- **Trafikverket NVDB** — CC0 1.0 Universal since 2017-04-03 ([trafikverket.se](https://www.trafikverket.se/e-tjanster/hamta-data-fran-trafikverket/) links `creativecommons.org/publicdomain/zero/1.0/deed.sv`; [OSM wiki](https://wiki.openstreetmap.org/wiki/Sweden/trafikverket)). Data product **Gatunamn** = "det officiellt adressbildande namnet på gatan" (municipal decision); Trafikverket is cleaning it in 2026 so it holds only names used in street addresses ([Vägnamn i NVDB](https://www.nvdb.se/sv/aktuellt/nyhetsarkiv/2026/vagnamn-i-nvdb/)). Download via Lastkajen (registration with e-mail + accept licence; Shapefile/GeoPackage), or the Datautbytesportal API. Gives street ↔ kommun (and geometry) but **no postnummer/postort**. Record count not published on the pages checked.
+- **OpenStreetMap** — ODbL 1.0. A list of street names extracted from OSM is a Derivative Database: "Where you make our data or any Derivative Database available to others, it must continue to be licensed under the ODbL" ([OSMF FAQ](https://osmfoundation.org/wiki/Licence/Licence_and_Legal_FAQ)); only *Produced Works* (maps etc.) escape share-alike. Embedding an OSM-derived JSON in an MIT repo therefore forces that data file to ODbL with attribution "© OpenStreetMap contributors" + link to openstreetmap.org/copyright, and downstream consumers inherit share-alike on the data. Not recommended.
+- **OpenAddresses** — 16 Swedish sources, all municipal (`sources/se/municipality_of_*.json`: Alingsås, Gislaved, Göteborg, Helsingborg, Höganäs, Kalmar, Kristinehamn, Malmö, Nacka, Sävsjö, Stockholm, Uppsala, Västerås, Vaxholm, Växjö, Österåker). OA does not relicense; licence is per source. Checked: Göteborg **CC0 1.0** (attribution "Göteborgs stad", shapefile, `gatunamn`), Malmö **CC0 1.0** (CSV with `ADRESSOMR`, `ADRESSPLAT`, `POSTNR`, `POSTORT`, cached 2024-11), Helsingborg **CC BY 4.0** ("Helsingborgs stad", GeoJSON with Gatunamn/Postnummer/Stad), Uppsala attribution required, licence name not stated (ArcGIS REST), Stockholm **no licence field**, data is a 2016 cache of a defunct WS. Usable for a few big cities only; no national coverage.
+- **Unique street-name counts.** SCB (Lägenhetsregistret 2019, [artikel](https://www.scb.se/hitta-statistik/artiklar/2021/pa-ringvagen-bor-det-flest/)): "Antalet gator eller vägar med en eller flera adresser är 399 175 i hela landet" — this is streets counted per place, not unique names. Most frequent: Ringvägen 205, Skogsvägen 204, Björkvägen 200, Skolgatan 196, Storgatan 180 (in 180 kommuner, 55,360 residents). A national unique-name count was not found; expect well under 399 k (probably 100–200 k) — **unverified**.
+
+## 5. Sizes
+
+| Level | Count | Source |
+|---|---|---|
+| Län | 21 | SCB |
+| Kommuner | 290 | SCB; 290 STAC items |
+| Tätorter (2023) | 2,017 | SCB WFS hits |
+| Postorter | ~1,740–1,780 | Postnummerservice 1,743; Kartanalys 1,740; GeoNames 1,780 |
+| Postnummer | ~17,000 total, ~10,500–10,840 deliverable/geographic | Postnummerservice, Kartanalys |
+| Streets with addresses (per place) | 399,175 (2019) | SCB |
+| Unique street names | not published; likely 100–200 k | unverified |
+| Address points | ~3.9 M (3,882,361 in 2022) | Lantmäteriet/SCB |
+| Lantmäteriet address GeoPackages | 324 MB zipped, 290 files | STAC |
+| GeoNames SE.zip | 96 KB (18,887 rows) | download |
+| Ortnamn national GeoPackage | 58.5 MB zipped; ~980 k names | STAC; Lantmäteriet |
+
+Embedding suggestion (size-driven, before licence): län + kommun + population (~20 KB) and tätorter with population (~100 KB) are trivially embeddable under CC0. A postnummer→postort table is ~300 KB raw. A national (street, postort, postnummer-range) table at ~400 k rows is roughly 15–25 MB JSON, ~3–5 MB gzipped — optional download pack territory; a default pack could carry the top N streets per kommun (e.g. 20 × 290 ≈ 6 k rows, <300 KB).
+
+## 6. Licence compatibility with an MIT repo
+
+| Source | Licence | Embed? | Required notice |
+|---|---|---|---|
+| SCB codes, population, tätorter | CC0 1.0 | Yes | None; optional "Källa: SCB" |
+| Lantmäteriet Topografi 50/100/250/1M (admin boundaries, tätort symbols) | CC0 | Yes | None (courtesy credit suggested) |
+| Lantmäteriet Ortnamn | CC BY 4.0 + HVD terms (no personal data, no review) | Yes, data file stays CC BY 4.0 inside the MIT repo | "Källa: Ortnamn Nedladdning, vektor, ©Lantmäteriet, bearbetad, CC BY 4.0" (product name, ©Lantmäteriet, processed-flag, CC BY 4.0) |
+| Lantmäteriet kommun-län-rike | CC BY 4.0 per STAC (CC0 per LM slide — unresolved) | Yes | Same attribution form as above unless CC0 is confirmed |
+| Lantmäteriet Belägenhetsadress | CC BY 4.0 + personal-data terms, purpose approval, EU-only storage, revocable | **Only with an approved application** whose stated purpose covers deriving and publishing an aggregated street/postnummer/postort list under CC BY 4.0. Raw points (house numbers, property links) should not be redistributed. Get written confirmation from geodatasupport@lm.se; treat as unresolved until then. | Product name, ©Lantmäteriet, "bearbetad", CC BY 4.0 |
+| GeoNames SE | CC BY 4.0 | Yes | Link/credit to www.geonames.org; note staleness |
+| Trafikverket NVDB Gatunamn | CC0 1.0 | Yes | None |
+| OpenStreetMap | ODbL 1.0 | Not practical (share-alike on the data) | "© OpenStreetMap contributors" + ODbL if ever used |
+| OpenAddresses Göteborg, Malmö | CC0 1.0 | Yes | None (Göteborg asks credit "Göteborgs stad") |
+| OpenAddresses Helsingborg | CC BY 4.0 | Yes | "Helsingborgs stad" |
+| OpenAddresses Stockholm/Uppsala | unspecified | No | — |
+| PostNord/Postnummerservice files | commercial, no redistribution | No | — |
+
+CC BY 4.0 and CC0 data files can sit in an MIT repo; the MIT licence covers the code, and a `DATA-LICENSES.md`/NOTICE lists each dataset with its licence and the attribution above. CC BY 4.0 § 2(a)(5)(B) (no technical protection measures) is irrelevant for a plain JSON file.
+
+## Could not verify
+- Whether the kommun-län-rike collection is CC0 or CC BY 4.0 (sources disagree).
+- Whether Lantmäteriet will approve an application whose purpose is publishing a derived open dataset; OSM/OpenAddresses experience suggests refusal is likely.
+- GeoNames Sweden source date; assume stale.
+- National count of unique street names; Gatunamn (NVDB) record count.
+- Exact number of active postnummer today (three sources: ~17 k total / 10.5–10.8 k deliverable).