Make great documentation goal 1.2, and renumber goals 13 to 16 as 12 to 15 wherever they are cited

.github/workflows/packages.yml

@@ -0,0 +1,162 @@
+name: Packages
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        required: true
+    secrets:
+      TAP_GITHUB_TOKEN:
+        required: true
+      PACKAGES_GPG_KEY:
+        required: true
+      PACKAGES_RSA_KEY:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The released tag to package, such as v0.1.0
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: packages
+  cancel-in-progress: false
+
+jobs:
+  packages:
+    name: Binaries, packages and image for ${{ inputs.tag }}
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+      packages: write
+    outputs:
+      latest: ${{ steps.latest.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.tag }}
+          path: src
+          fetch-depth: 0
+          persist-credentials: false
+      # Check out the packaging config at the workflow's commit, so a tag cut before the
+      # config existed can still be packaged.
+      - uses: actions/checkout@v7
+        with:
+          path: tooling
+          persist-credentials: false
+      - name: The Go version the gate tests
+        id: go
+        run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@v7
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: false
+      - name: Channels holding one version
+        id: latest
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
+          if [ "$latest" = "$TAG" ]; then
+            echo "latest=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
+            echo "OLDER_TAG=1" >> "$GITHUB_ENV"
+          fi
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: v2.18.3
+          workdir: src
+          args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      - name: The built binary and the published image print the tag
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
+          test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  dockerhub:
+    name: Docker Hub image for ${{ inputs.tag }}
+    needs: packages
+    runs-on: ubuntu-26.04
+    permissions:
+      packages: read
+    steps:
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Copy the GHCR image to Docker Hub, and check it prints the tag
+        env:
+          LATEST: ${{ needs.packages.outputs.latest }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
+          if [ "$LATEST" = true ]; then
+            tags+=(-t docker.io/larvit/fejkdata:latest)
+          fi
+          docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
+          test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
+
+  repos:
+    name: apt, dnf and apk repositories for ${{ inputs.tag }}
+    needs: packages
+    if: needs.packages.outputs.latest == 'true'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: The release's Linux packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ inputs.tag }}
+        run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
+      - uses: docker/setup-buildx-action@v4
+      - name: Install fejkdata from each repository, then export them
+        env:
+          PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+          PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          build() {
+            docker buildx build -f release-tooling/package-repos.Dockerfile \
+              --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
+              --build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
+          }
+          build --target checks --output type=local,dest=checks
+          build --target site --output type=local,dest=site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - id: deploy
+        uses: actions/deploy-pages@v5

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * *'
 
 permissions:
   contents: read
@@ -11,9 +14,10 @@ permissions:
 jobs:
   test:
     name: vet + fmt + tests
-    runs-on: ubuntu-24.04
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -26,11 +30,16 @@ jobs:
           else
             base='${{ github.event.before }}'
           fi
-          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
             echo "code=false" >> "$GITHUB_OUTPUT"
           else
             echo "code=true" >> "$GITHUB_OUTPUT"
           fi
+          if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
+            echo "packaging=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "packaging=true" >> "$GITHUB_OUTPUT"
+          fi
           echo "base=$base" >> "$GITHUB_OUTPUT"
       - name: CHANGELOG entry for data changes
         run: |
@@ -48,22 +57,84 @@ jobs:
       # failure is version-specific.
       - name: Lowest supported Go
         if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
-        run: docker build --build-arg GO_VERSION=1.22.12 .
+        run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
+      - name: Package repositories, signed with throwaway keys
+        if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
+        run: |
+          docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
+          PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
+            docker build -f release-tooling/package-repos.Dockerfile --target checks \
+            --secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
 
-  release:
-    name: Gitea release from CHANGELOG.md
-    needs: test
-    if: github.event_name == 'push'
-    runs-on: ubuntu-24.04
+  vulnerabilities:
+    name: Known vulnerabilities in this Go version and the required modules
+    runs-on: ubuntu-26.04
+    timeout-minutes: 10
     permissions:
+      actions: write
       contents: write
+      pull-requests: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - run: python3 release-tooling/publish_release.py
+      - name: Fetch govulncheck's report from vuln.go.dev
+        run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
+      - name: File each one in todo.md
+        id: file
+        run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
+      - name: Propose the filed items
+        if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
         env:
-          GITEA_API_URL: ${{ github.api_url }}
-          GITEA_REPOSITORY: ${{ github.repository }}
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
+            exit 0
+          fi
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git switch -C vulnerabilities
+          git commit -qam "File the known vulnerabilities govulncheck reports"
+          git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
+          gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
+            --body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
+          # A pull request the workflow's token opens starts no workflow.
+          gh workflow run test.yml --ref vulnerabilities
+
+  release:
+    name: Release from CHANGELOG.md
+    needs: [test, vulnerabilities]
+    if: github.event_name == 'push'
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    outputs:
+      tag: ${{ steps.publish.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - id: publish
+        run: python3 release-tooling/publish_release.py
+        env:
+          FORGE_API_URL: ${{ github.api_url }}
+          FORGE_REPOSITORY: ${{ github.repository }}
+          FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SHA: ${{ github.sha }}
+
+  packages:
+    needs: release
+    if: needs.release.outputs.tag != ''
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/packages.yml
+    with:
+      tag: ${{ needs.release.outputs.tag }}
+    secrets:
+      TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
+      PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
+      PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

AGENTS.md

@@ -1,11 +1,137 @@
 # Rules
 
-- Go runs only through `docker compose run --rm <test|vet|fmt|build>`; the merge gate is `docker build .` plus CI's changelog check.
+- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
 - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
 - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
 - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
+- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
+  only, `gh pr merge --rebase` where the branch is already on top of main.
+- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
+  its status once when a PR is marked ready, answer a review if one is there, and never
+  wait for one.
+- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
+  nothing else merges — no feature, no category, no data — bar a security fix, a
+  dependency bump, a repair of behaviour the README documents, the infrastructure
+  the round itself runs on, and a chunk the comprehension-floor decision lists as
+  shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
+  findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
+  again, until the score passes. At or above 7.0
+  every pull request scores with the four-seat run and answers it in one run. No merge
+  lowers the last score its own panel recorded: the nine-seat round and the four-seat
+  run are two ratchets, never compared with each other. A lower score blocks the merge
+  even where it may be panel noise: fix what the seats name until the score rises. A PR
+  that only records a run's scores and files what its seats name merges even when a
+  score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
 - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
-- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
-- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
+- Prose naming a source states what that source states: read the register's own field
+  before paraphrasing it, and name the register you read before claiming none
+  publishes a spelling.
 - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
-- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`eachToken`, `calc.factor`, `walkPath`) sit at 13–14 and stay whole; anything else that reaches 14 is decomposed.
+- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
+
+# Decisions
+
+In [docs/decisions.md](docs/decisions.md):
+
+- GitHub is canonical, and the module path names it
+- The Gitea copy stays, as a pull mirror of GitHub
+- The vocabulary sits below `doc.go`'s package clause, not in the package doc
+- Options and fields share one namespace
+- `{a|b}` stays beside nested choices
+- Flags follow getopt_long
+- A struct tag is a template by its shape
+- The CLI renders exactly the template on its stdin, and adds nothing
+- The CLI reads only the library's public API
+- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
+- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
+- `FakeTemplate` and `NewTemplate` both stay
+- The shipped data is embedded, not discovered
+- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
+- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
+- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
+- Every `{…}` draws afresh, and only a name keeps a pick
+- Reference sigils follow the filesystem
+- A change to what exists is a major; a minor only adds
+- An error is a contract by what it names, not its bytes
+- A load reports every mistake at once
+- Formats are checked after all data is loaded, so their errors name the category
+- Raising the lowest supported Go is a major
+- The format check runs on the latest Go only
+- The changelog heading is the one spelling of a release; CI cuts the tag
+- A `--data-path` override rebinds every reference to the category it replaces
+- The repeat cap bounds renders, not bytes
+- 64-bit targets only
+- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
+- Samples say what they emit, transforms what they do
+- A record is a template seen as columns; a Go struct is the one second schema
+- A struct's records follow Go's field access, and compile on first use
+- A category may reference itself, and a record's fences run at load
+- A record's column set is fixed before the first draw
+- Null is a `null` item, not a rate
+- A typed column holds one value, not composed text
+- A column that only reads one reference or name is the column it reads
+- A typed column's calc is refused unless proven
+- `Column` carries text, not a Go value
+- A unit takes the stem of what it is, and a file the stem of the units it holds
+- One name, one meaning
+- The performance gate asserts allocations, not wall-clock time
+- Rows live in a TSV, the shape in JSON
+- A selector is bracketed, and a dot inside it is literal
+- `parent` names the link column and the table alike
+- After a row, a path names a column or a linked table
+- A selected row is fixed, not drawn
+- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
+- A table selects by one key column, a code, never a free-form name
+- A parent row with no child row is a load error
+- A table is a record of string columns
+- The key map is built at load, the rest on first draw
+- Two categories may name one TSV
+- A cell may read a reference, and each row its own
+- A path is walked once without drawing before it is walked for real
+- A country's postal codes and streets are siblings under its locality
+- A locale package imports its country's geo package, and its `Modules()` returns both
+- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
+- A locale's `address` binds its country's record once and prints it whole
+- A postort's kommun comes from its name, its tätort or its codes, never from distance
+- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
+- `--list` stays a plain list of paths
+- A layout is always quoted
+- A title is a table under `sex`
+- A table owns the spelling of a selector on it
+- No builtin reads the clock, so a date is bounded by days, never by an age
+- `misc` is what every locale shares
+- A register's canonical spelling loses to the one its domain writes
+- `misc.territory` is the spine, and a `misc` table naming a territory links to it
+- A table whose register publishes no frequency draws evenly
+- `misc.timezone` weighs a zone by the people living in it
+- `misc.car` is one flat table, not a make linked to its models
+- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
+- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
+- `misc.tld` keys carry the leading dot, where other tables key on a bare code
+- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
+- `misc.territory` carries a currency code, it does not link to `misc.currency`
+- An extension may name two media types
+- The Swedish ids draw Skatteverket's test series
+- The US given names come from a mirror of the SSA file
+- `List` advertises direct descents only
+- A path draws through its compiled steps
+- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
+- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
+- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
+- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
+- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
+- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
+- A country with no open address register covering it whole ships its places and postal codes without streets
+- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
+- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
+- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
+
+# Scoring run
+
+The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
+
+- `{language}`: Go
+- `{kind}`: a Go library and CLI that renders fake data from JSON templates
+- `{domain}`: fake test-data generation and template languages
+- `{domain docs}`: the documentation of any fake-data library, such as Faker
+- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another

CHANGELOG.md

@@ -8,27 +8,99 @@ replacement, and each removed path, column or flag.
 
 ### Added
 
-- First release: the CLI, the library and the shipped data set.
-- Table categories: a category JSON naming a `rows` TSV beside it, with the options
-  `key`, `name`, `weight` and `parent`; a path selects a row by key or name,
-  `misc.country[SE]`, and descends to a linked table by name; linked tables draw
-  consistently within one render and draw group. `rows` is an option, so no
-  template may carry a field of that name. Refused at `New`: a `name` without a
-  `key`, a name spelling another row's key, a table named like a column of any
-  table above it, a table whose format or cell references its own family, and,
-  within one render and draw group, a path drawing a table another path selects a
-  row of, or two paths pinning different rows of one table.
-- `New` refuses a root choice of templates sharing one format and one set of string
-  fields, naming the rows TSV to write instead.
-- `misc.country`, `misc.currency`, `misc.language`, `misc.httpstatus` and
-  `misc.mimetype` are tables. `misc.country` is every ISO 3166 country that has a
-  capital, a currency and a TLD, with the columns `calling-code`, `capital`,
-  `currency`, `flag`, `languages`, `numeric` and `tld` added; `misc.currency` the
-  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
-  added, and its symbols from CLDR. `DATA-LICENSES.md` lists each source.
-- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
+- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
+  dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
+  the CLI installs without Go. 0.1.0 ships them too; the README's
+  [Install](https://github.com/larvit/fejkdata#install) lists each command.
+
+## [0.1.0] - 2026-10-13
+
+The first release: a Go library and a CLI that render locale-aware fake data from JSON
+templates, and the data they ship with. The
+[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
+Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
+library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
+and any release may change what a seed draws, so pin one exact version
+([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
+
+### Added
+
+- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
+  `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
+  repeats a run's output while the version and data stay the same, and without it each
+  run takes a new seed; `-n` renders the template that many times, and `--list`
+  prints the paths the data offers.
+  `--data-path` layers a directory of your own data over the shipped set, and
+  `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
+  a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
+  error such as data that fails to load, and `2` on misuse.
+- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
+  `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
+  from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
+  without data, it fails and names `WithDataFS`. The same seed repeats the draws
+  while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
+  and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
+  `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
+  `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
+  `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
+  and `IsTemplate` tells an inline template from a path. Data that fails to load gives
+  an error matching `ErrLoad`, and a record with no fields gives one matching
+  `ErrNoColumns`.
+- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
+  `cli.Main(modules...)` runs it over the modules given, so
+  `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
+  That CLI names itself by the file it runs from, and its `--version` prints its own
+  module's version before fejkdata's.
+- The template language
+  ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
+  Every value in a data file is a string, a choice, a template, a table, a number or a
+  boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
+  root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
+  or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
+  pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
+  digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
+  `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
+  transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
+  grammar, could mean two things or cannot render a valid value.
+- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
+  a template seen as columns. A column is a string unless it declares a `datatype`,
+  `integer`, `number` or `boolean`, and a `null` item draws it as null.
+- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
+  category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
+  `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
+  linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
+  `{/misc.timezone[Europe/Oslo]..territory.name}`.
+- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
+  a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
+  manifest at a source's root may carry a `namespace`, the dot path the source mounts
+  under, and an `index`, with which each category loads on the first call reaching it.
+- The shipped data, in Go packages that each export `FS` and `Modules()`:
+  `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
+  `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
+  `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
+  [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
+  names each source.
+  - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
+    `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
+    the Census Bureau. A person's first name and sex agree, and so does an `en_US`
+    person's title; `person.prefix` is null where a person has no title.
+    `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
+    `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
+    columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
+    assign.
+  - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
+    HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
+    symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
+    zones weighted by the people living in each, user agents and cars, all as tables;
+    and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
+    ObjectIds and UUIDs.
+  - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
     `locality`, `postal-code` and `street`, weighted by population and address counts
-  and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
-  `address` record over one consistent draw of them. `sv_SE.address` and
-  `en_US.address` read those records, so `en_US.address.street` no longer carries
-  `name` and `suffix`, and a locale folder loads only beside `geo`.
+    and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
+    and an `address` record that draws a street and a postal code inside one
+    locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
+    (postal town) the import can place ships with all its streets. A US place ships
+    only if it is incorporated and holds most of some postal code's land. Of those
+    places, every state capital but Honolulu ships, as does every place of 100,000
+    people or more and a tenth of all of them drawn by population. Each US place
+    carries a quarter of its streets, at least ten.

DATA-LICENSES.md

@@ -1,24 +1,31 @@
 # Data licenses
 
 Every shipped dataset, its source, its licence and the attribution it asks for. A
-`data-import/` script rebuilds each sourced table; a curated one is hand-written.
-The scripts run through `docker compose run --rm data-import data-import/<name>.py`
-and cache their downloads under `data-import/cache/`; `geo-se.py` needs a
-Trafikverket API key, free at [data.trafikverket.se](https://data.trafikverket.se/),
-in `TRAFIKVERKET_API_KEY` or a `--key-file`; `geo-us.py` fetches two TIGER/Line
-files per county it ships, a few hundred megabytes in all.
+`data-import/` script rebuilds each sourced table, run as the README's
+[Development](README.md#development) section says; a curated one is hand-written.
 
 | Table | Source | Licence | Attribution | Rebuild |
 |-------|--------|---------|-------------|---------|
 | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
 | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
 | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
-| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
+| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
 | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
-| `misc/country.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/country.py` |
+| `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
+| `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
+| `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
+| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
 | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
-| `misc/httpstatus.tsv` | curated (IANA HTTP status codes are facts) | — | — | — |
-| `misc/language.tsv` | curated (ISO 639-1 codes are facts) | — | — | — |
-| `misc/mimetype.tsv` | curated (IANA media types are facts) | — | — | — |
+| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
+| `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
+| `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` |
+| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from the [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `<syslog.h>` severity macros | facts | none required | `data-import/loglevel.py` |
+| `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` |
+| `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` |
+| `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` |
+| `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` |
+| `misc/timezone.tsv` | [IANA tzdb](https://www.iana.org/time-zones) 2026d `zone.tab` and the standard offset of each zone; weights from [GeoNames](https://www.geonames.org/) `cities15000` populations | [public domain](https://data.iana.org/time-zones/tzdb/LICENSE); [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | "Populations from GeoNames, www.geonames.org" | `data-import/timezone.py` |
+| `misc/tld.tsv` | [IANA Root Zone Database](https://www.iana.org/domains/root/db), checked against the root zone's [TLD list](https://data.iana.org/TLD/tlds-alpha-by-domain.txt) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/tld.py` |
+| `misc/useragent.tsv` | [top-user-agents](https://github.com/microlinkhq/top-user-agents) desktop and mobile lists | [MIT](https://github.com/microlinkhq/top-user-agents/blob/master/LICENSE.md) | "Copyright © 2020 Kiko Beats" | `data-import/useragent.py` |
 
-Every other category is hand-written JSON under [`data/`](data), MIT like the code.
+Every other category is hand-written under [`data/`](data), MIT like the code.

Dockerfile

@@ -1,19 +1,43 @@
-# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
-# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
-# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
-# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
-ARG GO_VERSION=1.26.4
-FROM golang:${GO_VERSION}
+ARG GO_VERSION=1.27.2
+FROM golang:${GO_VERSION} AS portable
 
 WORKDIR /app
 
-# Module layer cached separately from source.
 COPY go.mod ./
 RUN go mod download
 
 COPY . .
 RUN go vet ./... && \
 	go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
-	{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
-	  { echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
 	go test -race ./...
+
+FROM golang:${GO_VERSION} AS vulnerabilities
+WORKDIR /app
+COPY . .
+RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
+
+FROM scratch AS vulnerability-report
+COPY --from=vulnerabilities /report.json /
+
+FROM python:3.14.8-slim AS release-tooling
+COPY release-tooling/ /release-tooling/
+RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
+
+FROM python:3.14.8-slim AS vulnerability-filing
+COPY vulnerability-filing/ /vulnerability-filing/
+# The filer's test files an item into the plan itself.
+COPY todo.md /todo.md
+RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
+
+FROM goreleaser/goreleaser:v2.18.3 AS packaging
+WORKDIR /check
+COPY release-tooling/goreleaser.yaml ./
+RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
+
+FROM portable
+# BuildKit skips a stage nothing copies from.
+COPY --from=release-tooling /tested /tmp/release-tooling-tested
+COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
+COPY --from=packaging /tested /tmp/packaging-tested
+RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
+	{ echo "unformatted files:"; echo "$unformatted"; exit 1; }

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2026 Timewave
+Copyright (c) 2026 Larvit
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

arm.go

@@ -0,0 +1,117 @@
+package fejkdata
+
+import (
+	"fmt"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
+)
+
+// arm is one alternative of a {a|b} token or one operand, split into the head,
+// whose node `template.startOf` finds, and the tail of a dotted path into it.
+type arm struct {
+	spelling string // as written, for messages
+	head     string
+	tail     []string
+	levels   []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
+	steps    []pathStep
+	leaves   []node // every node the path may land on, one per variant it passes
+	kind     armKind
+	named    *nameBinding // namedRead: the binding of the name it reads through
+}
+
+// armKind is how expand reads an arm, fixed at compile.
+type armKind uint8
+
+const (
+	freshRead armKind = iota // drawn afresh at every read
+	namedRead                // read through a name, kept in its pick
+)
+
+// splitArm splits one name into head and tail. refs maps a reference to what
+// resolveRefs resolved it to; before that, a reference is whole.
+func splitArm(name string, refs map[string]resolvedRef) arm {
+	if grammar.IsRef(name) {
+		if r, resolved := refs[name]; resolved {
+			return pathArm(name, r.head, r.tail)
+		}
+		return pathArm(name, name, nil)
+	}
+	segs, err := grammar.SplitPath(name)
+	if err != nil || len(segs) == 1 {
+		return pathArm(name, name, nil)
+	}
+	return pathArm(name, segs[0], segs[1:])
+}
+
+func pathArm(name, head string, segs []string) arm {
+	return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
+}
+
+// key is the key of the leaf a lands on, the one key every way of writing this read shares.
+func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
+
+func checkSegments(a arm) error {
+	if len(a.tail) == 0 {
+		return nil
+	}
+	return grammar.CheckSegments(append([]string{a.head}, a.tail...))
+}
+
+// op is one compiled unit of a format string: a literal run, a name read,
+// or a builtin already prepared with its args.
+type op struct {
+	grammar.Token
+	arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
+	call builtinfunc.Call
+	// operands are the fields the builtin reads, in the order its operands func
+	// fixed; expand reads them before the call. nil for a builtin that reads none.
+	operands []arm
+}
+
+func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+	arms := make([]arm, len(names))
+	for i, name := range names {
+		a, err := t.compileArm(name, targets)
+		if err != nil {
+			return nil, err
+		}
+		arms[i] = a
+	}
+	return arms, nil
+}
+
+// compileArm compiles one read into a path: from the head it names, or, for a read through a
+// name, from what that name binds.
+func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
+	a := splitArm(name, t.refs.byName)
+	if !t.isName(a.head) {
+		start := t.startOf(a.head)
+		if start == nil {
+			panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
+		}
+		w := compilePath(start, a.tail)
+		a.steps, a.leaves = w.steps, w.leaves
+		return a, nil
+	}
+	b := t.nameScope.lookup(a.head)
+	if grammar.HasSelector(a.tail) {
+		if b.bindsField() {
+			return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
+		}
+		return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
+	}
+	target := targets[b]
+	full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
+	if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
+		return a, err
+	}
+	w := compilePath(target.start, full)
+	a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
+	a.levels = levelKeys(full, 0)
+	return a, nil
+}

bench_test.go

@@ -6,17 +6,17 @@ import (
 	"testing"
 )
 
-func benchGenerator(b *testing.B, dir string) *Generator {
+func benchGenerator(b *testing.B, data Option) *Generator {
 	b.Helper()
-	f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
+	f, err := New(data, WithSeed(1))
 	if err != nil {
 		b.Fatal(err)
 	}
 	return f
 }
 
-func benchPath(b *testing.B, dir, path string) {
-	f := benchGenerator(b, dir)
+func benchPath(b *testing.B, data Option, path string) {
+	f := benchGenerator(b, data)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
 	}
 }
 
-func BenchmarkPerson(b *testing.B)     { benchPath(b, "data", "sv_SE.person") }
-func BenchmarkAddress(b *testing.B)    { benchPath(b, "data", "sv_SE.address") }
-func BenchmarkWord(b *testing.B)       { benchPath(b, "data", "sv_SE.word") }
-func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data", "misc.creditcard") }
-func BenchmarkSSN(b *testing.B)        { benchPath(b, "data", "sv_SE.ssn") }
-func BenchmarkUUIDv7(b *testing.B)     { benchPath(b, "data", "misc.uuid") }
+func BenchmarkPerson(b *testing.B)       { benchPath(b, withShipped(), "sv_SE.person") }
+func BenchmarkAddress(b *testing.B)      { benchPath(b, withShipped(), "sv_SE.address") }
+func BenchmarkWord(b *testing.B)         { benchPath(b, withShipped(), "sv_SE.word") }
+func BenchmarkCreditcard(b *testing.B)   { benchPath(b, withShipped(), "misc.creditcard") }
+func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
+func BenchmarkUUIDv7(b *testing.B)       { benchPath(b, withShipped(), "misc.uuid") }
 
 func tmpData(b *testing.B, name, body string) string {
 	b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
 
 func BenchmarkCalc(b *testing.B) {
 	dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
-	benchPath(b, dir, "inv")
+	benchPath(b, WithDataPath(dir), "inv")
 }
 
 func BenchmarkLongLiteral(b *testing.B) {
 	dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "sql")
+	benchPath(b, WithDataPath(dir), "sql")
 }
 
 func BenchmarkRepeat(b *testing.B) {
 	dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
-	benchPath(b, dir, "many")
+	benchPath(b, WithDataPath(dir), "many")
 }
 
-// BenchmarkBound measures a format that binds a head and reads two paths from it —
-// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
-// two independent fields.
-func BenchmarkBound(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
-	benchPath(b, dir, "addr")
+// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
+// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
+// independent fields.
+func BenchmarkNamed(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-func BenchmarkUnbound(b *testing.B) {
+func BenchmarkUnnamed(b *testing.B) {
 	dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
-	benchPath(b, dir, "addr")
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
-// depth question is about.
-func BenchmarkBoundDeep(b *testing.B) {
-	dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
-	benchPath(b, dir, "addr")
+// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
+func BenchmarkNamedDeep(b *testing.B) {
+	dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
+	benchPath(b, WithDataPath(dir), "addr")
 }
 
-// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
-// expansion holds outgrow a single bucket.
-func BenchmarkBoundWide(b *testing.B) {
-	dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
+// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
+// outgrow a single bucket.
+func BenchmarkNamedWide(b *testing.B) {
+	dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
 		{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
 		{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
-	benchPath(b, dir, "row")
+	benchPath(b, WithDataPath(dir), "row")
+}
+
+func BenchmarkFirstFake(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		f, err := New(withShipped(), WithSeed(1))
+		if err != nil {
+			b.Fatal(err)
+		}
+		if _, err := f.Fake("sv_SE.address"); err != nil {
+			b.Fatal(err)
+		}
+	}
 }
 
 func BenchmarkNew(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if _, err := New(WithSeed(1)); err != nil {
+		if _, err := New(withShipped(), WithSeed(1)); err != nil {
 			b.Fatal(err)
 		}
 	}

builtins.go

@@ -1,468 +0,0 @@
-package fejkdata
-
-import (
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"unicode"
-)
-
-// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
-// and maxDecimals float/calc decimal places, so a fat-fingered or overflowing
-// argument fails at New instead of trying to allocate gigabytes — or panicking —
-// at render.
-const (
-	maxLen      = 1 << 20
-	maxDecimals = 1024
-)
-
-// builtins is the registry of {name(args)} functions. Derivations read the digits
-// emitted so far in the current expansion (place them after their payload);
-// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
-// from the rng, not the wall clock, so seeded output stays reproducible.
-var builtins = map[string]builtin{
-	"luhn":   {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
-	"mod11":  {arity: 0, prep: derive(mod11Check)},
-	"ean":    {arity: 0, prep: derive(eanCheck)},
-	"uuid":   {arity: 0, prep: sample(uuidV7)},
-	"ulid":   {arity: 0, prep: sample(ulid)},
-	"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)},
-	"hex":    {arity: 1, check: posIntArg, prep: chars(hexDigits)},
-	"digits": {arity: 1, check: posIntArg, prep: chars("0123456789"), number: func(token string, a []string) proven {
-		return printing(token, DataTypeString, bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
-	}},
-	"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
-	"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
-	"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
-		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string {
-			return base64.StdEncoding.EncodeToString(randBytes(s, n))
-		}
-	}},
-	"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
-		lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
-		return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
-	}, number: func(token string, a []string) proven {
-		return printing(token, DataTypeInteger, bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
-	}},
-	"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
-		lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
-		return func(s *session, _ string, _ []string) string {
-			return formatFloat(lo+s.Float64()*(hi-lo), dp)
-		}
-	}, number: func(token string, a []string) proven {
-		return printedNumber(token, bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
-	}},
-	"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn {
-		cc := a[0]
-		return func(s *session, _ string, _ []string) string { return iban(s, cc) }
-	}},
-	"calc":      {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands},
-	"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand},
-	"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
-	"ascii":     {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
-	// seq is the one stateful builtin: a per-session counter from 1, advancing on
-	// each call. An optional name selects an independent counter; no name uses the
-	// default one. Deterministic by construction, so seeded output stays stable.
-	"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
-		key := ""
-		if len(a) == 1 {
-			key = a[0]
-		}
-		return func(s *session, _ string, _ []string) string {
-			return strconv.FormatUint(s.next(key), 10)
-		}
-	}, number: func(token string, _ []string) proven {
-		return printing(token, DataTypeInteger, bounded(1, math.MaxInt64, true))
-	}},
-}
-
-// derive and sample are the two argument-free builtin shapes: a derivation reads
-// the output emitted so far, a sample reads only the rng. chars is the shape of a
-// sample of n characters drawn from an alphabet.
-func derive(f func(emitted string) string) func([]string) callFn {
-	return func([]string) callFn {
-		return func(_ *session, emitted string, _ []string) string { return f(emitted) }
-	}
-}
-
-func sample(f func(rng) string) func([]string) callFn {
-	return func([]string) callFn {
-		return func(s *session, _ string, _ []string) string { return f(s) }
-	}
-}
-
-func chars(alphabet string) func([]string) callFn {
-	return func(a []string) callFn {
-		n := atoi(a[0])
-		return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) }
-	}
-}
-
-const hexDigits = "0123456789abcdef"
-
-// formatFloat prints v to dp decimals, -1 for the shortest form, and a zero unsigned.
-func formatFloat(v float64, dp int) string {
-	s := strconv.FormatFloat(v, 'f', dp, 64)
-	if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
-		return s[1:]
-	}
-	return s
-}
-
-// transforms are the builtins that rewrite one operand's value; they nest, so
-// {lowercase(ascii(x))} folds then lowers.
-var transforms = map[string]func(string) string{
-	"ascii":     asciiFold,
-	"lowercase": strings.ToLower,
-	"uppercase": strings.ToUpper,
-}
-
-// unwrapTransform peels nested transform calls off an operand arg, returning the
-// field it finally names and the transforms to apply, innermost last.
-func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
-	for {
-		name, args, isCall := funcCall(arg)
-		if !isCall {
-			return arg, chain, nil
-		}
-		fn, isTransform := transforms[name]
-		if !isTransform {
-			return "", nil, fmt.Errorf("%s(%s) is not a transform, so it cannot be an operand", name, strings.Join(args, ","))
-		}
-		if len(args) != 1 {
-			return "", nil, fmt.Errorf("%s takes 1 arg, got %d", name, len(args))
-		}
-		chain = append(chain, fn)
-		arg = args[0]
-	}
-}
-
-func transformArg(fields map[string]node, a []string) error {
-	leaf, _, err := unwrapTransform(a[0])
-	if err != nil {
-		return err
-	}
-	if isRef(leaf) {
-		_, _, err := refShape(leaf)
-		return err
-	}
-	return checkArm(leaf, fields, false)
-}
-
-func transformOperand(a []string) []string {
-	leaf, _, err := unwrapTransform(a[0])
-	if err != nil {
-		return nil
-	}
-	return []string{leaf}
-}
-
-func transformPrep(outer func(string) string) func([]string) callFn {
-	return func(a []string) callFn {
-		_, chain, err := unwrapTransform(a[0])
-		if err != nil {
-			panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err))
-		}
-		return func(_ *session, _ string, operands []string) string {
-			v := operands[0]
-			for i := len(chain) - 1; i >= 0; i-- {
-				v = chain[i](v)
-			}
-			return outer(v)
-		}
-	}
-}
-
-// asciiFolds maps the Latin letters with diacritics or ligatures to ASCII.
-var asciiFolds = map[rune]string{
-	'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", 'Å': "A", 'Æ': "AE", 'Ç': "C",
-	'È': "E", 'É': "E", 'Ê': "E", 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I",
-	'Ð': "D", 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", 'Ø': "O",
-	'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", 'Þ': "Th", 'ß': "ss", 'Œ': "OE",
-	'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c",
-	'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i",
-	'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o",
-	'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'þ': "th", 'ÿ': "y", 'œ': "oe",
-}
-
-// asciiFold rewrites s to ASCII: folded Latin letters stay, any other non-ASCII
-// rune is dropped.
-func asciiFold(s string) string {
-	var b strings.Builder
-	for _, r := range s {
-		if r <= unicode.MaxASCII {
-			b.WriteRune(r)
-		} else {
-			b.WriteString(asciiFolds[r])
-		}
-	}
-	return b.String()
-}
-
-// atoi parses an arg a builtin's check already validated. It panics rather than
-// returning zero, so a check that stops covering its own args is a stack trace and
-// not a silently wrong length, range or decimal count.
-func atoi(s string) int {
-	n, err := strconv.Atoi(s)
-	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
-	}
-	return n
-}
-
-// atof is atoi for a float arg, and reports an unvalidated one the same way.
-func atof(s string) float64 {
-	f, err := strconv.ParseFloat(s, 64)
-	if err != nil {
-		panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
-	}
-	return f
-}
-
-func randBytes(r rng, n int) []byte {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = byte(r.IntN(256))
-	}
-	return b
-}
-
-func randChars(r rng, n int, alphabet string) string {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = alphabet[r.IntN(len(alphabet))]
-	}
-	return string(b)
-}
-
-// plainInt parses an integer arg written the one way: no sign, no leading zero.
-func plainInt(s string) (int, error) {
-	n, err := strconv.Atoi(s)
-	if errors.Is(err, strconv.ErrRange) {
-		return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
-	}
-	if err != nil {
-		return 0, fmt.Errorf("%q is not an integer", s)
-	}
-	if strconv.Itoa(n) != s {
-		return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n)
-	}
-	return n, nil
-}
-
-func posIntArg(_ map[string]node, a []string) error {
-	n, err := plainInt(a[0])
-	if errors.Is(err, strconv.ErrRange) {
-		return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
-	}
-	if err != nil {
-		return fmt.Errorf("count %w", err)
-	}
-	if n < 1 {
-		return fmt.Errorf("count %q must be positive", a[0])
-	}
-	if n > maxLen {
-		return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
-	}
-	return nil
-}
-
-func intRangeArgs(_ map[string]node, a []string) error {
-	lo, err := plainInt(a[0])
-	if err != nil {
-		return fmt.Errorf("int(min,max): min %w", err)
-	}
-	hi, err := plainInt(a[1])
-	if err != nil {
-		return fmt.Errorf("int(min,max): max %w", err)
-	}
-	if lo > hi {
-		return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
-	}
-	if lo == hi {
-		return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo)
-	}
-	if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
-		return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
-	}
-	return nil
-}
-
-func floatArgs(_ map[string]node, a []string) error {
-	lo, e1 := strconv.ParseFloat(a[0], 64)
-	hi, e2 := strconv.ParseFloat(a[1], 64)
-	if e1 != nil || e2 != nil {
-		return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
-	}
-	dp, err := plainInt(a[2])
-	if err != nil {
-		return fmt.Errorf("float(min,max,dp): decimals %w", err)
-	}
-	if math.IsNaN(lo) || math.IsNaN(hi) || math.IsInf(lo, 0) || math.IsInf(hi, 0) {
-		return fmt.Errorf("float(min,max,dp) needs finite bounds, got %q,%q", a[0], a[1])
-	}
-	if lo > hi {
-		return fmt.Errorf("float(min,max,dp): min %v > max %v", lo, hi)
-	}
-	if dp < 0 || dp > maxDecimals {
-		return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
-	}
-	if lo == hi {
-		return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64))
-	}
-	if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
-		return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
-	}
-	return nil
-}
-
-func seqArg(_ map[string]node, a []string) error {
-	if len(a) > 1 {
-		return fmt.Errorf("seq takes at most one name, got %d args", len(a))
-	}
-	if len(a) == 1 && a[0] == "" {
-		return fmt.Errorf("seq name must not be empty")
-	}
-	return nil
-}
-
-// nanoidAlphabet is the 64-char URL-safe set Nano IDs use.
-const nanoidAlphabet = "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-// uuidV7 builds an RFC 9562 v7 UUID. The 48-bit timestamp field is drawn from
-// the rng (not the clock) to stay reproducible, then the version (7) and variant
-// (10) bits are forced; the rest is random.
-func uuidV7(r rng) string {
-	b := randBytes(r, 16)
-	b[6] = b[6]&0x0f | 0x70
-	b[8] = b[8]&0x3f | 0x80
-	var sb strings.Builder
-	for i, x := range b {
-		if i == 4 || i == 6 || i == 8 || i == 10 {
-			sb.WriteByte('-')
-		}
-		sb.WriteByte(hexDigits[x>>4])
-		sb.WriteByte(hexDigits[x&0x0f])
-	}
-	return sb.String()
-}
-
-// crockford is the ULID/Crockford base32 alphabet (no I, L, O, U).
-const crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
-
-// ulid builds a 26-char ULID: 128 random bits (timestamp from the rng) encoded
-// big-endian into base32, the 130-bit stream left-padded with two zero bits.
-func ulid(r rng) string {
-	b := randBytes(r, 16)
-	out := make([]byte, 26)
-	for i := range out {
-		v := 0
-		for j := 0; j < 5; j++ { // 5 bits per char; the two leading pad bits are zero
-			real := 5*i + j - 2
-			bit := 0
-			if real >= 0 {
-				bit = int(b[real/8]>>(7-uint(real%8))) & 1
-			}
-			v = v<<1 | bit
-		}
-		out[i] = crockford[v]
-	}
-	return string(out)
-}
-
-// luhnCheck returns the Luhn check digit (0-9) over the digits of s; non-digit
-// runes are skipped. Doubling runs from the rightmost digit, so the result is
-// correct whatever the payload length.
-func luhnCheck(s string) int {
-	sum, double := 0, true
-	for i := len(s) - 1; i >= 0; i-- {
-		c := s[i]
-		if c < '0' || c > '9' {
-			continue
-		}
-		d := int(c - '0')
-		if double {
-			if d *= 2; d > 9 {
-				d -= 9
-			}
-		}
-		double = !double
-		sum += d
-	}
-	return (10 - sum%10) % 10
-}
-
-// mod11Check returns the weighted mod-11 check character over the digits of s
-// (weights 2..7 cycling from the right). A would-be value of 10 emits 'X', as in
-// ISBN-10 / ISO 7064; non-digits are skipped.
-func mod11Check(s string) string {
-	sum, w := 0, 2
-	for i := len(s) - 1; i >= 0; i-- {
-		c := s[i]
-		if c < '0' || c > '9' {
-			continue
-		}
-		sum += int(c-'0') * w
-		if w++; w > 7 {
-			w = 2
-		}
-	}
-	if chk := (11 - sum%11) % 11; chk != 10 {
-		return string(rune('0' + chk))
-	}
-	return "X"
-}
-
-// eanCheck returns the EAN-13 / UPC-A / ISBN-13 / GTIN check digit over the
-// digits of s: weights 3 and 1 alternating from the rightmost digit, mod 10.
-func eanCheck(s string) string {
-	sum, w := 0, 3
-	for i := len(s) - 1; i >= 0; i-- {
-		c := s[i]
-		if c < '0' || c > '9' {
-			continue
-		}
-		sum += int(c-'0') * w
-		w = 4 - w // 3 <-> 1
-	}
-	return string(rune('0' + (10-sum%10)%10))
-}
-
-// ibanLen maps a supported country code to the full IBAN length. The check digits
-// sit between the country code and the BBAN, so — unlike luhn/ean — iban can't be
-// a left-to-right derivation; it generates the whole value instead.
-var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
-
-func ibanArg(_ map[string]node, a []string) error {
-	if _, ok := ibanLen[a[0]]; !ok {
-		return fmt.Errorf("iban(%q): unsupported country code", a[0])
-	}
-	return nil
-}
-
-// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right
-// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
-// the result passes length and checksum validation, which is what fake data needs.
-func iban(r rng, cc string) string {
-	bban := make([]byte, ibanLen[cc]-4)
-	for i := range bban {
-		bban[i] = byte('0' + r.IntN(10))
-	}
-	rem := 0
-	feed := func(d int) { rem = (rem*10 + d) % 97 }
-	for _, c := range bban {
-		feed(int(c - '0'))
-	}
-	for i := 0; i < len(cc); i++ { // letters A-Z -> 10..35, fed as two digits
-		v := int(cc[i]-'A') + 10
-		feed(v / 10)
-		feed(v % 10)
-	}
-	feed(0)
-	feed(0)
-	return fmt.Sprintf("%s%02d%s", cc, 98-rem, bban)
-}

builtins_test.go

@@ -2,9 +2,14 @@ package fejkdata
 
 import (
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
 func TestBuiltinIDGenerators(t *testing.T) {
@@ -41,6 +46,7 @@ func TestBuiltinSamplesReproducible(t *testing.T) {
 		`"{uuid()}"`, `"{ulid()}"`,
 		`"{nanoid(12)}"`, `"{int(1,1000000)}"`,
 		`"{float(0,1,6)}"`, `"{base64(12)}"`, `"{iban(SE)}"`,
+		`"{date(2000-01-01,2020-12-31,'2006-01-02 15:04:05')}"`, `"{time('15:04:05')}"`,
 	} {
 		if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b {
 			t.Fatalf("%s not reproducible: %q != %q", tmpl, a, b)
@@ -98,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
 	}
 }
 
-// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
+// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
 // each call, named counters independent, and the whole thing scoped to one
-// session so a fresh Generator restarts at 1.
-func TestBuiltinSeqPerSession(t *testing.T) {
+// draw state so a fresh Generator restarts at 1.
+func TestBuiltinSeqPerGenerator(t *testing.T) {
 	f := engine(1)
 	for i := 1; i <= 5; i++ {
 		if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -118,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
 	if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
 		t.Fatalf("repeat seq = %q, want 1,2,3", got)
 	}
-	// ...and a fresh session restarts from 1.
+	// ...and a fresh generator restarts from 1.
 	if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
-		t.Fatalf("new session seq = %q, want 1", got)
+		t.Fatalf("new generator seq = %q, want 1", got)
 	}
 }
 
@@ -139,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
 		`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
 	}
 	for _, tmpl := range bad {
-		if _, err := compile(parse(t, tmpl)); err == nil {
+		if _, err := resolved(t, tmpl); err == nil {
 			t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
 		}
 	}
@@ -184,52 +190,229 @@ func ibanValid(s string) bool {
 	return rem == 1
 }
 
-// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
-// supplies prep, and args parsed at compile only behind a check.
-func TestRegistryShapes(t *testing.T) {
-	for name, b := range builtins {
-		if b.prep == nil {
-			t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
-		}
-		if b.arity != 0 && b.check == nil {
-			t.Errorf("builtin %q parses args in prep with no check", name)
-		}
-	}
-}
-
-// TestArgGuardsPanic pins the guards that report a builtin arg its check should
-// have rejected. No data reaches them — checkFunc runs a builtin's check before
-// compileOps ever calls prep — so they are exercised directly.
-func TestArgGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"atoi on an unvalidated arg": func() { atoi("nope") },
-		"atof on an unvalidated arg": func() { atof("nope") },
-	} {
-		mustPanic(t, name, call)
-	}
-}
-
-// mustPanic fails unless call panics, which is what separates a reported
-// invariant break from a silently wrong value.
+// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
+// separates a reported invariant break from a silently wrong value.
 func mustPanic(t *testing.T, name string, call func()) {
 	t.Helper()
 	defer func() {
-		if recover() == nil {
-			t.Errorf("%s: no panic, want the invariant reported", name)
+		if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
+			t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
 		}
 	}()
 	call()
 }
 
 func TestClassBuiltinArgs(t *testing.T) {
-	for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
-		if _, err := compile(parse(t, bad)); err == nil {
+	for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
+		if _, err := resolved(t, bad); err == nil {
 			t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
 		}
 	}
 	for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v", ok, err)
 		}
 	}
 }
+
+// TestBuiltinDateAndTime pins the two clock-free samples: date draws a second in
+// [from 00:00:00, to 23:59:59], both days reachable, and renders it in the quoted Go
+// layout, commas and English names included; time draws a second within one day.
+func TestBuiltinDateAndTime(t *testing.T) {
+	f := engine(1)
+	seen := map[string]bool{}
+	for i := 0; i < 500; i++ {
+		got := mustRender(t, f, `"{date(1990-01-01,1990-12-31,'2006-01-02')}"`)
+		if d, err := time.Parse("2006-01-02", got); err != nil || d.Year() != 1990 {
+			t.Fatalf("date = %q, want a 1990 calendar date (err %v)", got, err)
+		}
+		seen[got] = true
+	}
+	if len(seen) < 200 {
+		t.Fatalf("date drew %d distinct days of 365 in 500, want a uniform spread", len(seen))
+	}
+	lo, hi := false, false
+	for i := 0; i < 200; i++ {
+		got := mustRender(t, f, `"{date(2020-02-28,2020-02-29,'2006-01-02')}"`)
+		if got != "2020-02-28" && got != "2020-02-29" {
+			t.Fatalf("date(2020-02-28,2020-02-29) = %q, out of range", got)
+		}
+		lo, hi = lo || got == "2020-02-28", hi || got == "2020-02-29"
+	}
+	if !lo || !hi {
+		t.Fatalf("date never hit a bound: lo=%v hi=%v (bounds must be inclusive)", lo, hi)
+	}
+	if got := mustRender(t, f, `"{date(2020-07-04,2020-07-05,'January 2, 2006')}"`); got != "July 4, 2020" && got != "July 5, 2020" {
+		t.Fatalf("date with a comma in its layout = %q", got)
+	}
+	rfc := regexp.MustCompile(`^2021-\d\d-\d\dT\d\d:\d\d:\d\dZ$`)
+	clock := regexp.MustCompile(`^([01]\d|2[0-3]):[0-5]\d$`)
+	ampm := regexp.MustCompile(`^(1[0-2]|[1-9]):[0-5]\d (AM|PM)$`)
+	seconds := map[string]bool{}
+	for i := 0; i < 300; i++ {
+		got := mustRender(t, f, `"{date(2021-01-01,2021-12-31,'2006-01-02T15:04:05Z07:00')}"`)
+		if !rfc.MatchString(got) {
+			t.Fatalf("date in an RFC 3339 layout = %q, want %s", got, rfc)
+		}
+		seconds[got[17:19]] = true
+		if got := mustRender(t, f, `"{time('15:04')}"`); !clock.MatchString(got) {
+			t.Fatalf("time('15:04') = %q, want %s", got, clock)
+		}
+		if got := mustRender(t, f, `"{time('3:04 PM')}"`); !ampm.MatchString(got) {
+			t.Fatalf("time('3:04 PM') = %q, want %s", got, ampm)
+		}
+		got = mustRender(t, f, `"{date(1950-01-01,2000-12-31,'060102')}-238{luhn()}"`)
+		if d := digitsOnly(got); len(d) != 10 || !luhnValid(d) {
+			t.Fatalf("a personnummer over date() = %q, want ten Luhn-valid digits", got)
+		}
+	}
+	if len(seconds) < 30 {
+		t.Fatalf("date drew %d distinct seconds in 300, want the whole day, not midnight", len(seconds))
+	}
+}
+
+// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
+// layout is quoted.
+func TestBuiltinDateArgs(t *testing.T) {
+	for tmpl, want := range map[string]string{
+		`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`:        "1990-13-01",
+		`"{time('2006-01-02 15:04')}"`:                        "write date(from,to,layout)",
+		`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`:        "is after",
+		`"{date(1990-01-01,1990-12-31,2006-01-02)}"`:          "'2006-01-02'",
+		`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`:    "'",
+		`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
+		`"{date(1990-01-01,1990-12-31)}"`:                     "3 arguments",
+		`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`:     "'January 2, 2006'",
+		`"{time(3:04 PM, Mon)}"`:                              "'3:04 PM, Mon'",
+		`"{time(15:04)}"`:                                     "'15:04'",
+	} {
+		_, err := resolved(t, tmpl)
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
+		}
+	}
+}
+
+// TestBuiltinLayoutErrorsNameARunnableSpelling pins the two layout errors a user
+// can follow: a double-quoted layout is named single-quoted, without its own
+// quotes carried into the suggestion, whether it split on a comma or not.
+func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
+	_, err := compile(parse(t, `"{date(1990-01-01,1990-12-31,\"2006-01-02\")}"`))
+	if err == nil || !strings.Contains(err.Error(), "write '2006-01-02'") {
+		t.Fatalf("a double-quoted layout = %v, want it named single-quoted", err)
+	}
+	if strings.Contains(err.Error(), `'"`) {
+		t.Errorf("%v names a layout that renders its own quotes", err)
+	}
+	if !strings.Contains(err.Error(), "is double-quoted") {
+		t.Errorf("%v does not say which quotes were wrong", err)
+	}
+	_, err = compile(parse(t, `"{time(15:04)}"`))
+	if err == nil || !strings.Contains(err.Error(), "write '15:04'") {
+		t.Fatalf("a bare layout = %v, want it named quoted", err)
+	}
+	// The comma hint belongs to a layout that split, not to a call given extra args.
+	_, err = compile(parse(t, `"{time(0,12,'15:04')}"`))
+	if err == nil || !strings.Contains(err.Error(), "takes 1 argument, got 3") {
+		t.Fatalf("time with three args = %v, want the count named in the singular", err)
+	}
+	if strings.Contains(err.Error(), "holding a comma") {
+		t.Errorf("%v offers the comma hint though the layout is already quoted", err)
+	}
+}
+
+// TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
+// draws every second of it, and without one it prints that day.
+func TestBuiltinDateSpansOneDay(t *testing.T) {
+	f := engine(1)
+	seen := map[string]bool{}
+	for i := 0; i < 500; i++ {
+		got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02 15:04:05')}"`)
+		if !strings.HasPrefix(got, "2026-01-01 ") {
+			t.Fatalf("date over one day = %q, out of range", got)
+		}
+		seen[got] = true
+	}
+	if len(seen) < 400 {
+		t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
+	}
+	if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
+		t.Fatalf("one day in a date-only layout = %q, want that day", got)
+	}
+}
+
+func TestClassBuiltins(t *testing.T) {
+	cases := map[string]*regexp.Regexp{
+		`"{digits(1)}"`:          regexp.MustCompile(`^[0-9]$`),
+		`"{digits(3)}"`:          regexp.MustCompile(`^[0-9]{3}$`),
+		`"{int(1,9)}"`:           regexp.MustCompile(`^[1-9]$`),
+		`"{upper(1)}"`:           regexp.MustCompile(`^[A-Z]$`),
+		`"{lower(1)}"`:           regexp.MustCompile(`^[a-z]$`),
+		`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
+	}
+	f := engine(7)
+	for tmpl, re := range cases {
+		for i := 0; i < 100; i++ {
+			if got := mustRender(t, f, tmpl); !re.MatchString(got) {
+				t.Fatalf("%s produced %q, want %s", tmpl, got, re)
+			}
+		}
+	}
+}
+
+func TestFunctionTokenLuhn(t *testing.T) {
+	// {luhn()} appends a Luhn check digit over the digits emitted so far in the
+	// current expansion (non-digits skipped but kept). It reads the output
+	// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
+	f := engine(1)
+	cases := map[string]string{
+		`"811218987{luhn()}"`:                      "8112189876",  // personnummer body
+		`"7992739871{luhn()}"`:                     "79927398713", // classic Luhn vector
+		`"811218-987{luhn()}"`:                     "811218-9876", // '-' skipped, kept
+		`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876",  // over a rendered token
+	}
+	for tmpl, want := range cases {
+		if got := mustRender(t, f, tmpl); got != want {
+			t.Fatalf("%s = %q, want %q", tmpl, got, want)
+		}
+	}
+}
+
+func TestBuiltinCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{upper(x)}"`,         // count must be an integer
+		`"{int(a,b)}"`,         // non-integer args
+		`"{int(5,1)}"`,         // min > max
+		`"{nanoid(-1)}"`,       // negative count
+		`"{float(1,2,-1)}"`,    // negative decimals
+		`"{float(NaN,NaN,2)}"`, // bounds must be finite
+		`"{float(Inf,Inf,2)}"`, // same-sign infinities
+		`"{float(1,NaN,2)}"`,   // one NaN bound
+		`"{float(-Inf,1,2)}"`,  // one infinite bound
+		`"{iban(US)}"`,         // unsupported country
+		`"{seq(a,b)}"`,         // seq takes at most one name
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestArgErrorsNameTheSpelling(t *testing.T) {
+	if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
+		t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
+	}
+}
+
+func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
+	for long, short := range map[string]string{
+		`"{float(1,2,02)}"`:                 `"{float(1,2,2)}"`,
+		`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
+		`"{digits(+5)}"`:                    `"{digits(5)}"`,
+		`"{digits(05)}"`:                    `"{digits(5)}"`,
+		`"{int(007,9)}"`:                    `"{int(7,9)}"`,
+		`"{int(+1,5)}"`:                     `"{int(1,5)}"`,
+	} {
+		sameRenders(t, long, short)
+	}
+}

calc.go

@@ -2,392 +2,149 @@ package fejkdata
 
 import (
 	"fmt"
-	"math"
+	"slices"
 	"strconv"
 	"strings"
-	"unicode"
+
+	"github.com/larvit/fejkdata/internal/builtinfunc"
+	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/invariant"
 )
 
-// calcNode is a parsed expression node. It evaluates over the operand values expand
-// read, so the evaluator touches neither the rng nor the node tree.
-type calcNode interface {
-	eval(operands []string) float64
-}
-
-type calcNum float64 // a number literal
-type calcVar string  // a sibling-field name, before indexVars places it
-type calcIdx int     // an operand, by its position in the values expand read
-type calcNeg struct{ x calcNode }
-type calcBin struct { // a + - * / b
-	op   byte
-	l, r calcNode
-}
-
-func (n calcNum) eval([]string) float64 { return float64(n) }
-
-func (n calcIdx) eval(operands []string) float64 {
-	v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
-	if err != nil {
-		return math.NaN() // a non-numeric operand stays visible, never an error
+// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
+// field is a name, which checkCalcNames checks once names resolve.
+func checkCalcFields(args []string, fields map[string]node) error {
+	return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
+		if n, ok := fields[name]; ok {
+			return []node{n}
 		}
-	return v
+		return nil
+	})
 }
 
-// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
-// expression it compiles, so only a calcIdx reaches a render. It panics rather
-// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
-// silently wrong number.
-func (n calcVar) eval([]string) float64 {
-	panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
+// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
+func checkCalcNames(label string, t *template) error {
+	for _, o := range t.compiled.ops {
+		if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
+			continue
+		}
+		if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
+			return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
+		}
+	}
+	return nil
 }
 
-func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
-
-func (n calcBin) eval(operands []string) float64 {
-	l, r := n.l.eval(operands), n.r.eval(operands)
-	switch n.op {
-	case '+':
-		return l + r
-	case '-':
-		return l - r
-	case '*':
-		return l * r
-	default: // '/'
-		return l / r
+// operandNodes lists every node each operand of o may render, once its template and names resolve.
+func operandNodes(o op) func(name string) []node {
+	return func(name string) []node {
+		a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
+		if len(a.leaves) == 0 {
+			panic(invariant.Broken("calc operand %q was read before it was compiled", name))
+		}
+		return a.leaves
 	}
 }
 
-// checkCalc validates a calc token at compile time: a parseable expression whose
-// operands all name existing fields, and an optional non-negative integer dp. It
-// is a builtin check (fields first), so calc dispatches through the registry like
-// every other {name(args)} function.
-func checkCalc(fields map[string]node, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return fmt.Errorf("calc(%q): %w", args[0], err)
-	}
-	for _, name := range calcVars(expr) {
-		operand, ok := fields[name]
-		if !ok {
-			return fmt.Errorf("calc(%q): no field %q", args[0], name)
-		}
-		if text, never := neverNumeric(operand); never {
-			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
+// checkOperands refuses an operand that is never a number and a division by a constant zero.
+// operand lists every node an operand may render, nil while that is unknown.
+func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
+	for _, name := range c.Operands {
+		if rendered, never := allNeverNumeric(operand(name)); never {
+			return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
 		}
 	}
-	if divisor, zero := constantZeroDivisor(expr, fields); zero {
-		return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
-	}
-	if len(args) == 2 {
-		dp, err := plainInt(args[1])
-		if err != nil {
-			return fmt.Errorf("calc decimals %w", err)
-		}
-		if dp < 0 || dp > maxDecimals {
-			return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
-		}
+	if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
+		return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
 	}
 	return nil
 }
 
 // constantZeroDivisor finds a division whose right side is a constant zero: number
 // literals and fixed operands folded, anything that varies left unknown.
-func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
+func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
 	switch n := n.(type) {
-	case calcNeg:
-		return constantZeroDivisor(n.x, fields)
-	case calcBin:
-		if n.op == '/' {
-			if v, known := constantValue(n.r, fields); known && v == 0 {
-				return calcText(n.r), true
+	case grammar.CalcNeg:
+		return constantZeroDivisor(n.X, operand)
+	case grammar.CalcBin:
+		if n.Operator == '/' {
+			if v, known := constantValue(n.R, operand); known && v == 0 {
+				return grammar.CalcText(n.R), true
 			}
 		}
-		if d, zero := constantZeroDivisor(n.l, fields); zero {
+		if d, zero := constantZeroDivisor(n.L, operand); zero {
 			return d, true
 		}
-		return constantZeroDivisor(n.r, fields)
+		return constantZeroDivisor(n.R, operand)
 	}
 	return "", false
 }
 
 // constantValue evaluates an expression whose every operand is fixed.
-func constantValue(n calcNode, fields map[string]node) (float64, bool) {
+func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
 	switch n := n.(type) {
-	case calcNum:
+	case grammar.CalcNum:
 		return float64(n), true
-	case calcVar:
-		t, ok := fields[string(n)].(*template)
-		if !ok || !t.fixed || t.repeat > 1 {
+	case grammar.CalcVar:
+		nodes := operand(n.Name)
+		if len(nodes) != 1 {
 			return 0, false
 		}
-		v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
+		t, ok := nodes[0].(*template)
+		if !ok || t.repeat != 1 {
+			return 0, false
+		}
+		lit, fixed := t.fixedText()
+		if !fixed {
+			return 0, false
+		}
+		v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
 		return v, err == nil
-	case calcNeg:
-		v, ok := constantValue(n.x, fields)
+	case grammar.CalcNeg:
+		v, ok := constantValue(n.X, operand)
 		return -v, ok
-	case calcBin:
-		l, lok := constantValue(n.l, fields)
-		r, rok := constantValue(n.r, fields)
+	case grammar.CalcBin:
+		l, lok := constantValue(n.L, operand)
+		r, rok := constantValue(n.R, operand)
 		if !lok || !rok {
 			return 0, false
 		}
-		return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
+		return builtinfunc.Arith(n.Operator, l, r), true
 	}
 	return 0, false
 }
 
-// calcText spells an expression node the way an author would read it.
-func calcText(n calcNode) string {
-	switch n := n.(type) {
-	case calcNum:
-		return strconv.FormatFloat(float64(n), 'f', -1, 64)
-	case calcVar:
-		return string(n)
-	case calcNeg:
-		return "-" + calcText(n.x)
-	case calcBin:
-		return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
-	}
-	return "?"
-}
-
-// neverNumeric reports a node no render of which is a number: a null, fixed text that
-// does not parse, or a choice of only such items. text is one such render.
+// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
+// text that does not parse, or a choice of only such items. text is one such render.
 func neverNumeric(n node) (text string, never bool) {
 	switch n := n.(type) {
-	case *null:
+	case *nullItem:
 		return "", true
 	case *template:
-		if !n.fixed || n.repeat > 1 {
+		lit, fixed := n.fixedText()
+		if n.repeat == 0 {
+			return "", true
+		}
+		if !fixed || n.repeat != 1 {
 			return "", false
 		}
-		if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
-			return n.lit, true
+		if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
+			return lit, true
 		}
 	case *choice:
-		for _, it := range n.items {
-			t, itemNever := neverNumeric(it)
-			if !itemNever {
+		return allNeverNumeric(n.items)
+	}
+	return "", false
+}
+
+// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
+func allNeverNumeric(nodes []node) (text string, never bool) {
+	for _, n := range nodes {
+		t, nodeNever := neverNumeric(n)
+		if !nodeNever {
 			return "", false
 		}
 		text = t
 	}
-		return text, true
-	}
-	return "", false
-}
-
-// calcPrep parses the expression and decimals once, at compile time, and places each
-// operand name at the position expand will read it into. checkCalc proved both args
-// valid, so no step here can fail; dp -1 prints the minimal form.
-func calcPrep(args []string) callFn {
-	expr, err := parseCalc(args[0])
-	if err != nil { // a nil AST would be a nil dereference per render, with no message
-		panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
-	}
-	at := make(map[string]int)
-	for i, name := range calcVars(expr) {
-		at[name] = i
-	}
-	placed := indexVars(expr, at)
-	dp := calcDecimals(args)
-	return func(_ *session, _ string, operands []string) string {
-		return formatFloat(placed.eval(operands), dp)
-	}
-}
-
-// calcDecimals is a calc's decimals count, or -1 for the shortest form.
-func calcDecimals(args []string) int {
-	if len(args) == 2 {
-		return atoi(args[1])
-	}
-	return -1
-}
-
-// indexVars replaces each operand name with its position in the values expand reads.
-// Both sides take that order from calcVars, so they cannot drift.
-func indexVars(n calcNode, at map[string]int) calcNode {
-	switch n := n.(type) {
-	case calcVar:
-		i, placed := at[string(n)]
-		if !placed { // calcVars named every operand, so a miss means the two disagree
-			panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
-		}
-		return calcIdx(i)
-	case calcNeg:
-		return calcNeg{indexVars(n.x, at)}
-	case calcBin:
-		return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
-	}
-	return n
-}
-
-// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
-// an expression that does not parse, so one that does not simply names nothing.
-func calcOperands(args []string) []string {
-	if len(args) == 0 {
-		return nil
-	}
-	expr, err := parseCalc(args[0])
-	if err != nil {
-		return nil
-	}
-	return calcVars(expr)
-}
-
-// calcVars lists the distinct field names an expression reads, in the order it first
-// names each. That order is the contract between expand, which reads the operands
-// into a slice, and indexVars, which places each name at its position in it.
-func calcVars(n calcNode) []string {
-	var out []string
-	seen := map[string]bool{}
-	var walk func(calcNode)
-	walk = func(n calcNode) {
-		switch n := n.(type) {
-		case calcVar:
-			if name := string(n); !seen[name] {
-				seen[name] = true
-				out = append(out, name)
-			}
-		case calcNeg:
-			walk(n.x)
-		case calcBin:
-			walk(n.l)
-			walk(n.r)
-		}
-	}
-	walk(n)
-	return out
-}
-
-// calcParser is a recursive-descent parser over the expression runes, threading
-// expr -> term -> factor for the standard * / before + - precedence.
-type calcParser struct {
-	rs  []rune
-	pos int
-}
-
-// parseCalc parses a whole expression, requiring it to consume all input.
-func parseCalc(expr string) (calcNode, error) {
-	p := &calcParser{rs: []rune(expr)}
-	if p.space(); p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("empty expression")
-	}
-	n, err := p.expr()
-	if err != nil {
-		return nil, err
-	}
-	if p.space(); p.pos != len(p.rs) {
-		return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
-	}
-	return n, nil
-}
-
-func (p *calcParser) space() {
-	for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
-		p.pos++
-	}
-}
-
-func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
-func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
-
-// binary parses a left-associative run of next() operands joined by the given
-// operators, the one shape expr and term share.
-func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
-	n, err := next()
-	if err != nil {
-		return nil, err
-	}
-	for {
-		p.space()
-		if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
-			return n, nil
-		}
-		op := byte(p.rs[p.pos])
-		p.pos++
-		r, err := next()
-		if err != nil {
-			return nil, err
-		}
-		n = calcBin{op, n, r}
-	}
-}
-
-// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
-func (p *calcParser) factor() (calcNode, error) {
-	p.space()
-	if p.pos >= len(p.rs) {
-		return nil, fmt.Errorf("unexpected end of expression")
-	}
-	switch c := p.rs[p.pos]; {
-	case c == '-':
-		p.pos++
-		x, err := p.factor()
-		if err != nil {
-			return nil, err
-		}
-		return calcNeg{x}, nil
-	case c == '(':
-		p.pos++
-		n, err := p.expr()
-		if err != nil {
-			return nil, err
-		}
-		if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
-			return nil, fmt.Errorf("missing ')'")
-		}
-		p.pos++
-		return n, nil
-	case c == '.' || c >= '0' && c <= '9':
-		return p.number()
-	case c == '_' || unicode.IsLetter(c):
-		return p.ident()
-	default:
-		return nil, fmt.Errorf("unexpected %q", string(c))
-	}
-}
-
-func (p *calcParser) number() (calcNode, error) {
-	start, dot := p.pos, false
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c >= '0' && c <= '9' {
-			p.pos++
-		} else if c == '.' && !dot {
-			dot, p.pos = true, p.pos+1
-		} else {
-			break
-		}
-	}
-	v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
-	if err != nil {
-		return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
-	}
-	return calcNum(v), nil
-}
-
-// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
-// is always the minus operator, so a hyphenated field name can't be an operand.
-func (p *calcParser) ident() (calcNode, error) {
-	start := p.pos
-	for p.pos < len(p.rs) {
-		if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
-			p.pos++
-		} else {
-			break
-		}
-	}
-	return calcVar(string(p.rs[start:p.pos])), nil
-}
-
-func contains(bs []byte, b byte) bool {
-	for _, x := range bs {
-		if x == b {
-			return true
-		}
-	}
-	return false
+	return text, len(nodes) > 0
 }

calc_test.go

@@ -3,8 +3,11 @@ package fejkdata
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
 )
 
 // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
 	if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
 		t.Fatalf("calc(a - b) = %q, want 7", got)
 	}
+	if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
+		t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
+	}
 }
 
 // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
 	}
 }
 
-// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
-// is not a calc, and for one whose expression does not parse: nothing, either way.
-// checkCalc is what reports a bad expression, so the callers that run after it
-// never meet one — but they must not have to depend on that order to be safe.
-func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
-	for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
-		if got := tokenOperands(body); got != nil {
-			t.Errorf("tokenOperands(%q) = %v, want none", body, got)
+// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
+// call that is not a calc, and for one whose expression does not parse: none, either way.
+func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
+	for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
+		if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
+			t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
 		}
 	}
-	if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
-		t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
-	}
-}
-
-// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
-// the expression before prep sees it, and indexVars places every name calcVars
-// read, so a break in either is reported rather than computed around.
-func TestCalcGuardsPanic(t *testing.T) {
-	for name, call := range map[string]func(){
-		"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
-		"an operand name never placed":              func() { calcVar("n").eval(nil) },
-		"indexVars on a name it did not read":       func() { indexVars(calcVar("n"), map[string]int{}) },
-	} {
-		mustPanic(t, name, call)
+	if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
+		t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
 	}
 }
 
 // --- one draw, one value: a calc operand reads the expansion's draw ---
 
-// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
-// A field the format renders and a calc reads is drawn once per expansion, so the
-// operand shown is the operand computed — the correlation the dotted-path rule
-// already gives a level (see bound_test.go), applied to a plain sibling.
-func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
+// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
+// shows: the format and the calc read one name.
+func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
+		"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(3))
 	for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
 	}
 }
 
-// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
-// expansion, not to the calc, so the bare token rendering the same name reads it too.
-func TestCalcOperandSharesOneDraw(t *testing.T) {
-	dir := writeData(t, map[string]string{
-		"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
-	})
+// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
+func TestACalcOperandDrawsAfresh(t *testing.T) {
+	dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
 	f := newGenerator(t, dir, WithSeed(5))
 	for i := 0; i < 200; i++ {
-		got := fake(t, f, "same")
-		if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
-			t.Fatalf("same = %q, want one value twice", got)
-		}
-	}
-}
-
-// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
-// held, so an ordinary {w} {w} still draws twice.
-func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
-	dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
-	f := newGenerator(t, dir, WithSeed(5))
-	for i := 0; i < 200; i++ {
-		if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
+		if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
 			return
 		}
 	}
-	t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
+	t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
 }
 
-// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
-// its own expansion, so it draws again while staying self-consistent.
-func TestCalcHoldIsPerExpansion(t *testing.T) {
+// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
+// iteration, staying one value within it.
+func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
+		"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
 	})
 	f := newGenerator(t, dir, WithSeed(11))
 	varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
 	}
 }
 
-// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
-// {v} and its calc agree with each other, not with the outer pair.
-func TestCalcHoldIsPerTemplate(t *testing.T) {
+// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
+// pair agrees with itself, not with the outer pair.
+func TestTwoNamesPickApart(t *testing.T) {
 	dir := writeData(t, map[string]string{
-		"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
-			"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
+		"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
+			"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
 	})
 	f := newGenerator(t, dir, WithSeed(13))
 	differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["a","b"]}`:       `"x"`,
 		`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
 	} {
-		_, err := compile(parse(t, src))
+		_, err := resolved(t, src)
 		if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
 			t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
 		}
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
 		`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
 		`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
 	} {
-		if _, err := compile(parse(t, ok)); err != nil {
+		if _, err := resolved(t, ok); err != nil {
 			t.Errorf("compile(%s) = %v, want it accepted", ok, err)
 		}
 	}
@@ -257,7 +230,7 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 		`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`:   "divides by 0",
 		`{"format":"{calc(a/-0)}","a":"1"}`:              "divides by -0",
 	} {
-		if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
+		if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
 			t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
 		}
 	}
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
 	}
 	t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
 }
+
+func TestCalcCompileErrors(t *testing.T) {
+	for _, bad := range []string{
+		`"{calc(1/0)}"`, // a constant zero divisor
+		`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
+		`"{calc()}"`,        // calc needs an expression
+		`"{calc(1 +)}"`,     // dangling operator
+		`"{calc((1 + 2)}"`,  // unbalanced parenthesis
+		`"{calc(1 2)}"`,     // two operands, no operator
+		`"{calc(price)}"`,   // operand names no field
+		`"{calc(1, 2, 3)}"`, // too many args
+		`"{calc(1, x)}"`,    // decimals arg not an integer
+		`"{calc(1, -1)}"`,   // decimals negative
+	} {
+		if _, err := resolved(t, bad); err == nil {
+			t.Errorf("compile(%s) = nil error, want error", bad)
+		}
+	}
+}
+
+func TestCalcReadsAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3","5","7"]`,
+		"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	for i := 0; i < 50; i++ {
+		var k, double int
+		got := fake(t, f, "card")
+		if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
+			t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
+		}
+	}
+}
+
+func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":    `["2","3"]`,
+		"card": `"{/n as k}{calc(k * 2)}"`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	if got := fake(t, f, "card"); got != "4" && got != "6" {
+		t.Fatalf("card = %q, want 4 or 6", got)
+	}
+}
+
+func TestCalcRefusesABadOperand(t *testing.T) {
+	for card, want := range map[string]string{
+		`"{/word as w}{w}{calc(w * 2)}"`:    `operand "w" is never a number: it renders "def"`,
+		`"{/zero as z}{z}{calc(1 / z)}"`:    `divides by z, which is always zero`,
+		`"{calc(nope * 2)}"`:                `no field "nope"`,
+		`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
+	} {
+		dir := writeData(t, map[string]string{
+			"word": `["abc","def"]`,
+			"zero": `"0"`,
+			"card": card,
+		})
+		if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
+		}
+	}
+}
+
+func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
+	_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
+		"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
+	})))
+	if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
+		t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
+	}
+}
+
+func TestATypedColumnProvesACalcOverAName(t *testing.T) {
+	dir := writeData(t, map[string]string{
+		"n":     `"{int(1,9)}"`,
+		"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
+	})
+	f := newGenerator(t, dir, WithSeed(1))
+	r, err := f.FakeRecord("order")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
+		t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
+	}
+	if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
+		t.Fatalf("JSON = %s, want double written as a number", r.JSON())
+	}
+	bad := writeData(t, map[string]string{
+		"word":  `["1","x"]`,
+		"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
+	})
+	if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
+		t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
+	}
+}
+
+func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
+	if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
+		t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
+	}
+}

cli/cli.go

@@ -0,0 +1,706 @@
+// Package cli is the fejkdata command: it renders the template on its stdin with fake
+// values from the modules it is given and any directories layered over them. A program
+// passing its own modules to Main is a fejkdata CLI carrying them, for example
+// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
+package cli
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/larvit/fejkdata"
+)
+
+const usage = `Usage: <template> | COMMAND [flags]
+       COMMAND --list | --help | --version
+
+COMMAND reads one template from stdin and renders it:
+
+  echo 'id: {int(1,999)}' | COMMAND
+  echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
+  COMMAND < template.txt
+
+COMMAND <<'EOF'
+born {date(1990-01-01,2010-12-31,'2006-01-02')}
+EOF
+
+A template is a format string, or a JSON object, array, string or null. Its {…} tokens
+reach the data by reference from the root: {/path}, for any path --list prints, shipped
+or layered with --data-path. A reference names a category or a dotted path into one, a
+table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
+links to.
+
+Under --format text, the default, what you send is what renders, and COMMAND adds
+nothing: a format string keeps every byte, so echo's newline ends each render, and
+printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
+object, array or string drops the whitespace around it, and null alone renders nothing;
+42 or true alone is a format string, printed as written. An empty or blank stdin is a
+format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
+a backslash or a quote as written.
+
+With --format json, ndjson, csv or sql the template must be a record — a JSON
+template whose fields are its columns, or a lone reference such as {/path}:
+one reference and nothing else, with one newline after it allowed — and the rows are
+written as one JSON array, one JSON object per line, one CSV row (after a header), or
+one INSERT.
+
+  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
+      --format F         output form: text (default), json, ndjson, csv or sql
+  -h, --help             print this help, then exit
+      --list             list the paths the data offers, each written after {/ in a template, then exit;
+                         --repeat, --separator, --format and --table do nothing here
+      --no-shipped-data  load only the --data-path directories
+  -n, --repeat N         render the template N times, 0..1048576 (default 1); 0 prints nothing,
+                         [] under --format json, or the header under --format csv
+  -s, --seed N           same seed, version and data: identical output (default: a random seed)
+      --separator S      string between repeated renders (default empty); refused with a record
+                         --format, which has its own separators
+      --table T          the INSERT target for --format sql, ignored under any other
+                         (default: a lone reference's last segment, else records)
+      --version          print the version, then exit
+
+A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
+-- ends the flags.
+
+Exit codes: 0 success, 1 runtime error, 2 misuse.
+`
+
+type invocation struct {
+	args         []string
+	dirs         []string
+	format       string
+	help         bool
+	list         bool
+	noShipped    bool
+	repeat       int
+	seed         uint64
+	seeded       bool
+	separator    string
+	separatorSet bool
+	table        string
+	tableSet     bool
+	version      bool
+}
+
+type flagDef struct {
+	long  string
+	short string
+	value bool
+	set   func(*invocation, string) error
+}
+
+var flagDefs = []flagDef{
+	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
+	{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
+	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
+	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
+	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
+	{"repeat", "n", true, func(in *invocation, v string) error {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n > fejkdata.MaxRepeat {
+			return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
+		}
+		in.repeat = n
+		return nil
+	}},
+	{"seed", "s", true, func(in *invocation, v string) error {
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
+		}
+		in.seed, in.seeded = n, true
+		return nil
+	}},
+	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
+	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
+	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
+}
+
+func flagByLong(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].long == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+func flagByShort(name string) *flagDef {
+	for i := range flagDefs {
+		if flagDefs[i].short == name {
+			return &flagDefs[i]
+		}
+	}
+	return nil
+}
+
+// flagArg is one flag as written: its definition and the value attached to it,
+// if any.
+type flagArg struct {
+	def    *flagDef
+	value  string
+	inline bool
+}
+
+// splitFlags reads one argv element as flags: --name or --name=value, or -abc
+// where each letter is a short flag and the first that takes a value takes the
+// rest of the element.
+func splitFlags(arg string) ([]flagArg, error) {
+	if strings.HasPrefix(arg, "--") {
+		name, value, inline := strings.Cut(arg[2:], "=")
+		def := flagByLong(name)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag --%s", name)
+		}
+		return []flagArg{{def, value, inline}}, nil
+	}
+	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
+		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
+	}
+	var flags []flagArg
+	letters := arg[1:]
+	for i, r := range letters {
+		letter := string(r)
+		def := flagByShort(letter)
+		if def == nil {
+			return nil, fmt.Errorf("unknown flag -%s", letter)
+		}
+		if !def.value {
+			flags = append(flags, flagArg{def: def})
+			continue
+		}
+		rest := letters[i+len(letter):]
+		if strings.HasPrefix(rest, "=") {
+			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
+				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
+		}
+		return append(flags, flagArg{def, rest, rest != ""}), nil
+	}
+	return flags, nil
+}
+
+func parseArgs(argv []string) (invocation, error) {
+	in := invocation{repeat: 1, format: "text"}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		if arg == "--" {
+			in.args = append(in.args, argv[i+1:]...)
+			return in, nil
+		}
+		if len(arg) < 2 || arg[0] != '-' {
+			in.args = append(in.args, arg)
+			continue
+		}
+		flags, err := splitFlags(arg)
+		if err != nil {
+			return in, err
+		}
+		for _, fl := range flags {
+			if !fl.def.value {
+				if fl.inline {
+					return in, fmt.Errorf("--%s takes no value", fl.def.long)
+				}
+				_ = fl.def.set(&in, "")
+				continue
+			}
+			value := fl.value
+			if !fl.inline {
+				if i++; i >= len(argv) {
+					return in, fmt.Errorf("--%s needs a value", fl.def.long)
+				}
+				value = argv[i]
+			}
+			if err := fl.def.set(&in, value); err != nil {
+				return in, err
+			}
+		}
+	}
+	return in, nil
+}
+
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
+type recordFormat struct {
+	header func(*fejkdata.Record) string
+	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
+	writes string // what it writes between values, which --separator would seem to set
+}
+
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson writes one object per line.
+var recordFormats = map[string]recordFormat{
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
+	"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
+}
+
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
+func (in invocation) writesRecords() bool {
+	return in.format != "text"
+}
+
+// formatNames lists the --format values, text included, for the misuse error.
+func formatNames() string {
+	names := []string{"text"}
+	for name := range recordFormats {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
+}
+
+// checkFlags rejects a flag value or combination that cannot run.
+func (in invocation) checkFlags() error {
+	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
+		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
+	}
+	if in.tableSet && in.table == "" && in.format == "sql" {
+		return errors.New("--table names the INSERT target, so it cannot be empty")
+	}
+	if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
+		return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
+	}
+	return nil
+}
+
+func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
+
+// operandError refuses an argument that is no flag, naming the spelling that reads what
+// it most likely means: dropping it for -, a file for a file's name, else a template
+// piped in.
+func operandError(arg, name string) error {
+	unexpected := "unexpected argument '" + arg + "'"
+	if arg == "-" {
+		return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
+	}
+	if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
+		return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
+	}
+	ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
+	_, argLone := loneReference(arg)
+	inline, err := fejkdata.IsTemplate(arg)
+	_, refLone := loneReference(ref)
+	example := pipeHint(name)
+	switch {
+	case argLone || inline:
+		example = piped(arg, name)
+	case err == nil && refLone:
+		example = piped(ref, name)
+	}
+	return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
+}
+
+// check rejects a flag combination or an operand that cannot run, before any data is
+// read.
+func (in invocation) check(name string) error {
+	if err := in.checkFlags(); err != nil {
+		return err
+	}
+	if len(in.args) > 0 {
+		return operandError(in.args[0], name)
+	}
+	return nil
+}
+
+func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
+	if in.noShipped {
+		modules = nil
+	}
+	opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
+	for _, dir := range in.dirs {
+		opts = append(opts, fejkdata.WithDataPath(dir))
+	}
+	seed := in.seed
+	if !in.seeded {
+		seed = systemSeed
+	}
+	return append(opts, fejkdata.WithSeed(seed))
+}
+
+// write streams the input's renders to w, repeat of them joined by the separator; a write
+// failure surfaces from Flush, bufio keeping the first one.
+func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
+	if in.writesRecords() {
+		return in.writeRecords(f, src, w)
+	}
+	t, err := f.NewTemplate(src.template)
+	if err != nil {
+		return templateError(err)
+	}
+	out := bufio.NewWriter(w)
+	for i := 0; i < in.repeat; i++ {
+		if i > 0 {
+			out.WriteString(in.separator)
+		}
+		out.WriteString(t.Fake())
+	}
+	return out.Flush()
+}
+
+// recordStream builds the record drawer for the input, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
+	record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
+	if src.record == "" {
+		t, err := f.NewRecordTemplate(src.template)
+		if err != nil {
+			return nil, "", templateError(spacedReference(src.template, err))
+		}
+		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
+	}
+	table := in.table
+	if table == "" {
+		table = defaultTable(src.record)
+	}
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
+	record, table, err := in.recordStream(f, src)
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	if in.repeat == 0 {
+		return writeNoRecord(record, format, w)
+	}
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
+		r, err := record()
+		if err != nil {
+			return err
+		}
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
+		}
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
+}
+
+// writeNoRecord writes zero records: the header where the format writes one, an empty document
+// where it frames one, else nothing. It draws one record, which names the header's columns, so a
+// lone reference to nothing fails as it would with records.
+func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
+	r, err := record()
+	switch {
+	case err != nil:
+		return err
+	case format.header != nil:
+		_, err = io.WriteString(w, format.header(r)+"\n")
+	case format.open != "":
+		_, err = io.WriteString(w, format.open+format.close+"\n")
+	}
+	return err
+}
+
+// defaultTable names the INSERT target when --table is absent: the path's last
+// segment, or "records" when stdin is not a lone reference.
+func defaultTable(path string) string {
+	if path == "" {
+		return "records"
+	}
+	segments := strings.Split(withoutSelectors(path), ".")
+	return segments[len(segments)-1]
+}
+
+// withoutSelectors is path with its [selectors] cut out.
+func withoutSelectors(path string) string {
+	var names strings.Builder
+	for depth, i := 0, 0; i < len(path); i++ {
+		switch {
+		case path[i] == '[':
+			depth++
+		case path[i] == ']':
+			depth--
+		case depth == 0:
+			names.WriteByte(path[i])
+		}
+	}
+	return names.String()
+}
+
+// inputError marks a failure that is the input's own fault — nothing piped in, a template
+// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
+// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
+// error (exit 1).
+type inputError struct{ error }
+
+func (e inputError) Unwrap() error { return e.error }
+
+// jsonSpace is the whitespace JSON allows around a value.
+const jsonSpace = " \t\r\n"
+
+// input is what stdin holds: the template, and the path a lone reference names as a
+// record.
+type input struct {
+	template string
+	record   string
+}
+
+// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
+// reference once JSON's surrounding whitespace, or one final newline, is cut.
+func parseInput(raw string) input {
+	trimmed := strings.Trim(raw, jsonSpace)
+	text := raw
+	if json.Valid([]byte(trimmed)) {
+		text = trimmed
+	}
+	text, found := strings.CutSuffix(text, "\n")
+	if found {
+		text = strings.TrimSuffix(text, "\r")
+	}
+	record, _ := loneReference(text)
+	return input{template: raw, record: record}
+}
+
+// loneReference returns the path that text reads when it is one reference from the root
+// and nothing else, bare or as a JSON string.
+func loneReference(text string) (string, bool) {
+	var unquoted string
+	if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
+		text = unquoted
+	}
+	path, opens := strings.CutPrefix(text, "{/")
+	path, closes := strings.CutSuffix(path, "}")
+	if !opens || !closes || !isPath(path) {
+		return "", false
+	}
+	return path, true
+}
+
+// isPath reports whether a reference's body after its / is a path alone: no second /, no
+// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
+// It copies the grammar, since the CLI reads only the library's public API.
+func isPath(body string) bool {
+	if body == "" || strings.ContainsAny(body[:1], "/.") {
+		return false
+	}
+	names := withoutSelectors(body)
+	if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
+		return false
+	}
+	inline, err := fejkdata.IsTemplate(body)
+	return !inline && err == nil
+}
+
+// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
+// step up, and only a step up may end the path.
+func hasEmptySegment(path string) bool {
+	segments := strings.Split(path, ".")
+	stepUp := false
+	for i, seg := range segments[1:] {
+		last := i == len(segments)-2
+		switch {
+		case seg != "":
+			stepUp = false
+		case stepUp != last:
+			return true
+		default:
+			stepUp = !stepUp
+		}
+	}
+	return false
+}
+
+// spacedReference adds to a record refusal that whitespace around a lone reference made
+// the template text.
+func spacedReference(template string, err error) error {
+	if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
+		if path, lone := loneReference(trimmed); lone {
+			return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
+		}
+	}
+	return err
+}
+
+// piped is the command piping template into the command called name; some shells' echo
+// reads a backslash as an escape, so a template holding one goes through printf.
+func piped(template, name string) string {
+	if strings.ContainsRune(template, '\\') {
+		return "printf '%s' " + shellQuoted(template) + " | " + name
+	}
+	return "echo " + shellQuoted(template) + " | " + name
+}
+
+// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
+func shellQuoted(s string) string {
+	if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
+		return s
+	}
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// Main runs the command on the process's arguments and standard streams over modules, each
+// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
+// 2 misuse. --no-shipped-data leaves modules out.
+func Main(modules ...fs.FS) int {
+	var stdin io.Reader = os.Stdin
+	if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+		stdin = nil
+	}
+	return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
+}
+
+// commandName is the name the command runs under: the file arg0 names, less a Windows
+// .exe, or fejkdata where arg0 names none.
+func commandName(arg0 string) string {
+	name := filepath.Base(arg0)
+	if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
+		name = strings.TrimSuffix(name, ext)
+	}
+	if name == "." || name == "" {
+		return "fejkdata"
+	}
+	return name
+}
+
+// run is Main over the given streams, run as the command name, seeded by systemSeed where
+// no --seed is given. A nil stdin is a terminal, which holds no template.
+func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
+	in, err := parseArgs(args)
+	if err != nil {
+		return misuse(stderr, name, err)
+	}
+	if in.help {
+		io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
+		return 0
+	}
+	if in.version {
+		info, _ := debug.ReadBuildInfo()
+		fmt.Fprintln(stdout, versionLine(name, info))
+		return 0
+	}
+	if err := in.check(name); err != nil {
+		return misuse(stderr, name, err)
+	}
+	var src input
+	if !in.list {
+		if src, err = readStdin(stdin, name); err != nil {
+			return fail(stderr, name, err)
+		}
+	}
+	f, err := fejkdata.New(in.options(modules, systemSeed)...)
+	if err != nil {
+		return fail(stderr, name, err)
+	}
+	if in.list {
+		for _, p := range f.List() {
+			fmt.Fprintln(stdout, p)
+		}
+		return 0
+	}
+	if err := in.write(f, src, stdout); err != nil {
+		return fail(stderr, name, err)
+	}
+	return 0
+}
+
+func readStdin(stdin io.Reader, name string) (input, error) {
+	if stdin == nil {
+		return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
+	}
+	raw, err := io.ReadAll(stdin)
+	if err != nil {
+		return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
+	}
+	return parseInput(string(raw)), nil
+}
+
+// templateError is misuse unless the data the template reads failed to load.
+func templateError(err error) error {
+	if errors.Is(err, fejkdata.ErrLoad) {
+		return err
+	}
+	return inputError{err}
+}
+
+func fail(stderr io.Writer, name string, err error) int {
+	var te inputError
+	if errors.As(err, &te) {
+		return misuse(stderr, name, te.error)
+	}
+	report(stderr, name, err)
+	if errors.Is(err, fejkdata.ErrNoColumns) {
+		fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
+	}
+	return 1
+}
+
+func misuse(stderr io.Writer, name string, err error) int {
+	report(stderr, name, err)
+	fmt.Fprintf(stderr, "try '%s --help'\n", name)
+	return 2
+}
+
+// report writes err under the command name, in place of the library's own on each line.
+func report(stderr io.Writer, name string, err error) {
+	msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
+	fmt.Fprintf(stderr, "%s: %s\n", name, msg)
+}
+
+// fejkdataModule is the path of the module holding the library; the root package's path is
+// the module's.
+var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
+
+// versionLine names the fejkdata version the binary is built with. Where another module's
+// main package carries the CLI, the main module's version comes first, under the command name.
+func versionLine(name string, info *debug.BuildInfo) string {
+	if info == nil {
+		return "fejkdata devel"
+	}
+	if info.Main.Path == fejkdataModule {
+		return "fejkdata " + moduleVersion(info.Main)
+	}
+	for _, m := range info.Deps {
+		if m.Path == fejkdataModule {
+			return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
+		}
+	}
+	return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
+}
+
+// moduleVersion is the version m is built at: the replacement's where a replace directive
+// names one, and "devel" where it carries none, as a local directory does.
+func moduleVersion(m debug.Module) string {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Version == "" || m.Version == "(devel)" {
+		return "devel"
+	}
+	return m.Version
+}

cli/cli_test.go

@@ -0,0 +1,961 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// shipped holds the shipped modules as read from the checkout, each folder under ../data
+// holding a manifest, so these tests link no data package.
+var shipped = func() []fs.FS {
+	var modules []fs.FS
+	err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
+		if err == nil && d.Name() == ".fejkdata.json" {
+			modules = append(modules, os.DirFS(filepath.Dir(path)))
+		}
+		return err
+	})
+	if err != nil || len(modules) == 0 {
+		panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
+	}
+	return modules
+}()
+
+// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
+// layers its categories unprefixed with -d.
+var svSE, enUS, misc string
+
+func TestMain(m *testing.M) {
+	tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
+	if err != nil {
+		panic(err)
+	}
+	svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
+	code := m.Run()
+	os.RemoveAll(tmp)
+	os.Exit(code)
+}
+
+// bareCopy copies the category and rows files of the shipped module in data/<module> into a
+// folder under tmp, and returns that folder.
+func bareCopy(tmp, module string) string {
+	src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
+	if err := os.Mkdir(dst, 0o755); err != nil {
+		panic(err)
+	}
+	for _, pattern := range []string{"*.json", "*.tsv"} {
+		files, err := filepath.Glob(filepath.Join(src, pattern))
+		if err != nil {
+			panic(err)
+		}
+		for _, f := range files {
+			if strings.HasPrefix(filepath.Base(f), ".") {
+				continue
+			}
+			b, err := os.ReadFile(f)
+			if err == nil {
+				err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
+			}
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	return dst
+}
+
+func runOut(stdin string, args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+// runTerminal runs as from a shell with nothing piped in.
+func runTerminal(args ...string) (int, string, string) {
+	var out, errb bytes.Buffer
+	code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
+	return code, out.String(), errb.String()
+}
+
+func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
+	module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
+		t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
+	}
+	out.Reset()
+	if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
+		t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
+	}
+}
+
+func TestRunOutputsValue(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if out == "" || strings.Contains(out, "\n") {
+		t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
+	}
+}
+
+func TestRunDotPath(t *testing.T) {
+	code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person run = %d", code)
+	}
+	code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(last) == "" {
+		t.Fatal("empty person.last output")
+	}
+	if last == full {
+		t.Errorf("person.last %q should differ from person %q", last, full)
+	}
+}
+
+func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
+	var drawn, seeded, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
+		t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
+		t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
+	}
+	if drawn.String() != seeded.String() {
+		t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
+	}
+}
+
+func TestRunSeedSpellings(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"--seed=42", "--data-path", svSE},
+		{"-s", "42", "-d", svSE},
+		{"--data-path", svSE, "--seed", "42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+}
+
+func TestRunShortFlagValues(t *testing.T) {
+	_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
+	for _, args := range [][]string{
+		{"-s42", "-d", svSE},
+		{"-s", "42", "-d" + svSE},
+		{"-d", svSE, "-s42"},
+	} {
+		code, got, errb := runOut("{/address}", args...)
+		if code != 0 {
+			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
+		}
+		if got != want {
+			t.Errorf("run(%v) = %q, want %q", args, got, want)
+		}
+	}
+	_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
+	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
+		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-s=42", "-d", svSE}, "-s42"},
+		{[]string{"-s=42", "-d", svSE}, "--seed=42"},
+		{[]string{"-d=" + svSE}, "--data-path="},
+		{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
+		{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
+	} {
+		code, out, errb := runOut("{/word}", c.args...)
+		if code != 2 || out != "" {
+			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+	code, out, _ := runOut("", "-hd", svSE)
+	if code != 0 || !strings.Contains(out, "Usage") {
+		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
+	}
+}
+
+func TestRunDoubleDashEndsFlags(t *testing.T) {
+	code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
+	if code != 2 || !strings.Contains(errb, "stdin") {
+		t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSingleDashLongIsRejected(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
+		{[]string{"-seed=42", "-d", svSE}, "use --seed"},
+		{[]string{"-data-path", svSE}, "use --data-path"},
+		{[]string{"-list", "-d", svSE}, "use --list"},
+		{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
+		{[]string{"-x", "-d", svSE}, "unknown flag -x"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunFlagValues(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
+		{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
+		{[]string{"--seed", "x", "-d", svSE}, "--seed"},
+		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
+		{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
+		{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
+	} {
+		code, _, errb := runOut("{/person}", c.args...)
+		if code != 2 {
+			t.Errorf("run(%v) = %d, want 2", c.args, code)
+		}
+		if !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
+		}
+	}
+}
+
+func TestRunRepeat(t *testing.T) {
+	code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
+	}
+	_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
+	if short != out {
+		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
+	}
+}
+
+func TestRunSeparator(t *testing.T) {
+	code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if n := strings.Count(out, "\n"); n != 0 {
+		t.Errorf("want no newline, got %d: %q", n, out)
+	}
+	if !strings.Contains(out, ",") {
+		t.Errorf("values should be comma-joined: %q", out)
+	}
+}
+
+func TestRunRepeatAdvancesRNG(t *testing.T) {
+	code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	uniq := map[string]bool{}
+	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+		uniq[l] = true
+	}
+	if len(uniq) < 2 {
+		t.Errorf("repeat should vary output, all identical: %q", out)
+	}
+}
+
+func TestRunMisuse(t *testing.T) {
+	for _, c := range []struct {
+		stdin string
+		args  []string
+	}{
+		{"{/person}", []string{"-d", svSE, "word"}},
+		{"", []string{"-d", svSE, "--list", "person"}},
+		{"{/sv_SE.person}", []string{"--no-shipped-data"}},
+	} {
+		code, out, errb := runOut(c.stdin, c.args...)
+		if code != 2 {
+			t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
+		}
+		if !strings.Contains(errb, "try 'fejkdata --help'") {
+			t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
+		}
+		if out != "" {
+			t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
+		}
+	}
+}
+
+func TestRunTakesNoOperand(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "users.tmpl")
+	if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
+		{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
+		{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
+		{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
+		{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
+		{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
+		{[]string{"-"}, "drop it"},
+		{[]string{file}, "fejkdata < " + file},
+	} {
+		code, out, errb := runOut("", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+type failingReader struct{}
+
+func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
+
+func TestRunUnreadableStdinIsMisuse(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
+		t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
+	}
+}
+
+func TestRunTerminalStdinIsMisuse(t *testing.T) {
+	code, out, errb := runTerminal("--seed", "1")
+	if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
+		t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
+	}
+	for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
+		if code, out, errb := runTerminal(args...); code != 0 || out == "" {
+			t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
+		}
+	}
+}
+
+func TestRunRendersExactlyWhatIsSent(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"hihi":                           "hihi",
+		"hihi\n":                         "hihi\n",
+		"hihi\r\n":                       "hihi\r\n",
+		"hihi\n\n":                       "hihi\n\n",
+		"{{x}}\n":                        "{x}\n",
+		"\"a\"\n":                        "a",
+		" \t\"a\"\r\n\n":                 "a",
+		"{/misc.territory[SE].alpha2}":   "SE",
+		"{/misc.territory[SE].alpha2}\n": "SE\n",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
+		}
+	}
+	if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
+		t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
+	}
+	if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
+		t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
+	}
+}
+
+func TestRunRendersBlankStdin(t *testing.T) {
+	for _, stdin := range []string{"", "\n", " \r\n\t"} {
+		if code, out, errb := runOut(stdin); code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunTextPrintsAsWritten(t *testing.T) {
+	for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
+		code, out, errb := runOut(stdin)
+		if code != 0 || out != stdin {
+			t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
+		}
+	}
+}
+
+func TestRunMultipleDataPaths(t *testing.T) {
+	code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
+	if code != 0 {
+		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
+	}
+	if strings.TrimSpace(out) == "" {
+		t.Fatal("empty output for multi-dir run")
+	}
+}
+
+func TestRunUnknownCategoryFails(t *testing.T) {
+	code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if !strings.Contains(errb, "nope") {
+		t.Errorf("stderr %q should name the unknown category", errb)
+	}
+}
+
+func TestRunList(t *testing.T) {
+	code, out, errb := runOut("", "--data-path", svSE, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	for _, want := range []string{"person", "person.last", "address"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("list output missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRunHelpOnStdout(t *testing.T) {
+	for _, h := range []string{"-h", "--help"} {
+		code, out, errb := runOut("", h)
+		if code != 0 {
+			t.Errorf("%s = %d, want 0", h, code)
+		}
+		if !strings.Contains(out, "Usage") || errb != "" {
+			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
+		}
+	}
+}
+
+func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
+	fejkdata := func(version string, replace *debug.Module) *debug.Module {
+		return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
+	}
+	mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
+	for _, c := range []struct {
+		info *debug.BuildInfo
+		want string
+	}{
+		{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
+		{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
+		{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
+		{nil, "fejkdata devel"},
+	} {
+		if got := versionLine("mycli", c.info); got != c.want {
+			t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
+		}
+	}
+}
+
+func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
+	for arg0, want := range map[string]string{
+		"/usr/local/bin/mycli": "mycli",
+		"mycli.EXE":            "mycli",
+		"mycli.exe":            "mycli",
+		"my.cli":               "my.cli",
+		"":                     "fejkdata",
+	} {
+		if got := commandName(arg0); got != want {
+			t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
+		}
+	}
+}
+
+func TestRunNamesItselfByTheCommandName(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
+		t.Errorf("mycli --nope = %d, %q", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
+		t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
+	}
+	errb.Reset()
+	if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
+		t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
+	}
+	if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
+		t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	code, out, errb := runOut("", "--version")
+	if code != 0 {
+		t.Fatalf("--version = %d, stderr=%q", code, errb)
+	}
+	version, ok := strings.CutPrefix(out, "fejkdata ")
+	if !ok || strings.TrimSpace(version) == "" {
+		t.Errorf("--version = %q, want the command name and a version on stdout", out)
+	}
+}
+
+func TestRunMissingDirFails(t *testing.T) {
+	code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
+	if code != 1 {
+		t.Fatalf("run = %d, want 1", code)
+	}
+	if errb == "" {
+		t.Error("want an error message on stderr")
+	}
+}
+
+func TestRunShippedDataByDefault(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
+	}
+	code, list, _ := runOut("", "--list")
+	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
+		t.Errorf("--list without --data-path = %d, %q", code, list)
+	}
+	code, _, errb = runOut("{/person}")
+	if code != 2 || !strings.Contains(errb, "person") {
+		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
+	}
+}
+
+func TestRunSelectsATableRow(t *testing.T) {
+	dir := t.TempDir()
+	for name, content := range map[string]string{
+		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
+		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
+		t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
+	}
+	if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
+		t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
+	}
+	if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
+		t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
+	}
+	if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
+		t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
+	}
+	if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
+		t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
+	}
+}
+
+func TestUsageExamplesRenderWithNoData(t *testing.T) {
+	examples := regexp.MustCompile(`(?m)^  echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n  "+pipeHint("COMMAND")+"\n", -1)
+	if len(examples) < 3 {
+		t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
+	}
+	for _, e := range examples {
+		if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
+			t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
+		}
+	}
+	for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
+		if token != "{/path}" && token != "{/table[KEY]}" {
+			t.Errorf("usage names %s, a path a custom CLI may not carry", token)
+		}
+	}
+	if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
+		t.Errorf("usage does not list the exit codes")
+	}
+}
+
+func TestRunOperandMisuseBeforeLoad(t *testing.T) {
+	code, _, errb := runOut("", "--no-shipped-data", "x")
+	if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
+		t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
+	}
+}
+
+func TestRunInlineTemplate(t *testing.T) {
+	code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
+	if code != 0 || (out != "name: bosse" && out != "name: lina") {
+		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
+	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
+		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
+	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
+		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
+	}
+	code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
+	if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
+		t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
+	}
+}
+
+func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
+	for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
+		if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
+			t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
+		}
+	}
+	if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
+		t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
+	}
+}
+
+func TestRunTemplateMisuse(t *testing.T) {
+	for stdin, want := range map[string]string{
+		"{bad":                  "unterminated",
+		"name: {/no.such.path}": "no entry",
+		"a } b":                 "}}",
+		"{//sv_SE.person}":      "write {/sv_SE.person}",
+	} {
+		code, out, errb := runOut(stdin, "--seed", "1")
+		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
+			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
+		}
+		if strings.Contains(errb, "fejkdata: fejkdata:") {
+			t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
+		}
+	}
+}
+
+func TestRunNoShippedData(t *testing.T) {
+	code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
+		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
+	}
+	code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
+	if code != 0 || strings.TrimSpace(out) == "" {
+		t.Errorf("run = %d, out=%q", code, out)
+	}
+	if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
+		t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
+	}
+	if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
+		t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
+	}
+}
+
+func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{nil, ""},
+		{[]string{"--format", "json"}, "[]\n"},
+		{[]string{"--format", "ndjson"}, ""},
+		{[]string{"--format", "csv"}, "first,last\n"},
+		{[]string{"--format", "sql"}, ""},
+	} {
+		if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
+			t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
+		}
+	}
+	if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
+		t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
+	}
+}
+
+func TestRunRepeatIsBounded(t *testing.T) {
+	for _, args := range [][]string{
+		{"--repeat", "9223372036854775807"},
+		{"--repeat", "1048577"},
+		{"-n", "99999999999"},
+	} {
+		code, out, errb := runOut("{/sv_SE.word}", args...)
+		if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
+			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
+		}
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
+	if code != 0 || len(out) != 1048576 {
+		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
+	}
+}
+
+func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
+	dir := recordDir(t)
+	for _, c := range []struct {
+		stdin         string
+		base, ignored []string
+	}{
+		{"", []string{"--list"}, []string{"-n", "3"}},
+		{"", []string{"--list"}, []string{"--separator", ","}},
+		{"", []string{"--list"}, []string{"--format", "json"}},
+		{"", []string{"--list"}, []string{"--table", "t"}},
+		{"{/users}", nil, []string{"--table", "t"}},
+		{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
+	} {
+		args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
+		_, want, _ := runOut(c.stdin, args...)
+		code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
+		if code != 0 || want == "" || got != want {
+			t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
+		}
+	}
+}
+
+func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
+	if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
+		t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
+	}
+}
+
+func TestRunRendersAnyJSONValue(t *testing.T) {
+	for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
+		if code, out, errb := runOut(stdin); code != 0 || out != want {
+			t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
+		}
+	}
+}
+
+func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "-ä")
+	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
+		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
+	}
+}
+
+func TestRunEmptyDataPathIsNamed(t *testing.T) {
+	code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
+	if code != 1 || !strings.Contains(errb, "empty") {
+		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
+	}
+}
+
+func recordDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
+	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestRunRecordJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
+		}
+	}
+}
+
+func TestRunRecordCSVRoundTrips(t *testing.T) {
+	code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 4 {
+		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
+	}
+	if lines[0] != "first,last" {
+		t.Fatalf("csv header = %q, want first,last", lines[0])
+	}
+	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
+	if err != nil {
+		t.Fatalf("csv output is not valid CSV: %v", err)
+	}
+	if len(r) != 4 || len(r[0]) != 2 {
+		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
+	}
+}
+
+func TestRunRecordSQL(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
+		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
+	}
+}
+
+func TestRunRecordDefaultTable(t *testing.T) {
+	code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
+	}
+}
+
+func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
+	code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
+	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
+		t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
+	}
+	code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
+	if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
+		t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
+	}
+}
+
+func TestRunRecordInlineTemplate(t *testing.T) {
+	code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
+	if code != 0 {
+		t.Fatalf("inline record = %d, stderr=%q", code, errb)
+	}
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
+	}
+}
+
+func TestRunRecordMisuse(t *testing.T) {
+	for _, c := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
+		{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
+		{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
+		{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
+		{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
+	} {
+		code, out, errb := runOut("{/users}", c.args...)
+		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
+			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
+		}
+	}
+}
+
+func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
+	if code != 1 {
+		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
+	}
+}
+
+func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
+	dir := t.TempDir()
+	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
+	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
+	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
+	}
+	code, _, errb = runOut(content, "--format", "json")
+	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
+		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
+	}
+	if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
+		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
+	}
+}
+
+func TestRunNumbersFollowTheShell(t *testing.T) {
+	_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
+	code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
+	if code != 0 || got != want {
+		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	}
+}
+
+// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
+// for columns: the record to write, and the quoting a shell needs to take it. The
+// template is well-formed and only the data cannot serve it, so the code is 1.
+func TestRunRecordOfAFieldlessPath(t *testing.T) {
+	code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
+	if code != 1 {
+		t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
+	}
+	for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
+		if !strings.Contains(errb, want) {
+			t.Errorf("stderr = %q, want it to name %s", errb, want)
+		}
+	}
+	if out != "" {
+		t.Errorf("stdout = %q, want nothing", out)
+	}
+}
+
+func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
+	dir := t.TempDir()
+	for name, data := range map[string]string{
+		".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
+		"bad.json":       `"{/nope}"`,
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
+		if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
+			t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
+		}
+	}
+}

cli/lone_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// grammarLoneReference is the path a format reads when the grammar parses it as one path
+// read of one reference from the root, and nothing else.
+func grammarLoneReference(text string) (string, bool) {
+	toks, err := grammar.ParseFormat(text)
+	if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
+		return "", false
+	}
+	sigil, path, err := grammar.RefShape(toks[0].Arms[0])
+	if err != nil || sigil != "/" {
+		return "", false
+	}
+	return path, true
+}
+
+func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
+	for _, text := range []string{
+		" {/x}",
+		"x",
+		"{.x}",
+		"{/.x}",
+		"{//x}",
+		"{/a...b}",
+		"{/a..b}",
+		"{/a..}",
+		"{/a.[x]}",
+		"{/a.}",
+		"{/a|/b}",
+		"{/a}{/b}",
+		"{/misc.currency[US Dollar (Next day)]}",
+		"{/sv_SE.person}",
+		"{/t[a as b].c}",
+		"{/t[f(x)]}",
+		"{/x as p}",
+		"{/x(1)}",
+		"{/x}}",
+		"{/}",
+	} {
+		path, lone := loneReference(text)
+		want, wantLone := grammarLoneReference(text)
+		if lone != wantLone || path != want {
+			t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
+		}
+	}
+}

cmd/fejkdata/main.go

@@ -1,495 +1,12 @@
-// Command fejkdata prints fake values from the shipped data and any directories
-// layered over it.
-//
-//	fejkdata sv_SE.person                        # a full person
-//	fejkdata sv_SE.person.last                   # just the surname
-//	fejkdata --data-path ./mydata sv_SE.person   # layer custom data; the last dir wins
-//	fejkdata --seed 42 sv_SE.address
+// Command fejkdata renders the template on its stdin with fake values from every shipped
+// module; package cli documents it.
 package main
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
 	"os"
-	"runtime/debug"
-	"sort"
-	"strconv"
-	"strings"
 
-	"gitea.larvit.se/larvit/fejkdata"
+	"github.com/larvit/fejkdata/cli"
+	"github.com/larvit/fejkdata/data"
 )
 
-const usage = `Usage: fejkdata [flags] <path|template>
-
-  <path>                 a category, or a dotted path into one (person, person.last);
-                         a table's row by key or name: 'misc.country[SE]', 'misc.country[Sweden].tld'
-  <template>             a format string or JSON value to render inline, e.g.
-                         'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
-
-An argument containing a { token, or a JSON object, array or string, is a
-template; any other argument is a path (a path never contains a brace or a quote,
-and a bracket only as a [selector] after a table's name). Templates reach the
-data by reference from the root — {/sv_SE.person.last} — whether the data is
-shipped or layered with --data-path. An argument carrying a closing brace or a
-quote but no valid JSON names neither.
-
-With --format json, ndjson, csv or sql the argument must name a record — a
-template whose fields are its columns — and the rows are written as one JSON
-array, one JSON object per line, one CSV row (after a header), or one INSERT.
-
-  -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, ndjson, csv or sql
-  -h, --help             print this help, then exit
-      --list             list the paths the data offers, then exit
-      --no-shipped-data  load only the --data-path directories
-  -n, --repeat N         render the value N times, 1..1048576 (default 1)
-  -s, --seed N           seed for reproducible output
-      --separator S      string between repeated values (default newline)
-      --table T          the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
-      --version          print the version, then exit
-
-Flags may come before or after <path|template>; -- ends the flags. A short flag's
-value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
-`
-
-type invocation struct {
-	dirs         []string
-	format       string
-	formatSet    bool
-	help         bool
-	list         bool
-	noShipped    bool
-	paths        []string
-	repeat       int
-	repeatSet    bool
-	seed         uint64
-	seeded       bool
-	separator    string
-	separatorSet bool
-	table        string
-	tableSet     bool
-	version      bool
-}
-
-type flagDef struct {
-	long  string
-	short string
-	value bool
-	set   func(*invocation, string) error
-}
-
-var flagDefs = []flagDef{
-	{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
-	{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
-	{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
-	{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
-	{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
-	{"repeat", "n", true, func(in *invocation, v string) error {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > fejkdata.MaxRepeat {
-			return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
-		}
-		in.repeat, in.repeatSet = n, true
-		return nil
-	}},
-	{"seed", "s", true, func(in *invocation, v string) error {
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
-		}
-		in.seed, in.seeded = n, true
-		return nil
-	}},
-	{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
-	{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
-	{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
-}
-
-func flagByLong(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].long == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-func flagByShort(name string) *flagDef {
-	for i := range flagDefs {
-		if flagDefs[i].short == name {
-			return &flagDefs[i]
-		}
-	}
-	return nil
-}
-
-// flagArg is one flag as written: its definition and the value attached to it,
-// if any.
-type flagArg struct {
-	def    *flagDef
-	value  string
-	inline bool
-}
-
-// splitFlags reads one argv element as flags: --name or --name=value, or -abc
-// where each letter is a short flag and the first that takes a value takes the
-// rest of the element.
-func splitFlags(arg string) ([]flagArg, error) {
-	if strings.HasPrefix(arg, "--") {
-		name, value, inline := strings.Cut(arg[2:], "=")
-		def := flagByLong(name)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag --%s", name)
-		}
-		return []flagArg{{def, value, inline}}, nil
-	}
-	if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
-		return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
-	}
-	var flags []flagArg
-	letters := arg[1:]
-	for i, r := range letters {
-		letter := string(r)
-		def := flagByShort(letter)
-		if def == nil {
-			return nil, fmt.Errorf("unknown flag -%s", letter)
-		}
-		if !def.value {
-			flags = append(flags, flagArg{def: def})
-			continue
-		}
-		rest := letters[i+len(letter):]
-		if strings.HasPrefix(rest, "=") {
-			return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
-				letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
-		}
-		return append(flags, flagArg{def, rest, rest != ""}), nil
-	}
-	return flags, nil
-}
-
-func parseArgs(argv []string) (invocation, error) {
-	in := invocation{repeat: 1, separator: "\n", format: "text"}
-	for i := 0; i < len(argv); i++ {
-		arg := argv[i]
-		if arg == "--" {
-			in.paths = append(in.paths, argv[i+1:]...)
-			return in, nil
-		}
-		if len(arg) < 2 || arg[0] != '-' {
-			in.paths = append(in.paths, arg)
-			continue
-		}
-		flags, err := splitFlags(arg)
-		if err != nil {
-			return in, err
-		}
-		for _, fl := range flags {
-			if !fl.def.value {
-				if fl.inline {
-					return in, fmt.Errorf("--%s takes no value", fl.def.long)
-				}
-				_ = fl.def.set(&in, "")
-				continue
-			}
-			value := fl.value
-			if !fl.inline {
-				if i++; i >= len(argv) {
-					return in, fmt.Errorf("--%s needs a value", fl.def.long)
-				}
-				value = argv[i]
-			}
-			if err := fl.def.set(&in, value); err != nil {
-				return in, err
-			}
-		}
-	}
-	return in, nil
-}
-
-// recordFormat is one way to write a record out: the line each record renders,
-// the header that precedes the first one, and the open/close frame plus the
-// between-record separator a document form needs.
-type recordFormat struct {
-	header func(*fejkdata.Record) string
-	line   func(r *fejkdata.Record, table string) string
-	open   string
-	close  string
-	sep    string
-}
-
-// recordFormats is every --format that writes records. json frames the records
-// as one array document; ndjson is the same column, one object per line.
-var recordFormats = map[string]recordFormat{
-	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
-	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
-	"ndjson": {line: jsonLine, sep: "\n"},
-	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
-}
-
-func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
-
-// writesRecords reports whether the format writes records rather than plain text.
-func (in invocation) writesRecords() bool {
-	return in.format != "text"
-}
-
-// formatNames lists the --format values, text included, for the misuse error.
-func formatNames() string {
-	names := []string{"text"}
-	for name := range recordFormats {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-	return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
-}
-
-// checkFlags rejects a flag value or combination that cannot run.
-func (in invocation) checkFlags() error {
-	if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
-		return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
-	}
-	if in.tableSet && in.table == "" {
-		return errors.New("--table names the INSERT target, so it cannot be empty")
-	}
-	if in.tableSet && in.format != "sql" {
-		return errors.New("--table names the INSERT target, so it needs --format sql")
-	}
-	if in.writesRecords() && in.separatorSet {
-		return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
-	}
-	return nil
-}
-
-// check rejects a flag combination or an argument that cannot run, and reports
-// what the argument names, so its shape is settled before any data is read.
-func (in invocation) check() (argKind, error) {
-	if err := in.checkFlags(); err != nil {
-		return argPath, err
-	}
-	if in.list && len(in.paths) > 0 {
-		return argPath, errors.New("--list takes no path")
-	}
-	if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
-		return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
-	}
-	if in.list {
-		return argPath, nil
-	}
-	if len(in.paths) != 1 {
-		return argPath, fmt.Errorf("expected one path or template, got %d", len(in.paths))
-	}
-	return classify(in.paths[0])
-}
-
-func (in invocation) options() []fejkdata.Option {
-	var opts []fejkdata.Option
-	if in.noShipped {
-		opts = append(opts, fejkdata.WithoutShippedData())
-	}
-	for _, dir := range in.dirs {
-		opts = append(opts, fejkdata.WithDataPath(dir))
-	}
-	if in.seeded {
-		opts = append(opts, fejkdata.WithSeed(in.seed))
-	}
-	return opts
-}
-
-// write streams the argument's renders to w, repeat of them joined by the
-// separator and ended by a newline. A value that renders once renders every time,
-// so a render failure comes before anything is written; a write failure surfaces
-// from Flush, bufio keeping the first one.
-func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	if in.writesRecords() {
-		return in.writeRecords(f, kind, w)
-	}
-	draw, err := in.textDraw(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	out := bufio.NewWriter(w)
-	for i := 0; i < in.repeat; i++ {
-		v, err := draw()
-		if err != nil {
-			return err
-		}
-		if i > 0 {
-			out.WriteString(in.separator)
-		}
-		out.WriteString(v)
-	}
-	out.WriteString("\n")
-	return out.Flush()
-}
-
-// textDraw builds what one text render yields: the value, from a path or an
-// inline template.
-func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if kind != argTemplate {
-		return func() (string, error) { return f.Fake(arg) }, nil
-	}
-	t, err := f.NewTemplate(arg)
-	if err != nil {
-		return nil, templateError{err}
-	}
-	return func() (string, error) { return t.Fake(), nil }, nil
-}
-
-// recordStream builds the record drawer for the argument, plus the INSERT table
-// a sql format names.
-func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
-	record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
-	if kind == argTemplate {
-		t, err := f.NewRecordTemplate(arg)
-		if err != nil {
-			return nil, "", templateError{err}
-		}
-		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
-	}
-	table := in.table
-	if table == "" {
-		table = defaultTable(arg, kind)
-	}
-	return record, table, nil
-}
-
-// writeRecords streams a record per line in the chosen format, framing a document
-// form with its open/close brackets and a header preceding the first record.
-func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	record, table, err := in.recordStream(f, kind, in.paths[0])
-	if err != nil {
-		return err
-	}
-	format := recordFormats[in.format]
-	out := bufio.NewWriter(w)
-	if format.open != "" {
-		out.WriteString(format.open)
-		out.WriteByte('\n')
-	}
-	for i := 0; i < in.repeat; i++ {
-		r, err := record()
-		if err != nil {
-			return err
-		}
-		if i == 0 && format.header != nil {
-			out.WriteString(format.header(r))
-			out.WriteByte('\n')
-		}
-		if i > 0 {
-			out.WriteString(format.sep)
-		}
-		out.WriteString(format.line(r, table))
-	}
-	if format.close != "" {
-		out.WriteByte('\n')
-		out.WriteString(format.close)
-	}
-	out.WriteByte('\n')
-	return out.Flush()
-}
-
-// defaultTable names the INSERT target when --table is absent: the path's last
-// segment, or "records" for an inline template that sits in no folder.
-func defaultTable(arg string, kind argKind) string {
-	if kind == argTemplate {
-		return "records"
-	}
-	var names strings.Builder // the path with its [selectors] cut out
-	for depth, i := 0, 0; i < len(arg); i++ {
-		switch {
-		case arg[i] == '[':
-			depth++
-		case arg[i] == ']':
-			depth--
-		case depth == 0:
-			names.WriteByte(arg[i])
-		}
-	}
-	segments := strings.Split(names.String(), ".")
-	return segments[len(segments)-1]
-}
-
-// templateError marks a render failure that is the argument's own fault — an
-// inline template that does not compile. run reports it as misuse (exit 2, with a
-// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
-type templateError struct{ error }
-
-func (e templateError) Unwrap() error { return e.error }
-
-type argKind int
-
-const (
-	argPath argKind = iota
-	argTemplate
-)
-
-// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
-func classify(arg string) (argKind, error) {
-	inline, err := fejkdata.IsTemplate(arg)
-	if inline {
-		return argTemplate, nil
-	}
-	return argPath, err
-}
-
-func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
-
-// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
-func run(args []string, stdout, stderr io.Writer) int {
-	in, err := parseArgs(args)
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	if in.help {
-		fmt.Fprint(stdout, usage)
-		return 0
-	}
-	if in.version {
-		fmt.Fprintln(stdout, "fejkdata "+buildVersion())
-		return 0
-	}
-	kind, err := in.check()
-	if err != nil {
-		return misuse(stderr, err)
-	}
-	f, err := fejkdata.New(in.options()...)
-	if errors.Is(err, fejkdata.ErrNoData) {
-		return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
-	}
-	if err != nil {
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	if in.list {
-		for _, p := range f.List() {
-			fmt.Fprintln(stdout, p)
-		}
-		return 0
-	}
-	if err := in.write(f, kind, stdout); err != nil {
-		var te templateError
-		if errors.As(err, &te) {
-			return misuse(stderr, te.error)
-		}
-		fmt.Fprintln(stderr, err)
-		return 1
-	}
-	return 0
-}
-
-func misuse(stderr io.Writer, err error) int {
-	// A library error already names the program, so the prefix is not doubled.
-	fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
-	return 2
-}
-
-// buildVersion is the module version go install stamps into the binary, or "devel".
-func buildVersion() string {
-	if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
-		return info.Main.Version
-	}
-	return "devel"
-}
+func main() { os.Exit(cli.Main(data.Modules()...)) }

cmd/fejkdata/main_test.go

@@ -2,614 +2,46 @@ package main
 
 import (
 	"bytes"
-	"encoding/csv"
-	"encoding/json"
 	"os"
-	"path/filepath"
-	"regexp"
+	"os/exec"
 	"strings"
 	"testing"
 )
 
-const (
-	svSE = "../../data/sv_SE"
-	enUS = "../../data/en_US"
-	misc = "../../data/misc"
-)
-
-func runOut(args ...string) (int, string, string) {
-	var out, errb bytes.Buffer
-	code := run(args, &out, &errb)
-	return code, out.String(), errb.String()
+func TestMain(m *testing.M) {
+	if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
+		main()
+	}
+	os.Exit(m.Run())
 }
 
-func TestRunOutputsValue(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatalf("empty output, stderr=%q", errb)
-	}
-	if !strings.HasSuffix(out, "\n") {
-		t.Errorf("output should end with newline, got %q", out)
+func TestMainRendersTheShippedData(t *testing.T) {
+	cmd := exec.Command(os.Args[0], "--seed", "1")
+	cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
+	cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	out, err := cmd.Output()
+	if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
+		t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
 	}
 }
 
-func TestRunDotPath(t *testing.T) {
-	code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("person run = %d", code)
+func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
+	var outs [2]string
+	for i := range outs {
+		cmd := exec.Command(os.Args[0])
+		cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
+		cmd.Stdin = strings.NewReader("{uuid()}")
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+		out, err := cmd.Output()
+		if err != nil || len(out) == 0 {
+			t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
 		}
-	code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
-	if code != 0 {
-		t.Fatalf("person.last run = %d, stderr=%q", code, errb)
+		outs[i] = string(out)
 	}
-	if strings.TrimSpace(last) == "" {
-		t.Fatal("empty person.last output")
-	}
-	if last == full {
-		t.Errorf("person.last %q should differ from person %q", last, full)
-	}
-}
-
-func TestRunSeedSpellings(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"--seed=42", "--data-path", svSE, "address"},
-		{"-s", "42", "-d", svSE, "address"},
-		{"--data-path", svSE, "address", "--seed", "42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-}
-
-func TestRunShortFlagValues(t *testing.T) {
-	_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
-	for _, args := range [][]string{
-		{"-s42", "-d", svSE, "address"},
-		{"-s", "42", "-d" + svSE, "address"},
-		{"-d", svSE, "address", "-s42"},
-	} {
-		code, got, errb := runOut(args...)
-		if code != 0 {
-			t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
-		}
-		if got != want {
-			t.Errorf("run(%v) = %q, want %q", args, got, want)
-		}
-	}
-	_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
-	if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
-		t.Errorf("-n3 gave %d lines: %q", len(lines), three)
-	}
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
-		{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
-		{[]string{"-d=" + svSE, "address"}, "--data-path="},
-		{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
-		{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" {
-			t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-	code, out, _ := runOut("-hd", svSE)
-	if code != 0 || !strings.Contains(out, "Usage") {
-		t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
-	}
-}
-
-func TestRunDoubleDashEndsFlags(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--", "person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, _, errb = runOut("--data-path", svSE, "--", "--list")
-	if code != 1 || !strings.Contains(errb, "--list") {
-		t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestRunSingleDashLongIsRejected(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
-		{[]string{"-data-path", svSE, "person"}, "use --data-path"},
-		{[]string{"-list", "-d", svSE}, "use --list"},
-		{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
-		{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunFlagValues(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
-		{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
-		{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
-		{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
-		{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
-		{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
-	} {
-		code, _, errb := runOut(c.args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", c.args, code)
-		}
-		if !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
-		}
-	}
-}
-
-func TestRunRepeat(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 3 {
-		t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
-	}
-	_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
-	if short != out {
-		t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
-	}
-}
-
-func TestRunSeparator(t *testing.T) {
-	code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if n := strings.Count(out, "\n"); n != 1 {
-		t.Errorf("want one trailing newline, got %d: %q", n, out)
-	}
-	if !strings.Contains(out, ",") {
-		t.Errorf("values should be comma-joined: %q", out)
-	}
-}
-
-func TestRunRepeatAdvancesRNG(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	uniq := map[string]bool{}
-	for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
-		uniq[l] = true
-	}
-	if len(uniq) < 2 {
-		t.Errorf("repeat should vary output, all identical: %q", out)
-	}
-}
-
-func TestRunMisuse(t *testing.T) {
-	for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
-		code, out, errb := runOut(args...)
-		if code != 2 {
-			t.Errorf("run(%v) = %d, want 2", args, code)
-		}
-		if !strings.Contains(errb, "try 'fejkdata --help'") {
-			t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
-		}
-		if out != "" {
-			t.Errorf("run(%v) stdout = %q, want nothing", args, out)
-		}
-	}
-}
-
-func TestRunMultipleDataPaths(t *testing.T) {
-	code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
-	if code != 0 {
-		t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
-	}
-	if strings.TrimSpace(out) == "" {
-		t.Fatal("empty output for multi-dir run")
-	}
-}
-
-func TestRunUnknownCategoryFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", svSE, "nope")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if !strings.Contains(errb, "nope") {
-		t.Errorf("stderr %q should name the unknown category", errb)
-	}
-}
-
-func TestRunList(t *testing.T) {
-	code, out, errb := runOut("--data-path", svSE, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	for _, want := range []string{"person", "person.last", "address"} {
-		if !strings.Contains(out, want) {
-			t.Errorf("list output missing %q:\n%s", want, out)
-		}
-	}
-}
-
-func TestRunHelpOnStdout(t *testing.T) {
-	for _, h := range []string{"-h", "--help"} {
-		code, out, errb := runOut(h)
-		if code != 0 {
-			t.Errorf("%s = %d, want 0", h, code)
-		}
-		if !strings.Contains(out, "Usage") || errb != "" {
-			t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
-		}
-	}
-}
-
-func TestRunVersion(t *testing.T) {
-	code, out, errb := runOut("--version")
-	if code != 0 {
-		t.Fatalf("--version = %d, stderr=%q", code, errb)
-	}
-	version, ok := strings.CutPrefix(out, "fejkdata ")
-	if !ok || strings.TrimSpace(version) == "" {
-		t.Errorf("--version = %q, want the command name and a version on stdout", out)
-	}
-}
-
-func TestRunMissingDirFails(t *testing.T) {
-	code, _, errb := runOut("--data-path", "../../data/nope", "person")
-	if code != 1 {
-		t.Fatalf("run = %d, want 1", code)
-	}
-	if errb == "" {
-		t.Error("want an error message on stderr")
-	}
-}
-
-func TestRunShippedDataByDefault(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "sv_SE.person")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
-	}
-	code, list, _ := runOut("--list")
-	if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
-		t.Errorf("--list without --data-path = %d, %q", code, list)
-	}
-	code, _, errb = runOut("person")
-	if code != 1 || !strings.Contains(errb, "person") {
-		t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
-	}
-}
-
-func TestClassify(t *testing.T) {
-	for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
-		if got, err := classify(arg); err != nil || got != want {
-			t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
-		}
-	}
-	if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
-		t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
-	}
-	if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
-		t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
-	}
-}
-
-func TestRunSelectsATableRow(t *testing.T) {
-	dir := t.TempDir()
-	for name, content := range map[string]string{
-		"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
-		"region.tsv":  "code\tname\n01\tStockholms län\n12\tSkåne län\n",
-	} {
-		if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
-		t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
-		t.Fatalf("region[Skåne län].code = %d, %q", code, out)
-	}
-	if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
-		t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
-		t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
-	}
-	if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
-		t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.country[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
-		t.Fatalf("--format csv misc.country[SE] = %d, %q", code, out)
-	}
-}
-
-func TestUsageReferencesResolve(t *testing.T) {
-	for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
-		path := token[2 : len(token)-1]
-		code, out, errb := runOut("--seed", "1", path)
-		if code != 0 || strings.TrimSpace(out) == "" {
-			t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
-		}
-	}
-}
-
-func TestRunShapeMisuseBeforeLoad(t *testing.T) {
-	code, _, errb := runOut("--no-shipped-data", "[abc]")
-	if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
-		t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
-	}
-}
-
-func TestRunInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
-	if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
-		t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
-	if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
-		t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
-	}
-	code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
-	if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
-		t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
-	}
-}
-
-func TestRunTemplateMisuse(t *testing.T) {
-	for arg, want := range map[string]string{
-		"{bad":                  "unterminated",
-		"[red,green]":           `starts with "["`,
-		`{"format":"x"}`:        "is a string",
-		"name: {/no.such.path}": "no entry",
-		"{/sv_SE.person}":       "write sv_SE.person",
-		"x[1]y":                 `"]"`,
-		` ["a","b"] `:           "may not be padded",
-	} {
-		code, out, errb := runOut("--seed", "1", arg)
-		if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
-			t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
-		}
-		if strings.Contains(errb, "fejkdata: fejkdata:") {
-			t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
-		}
-	}
-}
-
-func TestRunNoShippedData(t *testing.T) {
-	code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
-		t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
-	}
-	code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
-	if code != 0 || strings.TrimSpace(out) == "" {
-		t.Errorf("run = %d, out=%q", code, out)
-	}
-	code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
-	if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
-		t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
-	}
-}
-
-func TestRunRepeatIsBounded(t *testing.T) {
-	for _, args := range [][]string{
-		{"--repeat", "9223372036854775807", "sv_SE.word"},
-		{"--repeat", "1048577", "sv_SE.word"},
-		{"-n", "99999999999", "sv_SE.word"},
-	} {
-		code, out, errb := runOut(args...)
-		if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
-			t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
-		}
-	}
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
-	if code != 0 || len(out) != 1048576+1 {
-		t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
-	}
-}
-
-func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
-	for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
-		code, _, errb := runOut(args...)
-		if code != 2 || !strings.Contains(errb, "--list takes no") {
-			t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
-		}
-	}
-}
-
-func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
-	code, _, errb := runOut("-ä", "sv_SE.word")
-	if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
-		t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
-	}
-}
-
-func TestRunEmptyDataPathIsNamed(t *testing.T) {
-	code, _, errb := runOut("--data-path=", "sv_SE.word")
-	if code != 1 || !strings.Contains(errb, "empty") {
-		t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
-	}
-}
-
-func recordDir(t *testing.T) string {
-	t.Helper()
-	dir := t.TempDir()
-	content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
-	if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
-func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil {
-		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
-	}
-	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
-		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
-	}
-}
-
-func TestRunRecordNDJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
-	}
-	for _, line := range lines {
-		var m map[string]string
-		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
-			t.Fatalf("ndjson line %q is not one object: %v", line, err)
-		}
-	}
-}
-
-func TestRunRecordCSVRoundTrips(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 4 {
-		t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
-	}
-	if lines[0] != "first,last" {
-		t.Fatalf("csv header = %q, want first,last", lines[0])
-	}
-	r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
-	if err != nil {
-		t.Fatalf("csv output is not valid CSV: %v", err)
-	}
-	if len(r) != 4 || len(r[0]) != 2 {
-		t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
-	}
-}
-
-func TestRunRecordSQL(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
-	if code != 0 {
-		t.Fatalf("run = %d, stderr=%q", code, errb)
-	}
-	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
-	if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
-		t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
-	}
-}
-
-func TestRunRecordDefaultTable(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
-	if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
-		t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
-	}
-}
-
-func TestRunRecordInlineTemplate(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
-	if code != 0 {
-		t.Fatalf("inline record = %d, stderr=%q", code, errb)
-	}
-	var arr []map[string]string
-	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
-		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
-	}
-}
-
-func TestRunRecordMisuse(t *testing.T) {
-	for _, c := range []struct {
-		args []string
-		want string
-	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
-		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
-		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
-		{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
-	} {
-		code, out, errb := runOut(c.args...)
-		if code != 2 || out != "" || !strings.Contains(errb, c.want) {
-			t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
-		}
-	}
-}
-
-func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
-	if code != 1 {
-		t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
-	}
-}
-
-func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
-	dir := t.TempDir()
-	content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
-	if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
-	if code != 1 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
-	}
-	code, _, errb = runOut("--format", "json", content)
-	if code != 2 || !strings.Contains(errb, "carries repeat 3") {
-		t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
-	}
-	if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
-		t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
-	}
-}
-
-func TestRunNumbersFollowTheShell(t *testing.T) {
-	_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
-	code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
-	if code != 0 || got != want {
-		t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
+	if outs[0] == outs[1] {
+		t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
 	}
 }

compose.yaml

@@ -1,21 +1,5 @@
-# Docker-based dev workflow — no local tooling beyond Docker is required.
-#
-#   docker compose run --rm test
-#   docker compose run --rm vet
-#   docker compose run --rm build
-#
-# The full gate — vet, formatting and tests — is `docker build .`, which is what
-# CI runs.
-#
-# Build caches persist in the `gocache` volume, so re-runs are fast and the
-# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
-# ownership when run with `--user "$(id -u):$(id -g)"`.
-#
-# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
-# supported Go with: GO_VERSION=1.22 docker compose run --rm test
-
 x-go: &go
-  image: golang:${GO_VERSION:-1.26.4}
+  image: golang:${GO_VERSION:-1.27.2}
   working_dir: /app
   volumes:
     - .:/app
@@ -33,12 +17,16 @@ services:
 
   cover:
     <<: *go
-    command: go test -cover ./...
+    command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
 
   bench:
     <<: *go
     command: go test -run XXX -bench . -benchmem ./...
 
+  generate:
+    <<: *go
+    command: go generate ./...
+
   vet:
     <<: *go
     command: go vet ./...
@@ -47,6 +35,10 @@ services:
     <<: *go
     command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
 
+  vulncheck:
+    <<: *go
+    command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
+
   build:
     <<: *go
     command: go build ./...
@@ -59,13 +51,12 @@ services:
     <<: *go
     command: go mod tidy
 
-  # Interactive shell for ad-hoc work: docker compose run --rm dev
   dev:
     <<: *go
     command: sh
 
   data-import:
-    image: python:3.14.7-slim
+    image: python:3.14.8-slim
     working_dir: /app
     volumes:
       - .:/app

data-import/currency.py

@@ -1,35 +1,29 @@
 #!/usr/bin/env python3
 """Rebuild data/misc/currency.tsv from datasets/currency-codes (PDDL) and CLDR's symbols (Unicode).
 
-    data-import/currency.py [--source URL_OR_FILE] [--symbols URL_OR_FILE ...] [--out FILE]
+    data-import/currency.py [--source URL_OR_FILE] [--symbols URL_OR_FILE ...] [--cache DIR] [--out FILE]
 
 The symbols come from the first locale file that has one, narrow symbols before wide.
 """
 import argparse
 import csv
 import io
-import re
-import sys
-import urllib.request
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
+import source
+import tsv
+
 SOURCE = "https://raw.githubusercontent.com/datasets/currency-codes/main/data/codes-all.csv"
 SYMBOLS = [
     "https://raw.githubusercontent.com/unicode-org/cldr/main/common/main/en.xml",
     "https://raw.githubusercontent.com/unicode-org/cldr/main/common/main/root.xml",
 ]
 OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "currency.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
 COLUMNS = ["code", "decimals", "name", "numeric", "symbol"]
 
 
-def read(source):
-    if re.match(r"^https?://", source):
-        with urllib.request.urlopen(source, timeout=60) as r:
-            return r.read().decode("utf-8")
-    return Path(source).read_text(encoding="utf-8")
-
-
 def symbols(xml_texts):
     """CLDR's symbol per code: the first locale's narrow symbol, else the first locale's wide one."""
     narrow, wide = {}, {}
@@ -58,24 +52,16 @@ def rows(text, symbol):
         }
 
 
-def write(out, table):
-    lines = ["\t".join(COLUMNS)]
-    for row in sorted(table, key=lambda r: r["code"]):
-        cells = [row[c] for c in COLUMNS]
-        assert all(cells) and not any("\t" in c or "\n" in c for c in cells), row
-        lines.append("\t".join(cells))
-    Path(out).write_text("\n".join(lines) + "\n", encoding="utf-8")
-    return len(lines) - 1
-
-
 def main():
     p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
     p.add_argument("--source", default=SOURCE)
     p.add_argument("--symbols", nargs="+", default=SYMBOLS)
     p.add_argument("--out", default=str(OUT))
     a = p.parse_args()
-    n = write(a.out, rows(read(a.source), symbols(read(s) for s in a.symbols)))
-    print(f"{a.out}: {n} rows", file=sys.stderr)
+    symbol = symbols(source.fetch(s, a.cache, Path(s).name).decode("utf-8") for s in a.symbols)
+    table = rows(source.fetch(a.source, a.cache, "codes-all.csv").decode("utf-8"), symbol)
+    tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["code"]))
 
 
 if __name__ == "__main__":

data-import/geo-se.py

@@ -1,28 +1,24 @@
 #!/usr/bin/env python3
 """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
 
-    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
-
-A locality is a GeoNames postort, placed in the municipality it names, else of its
-tätort, else of most of its codes, and weighted by its tätort's population, else its
-municipality's, else 200. Box codes are dropped by the digit after the postort's own
-prefix. Each NVDB street segment goes to the nearest postal code centroid; a locality
-keeps the N names with most segments.
+    TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import csv
 import io
 import json
-import math
 import os
 import re
 import sys
 import urllib.request
-import xml.etree.ElementTree as ET
 import zipfile
 from pathlib import Path
 
+import geo
+import source
+import tsv
+import xlsx
+
 CODES = "https://www.scb.se/contentassets/7a89e48960f741e08918e489ea36354a/kommunlankod-2026.xlsx"
 POPULATION = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0101/BE0101A/BefolkningNy"
 POPULATION_QUERY = {
@@ -37,38 +33,20 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
 POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
 NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
 NVDB_PAGE = 50000
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
-CACHE = Path(__file__).resolve().parent / "cache"
 TIMEZONE = "Europe/Stockholm"
+# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
 ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
+# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
 UNMATCHED_POPULATION = 200
-XLSX_NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"}
-
-
-def fetch(url, cache, name, data=None, headers=None):
-    path = cache / name
-    if not path.exists():
-        req = urllib.request.Request(url, data=data, headers=headers or {})
-        with urllib.request.urlopen(req, timeout=300) as r:
-            path.write_bytes(r.read())
-    return path.read_bytes()
-
-
-def xlsx_rows(data):
-    z = zipfile.ZipFile(io.BytesIO(data))
-    strings = ["".join(t.text or "" for t in si.iter("{%s}t" % XLSX_NS["m"])) for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", XLSX_NS)]
-    sheet = ET.fromstring(z.read("xl/worksheets/sheet1.xml"))
-    for row in sheet.findall(".//m:row", XLSX_NS):
-        cells = []
-        for c in row.findall("m:c", XLSX_NS):
-            v = c.find("m:v", XLSX_NS)
-            cells.append("" if v is None else strings[int(v.text)] if c.get("t") == "s" else v.text)
-        yield cells
+# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
+# or a description in one.
+NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
+PARTICLES = {"af", "de", "von"}
 
 
 def scb_codes(cache):
     regions, municipalities = {}, {}
-    for cells in xlsx_rows(fetch(CODES, cache, "kommunlankod.xlsx")):
+    for cells in xlsx.rows(source.fetch(CODES, cache, "kommunlankod.xlsx", magic=b"PK")):
         if len(cells) < 2 or not re.fullmatch(r"\d{2}|\d{4}", cells[0]):
             continue
         (regions if len(cells[0]) == 2 else municipalities)[cells[0]] = cells[1].strip()
@@ -77,12 +55,12 @@ def scb_codes(cache):
 
 def scb_population(cache):
     body = json.dumps(POPULATION_QUERY).encode()
-    data = fetch(POPULATION, cache, "befolkning.json", data=body, headers={"Content-Type": "application/json"})
+    data = source.fetch(POPULATION, cache, "befolkning.json", data=body, headers={"Content-Type": "application/json"})
     return {row["key"][0]: row["values"][0] for row in json.loads(data.decode("utf-8-sig"))["data"]}
 
 
 def scb_tatorter(cache):
-    text = fetch(TATORTER, cache, "tatorter.csv").decode("utf-8")
+    text = source.fetch(TATORTER, cache, "tatorter.csv").decode("utf-8")
     by_name = collections.defaultdict(list)
     for r in csv.DictReader(io.StringIO(text)):
         by_name[r["tatort"]].append((r["kommun"], int(r["bef"])))
@@ -90,7 +68,7 @@ def scb_tatorter(cache):
 
 
 def geonames(cache):
-    z = zipfile.ZipFile(io.BytesIO(fetch(POSTAL_CODES, cache, "SE.zip")))
+    z = zipfile.ZipFile(io.BytesIO(source.fetch(POSTAL_CODES, cache, "SE.zip", magic=b"PK")))
     rows = []
     for line in z.read("SE.txt").decode("utf-8").splitlines():
         f = line.split("\t")
@@ -102,7 +80,7 @@ def geonames(cache):
 def nvdb_segments(cache, key):
     path = cache / "nvdb-gatunamn.tsv"
     if not path.exists():
-        with open(path, "w", encoding="utf-8") as out:
+        with open(path.with_suffix(".part"), "w", encoding="utf-8") as out:
             change = "0"
             while True:
                 query = (
@@ -123,41 +101,17 @@ def nvdb_segments(cache, key):
                 change = result["INFO"]["LASTCHANGEID"]
                 if len(rows) < NVDB_PAGE:
                     break
+        path.with_suffix(".part").rename(path)
     for line in path.read_text(encoding="utf-8").splitlines():
         name, lon, lat = line.split("\t")
         yield name, float(lat), float(lon)
 
 
-class Nearest:
-    """Nearest point by an equirectangular distance, over a degree grid."""
-
-    def __init__(self, points, cell=0.05):
-        self.cell = cell
-        self.grid = collections.defaultdict(list)
-        for lat, lon, value in points:
-            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
-
-    def find(self, lat, lon):
-        ci, cj = int(lat // self.cell), int(lon // self.cell)
-        best, best_d = None, math.inf
-        ring = 0
-        while ring < 400:
-            for i in range(ci - ring, ci + ring + 1):
-                for j in range(cj - ring, cj + ring + 1):
-                    if max(abs(i - ci), abs(j - cj)) != ring:
-                        continue
-                    for plat, plon, value in self.grid.get((i, j), ()):
-                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
-                        if d < best_d:
-                            best, best_d = value, d
-            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
-                return best
-            ring += 1
-        return best
-
-
 def street_delivery(name, codes):
-    """The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
+    """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
+    least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
+    for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
+    docs/research/research-geo-se.md, Postal codes → Structure."""
     if name in ONE_POSITION:
         return [c for c in codes if c[1] != "0"]
     largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -187,77 +141,71 @@ def population_of(name, municipality, tatorter, municipalities, population):
 
 
 def well_cased(name):
+    """Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
+    capitals or lower case."""
     return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
 
 
-def write(path, columns, rows):
-    lines = ["\t".join(columns)]
-    for row in rows:
-        cells = [str(row[c]) for c in columns]
-        assert not any(re.search(r"[\t\n{}]", c) for c in cells), row
-        lines.append("\t".join(cells))
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    print(f"{path}: {len(rows)} rows", file=sys.stderr)
-
-
-def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
-    if not key:
-        sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
-
-    regions, municipalities = scb_codes(cache)
-    population = scb_population(cache)
-    tatorter = scb_tatorter(cache)
-    codes = geonames(cache)
-
+def localities(codes, tatorter, municipalities, population):
+    """Each postort with its municipality, weight, centroid and street-delivery codes."""
     by_locality = collections.defaultdict(list)
     for r in codes:
         by_locality[r["locality"]].append(r)
-    localities, how = {}, collections.Counter()
+    out, how = {}, collections.Counter()
     for name, rows in by_locality.items():
         municipality, method = municipality_of(name, rows, tatorter, municipalities)
         how[method] += 1
         kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
-        with_point = [r for r in rows if r["lat"] is not None]
-        if municipality is None or not kept or not with_point or not well_cased(name):
+        point = geo.centroid(rows)
+        if municipality is None or not kept or point is None or not well_cased(name):
             continue
-        lat = sum(r["lat"] for r in with_point) / len(with_point)
-        lon = sum(r["lon"] for r in with_point) / len(with_point)
-        localities[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
-    print(f"municipality by {dict(how)}; {len(by_locality) - len(localities)} postorter dropped", file=sys.stderr)
+        lat, lon = point
+        out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
+    print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
+    return out
 
-    centroids = [(r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities]
-    nearest_locality = Nearest(centroids)
-    segments = collections.Counter()
-    for name, lat, lon in nvdb_segments(cache, key):
-        segments[(nearest_locality.find(lat, lon), name)] += 1
-    streets_of = collections.defaultdict(list)
-    for (locality, name), n in segments.items():
-        streets_of[locality].append((n, name))
-    streets = []
-    for locality in sorted(streets_of):
-        for n, name in sorted(streets_of[locality], key=lambda s: (-s[0], s[1]))[: a.streets_per_locality]:
-            streets.append({"name": name, "locality": locality, "segments": n})
-    for name in [l for l in localities if l not in streets_of]:
-        del localities[name]
 
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in localities.values()))
-    if empty:
-        sys.exit(f"municipalities without a locality: {empty}")
-    write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
-    write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
-    write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(localities.items())])
-    write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in localities.values() for c in l["codes"]), key=lambda r: r["code"]))
-    write(out / "street.tsv", ["name", "locality", "segments"], streets)
+def street_name(name):
+    # A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
+    opening = name[0].isupper() or name.split()[0] in PARTICLES
+    return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
+
+
+def streets(segments, codes, localities, share, minimum):
+    """The names with most segments per locality, each segment at its nearest code centroid."""
+    nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
+    count = collections.Counter()
+    for name, lat, lon in segments:
+        if street_name(name):
+            count[(nearest.find(lat, lon), name)] += 1
+    return geo.top_streets(count, share, minimum, "segments")
+
+
+def main():
+    a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
+    key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
+    if not key:
+        sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
+    cache, out = geo.directories(a)
+
+    region_names, municipality_names = scb_codes(cache)
+    population = scb_population(cache)
+    codes = geonames(cache)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
+    named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
+    geo.log_dropped_localities(dropped["locality"])
+    if dropped["municipality"] or dropped["region"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
+    places = kept["locality"]
+
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
+    tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
+    tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo-us.py

@@ -1,25 +1,23 @@
 #!/usr/bin/env python3
-"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
+"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
 
-    data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
-
-A locality is an incorporated place, or a consolidated city's balance, of at least N
-people, in the county holding most of it. Its postal codes are the ZCTAs mostly inside
-it, weighted by their TIGER address ranges, and its streets the N names with most
-address ranges in those codes.
+    data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
 """
-import argparse
 import collections
 import concurrent.futures
 import csv
 import io
+import math
+import random
 import re
 import struct
 import sys
-import time
-import urllib.request
+import urllib.parse
 import zipfile
-from pathlib import Path
+
+import geo
+import source
+import tsv
 
 GAZETTEER = "https://www2.census.gov/geo/docs/maps-data/data/gazetteer/2026_Gazetteer/2026_Gaz_{}_national.zip"
 POPULATION = "https://www2.census.gov/programs-surveys/popest/datasets/2020-2025/{}"
@@ -28,13 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
 PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
 ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
 TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
-OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
-CACHE = Path(__file__).resolve().parent / "cache"
 ESTIMATE = "POPESTIMATE2025"
 CDP = "57"
+# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
+CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
+SELECT ?iso ?capitalLabel ?gnis WHERE {
+  ?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
+  OPTIONAL { ?capital wdt:P590 ?gnis. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}"""})
+SHIPPED_POPULATION = 100000
+HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
 SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
-# Places whose Census name is a merged government's; the postal city is what an address carries.
-NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
+# Places whose Census name is not the postal city an address carries.
+NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
 TIMEZONES = {
     "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
     "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -52,24 +57,6 @@ TIMEZONES = {
 }
 
 
-def fetch(url, cache, name, magic=b""):
-    path = cache / name
-    for attempt in range(1, 6):
-        if path.exists():
-            return path.read_bytes()
-        req = urllib.request.Request(url, headers={"User-Agent": "fejkdata data-import"})
-        try:
-            with urllib.request.urlopen(req, timeout=600) as r:
-                data = r.read()
-        except OSError:
-            data = b""
-        if data.startswith(magic) and b"Request Rejected" not in data[:512]:
-            path.write_bytes(data)
-        else:
-            time.sleep(10 * attempt)
-    sys.exit(f"{url}: no valid download in 5 attempts")
-
-
 def text(data):
     try:
         return data.decode("utf-8-sig")
@@ -78,14 +65,14 @@ def text(data):
 
 
 def gazetteer(cache, kind):
-    z = zipfile.ZipFile(io.BytesIO(fetch(GAZETTEER.format(kind), cache, f"gaz_{kind}.zip", magic=b"PK")))
+    z = zipfile.ZipFile(io.BytesIO(source.fetch(GAZETTEER.format(kind), cache, f"gaz_{kind}.zip", magic=b"PK")))
     rows = text(z.read(z.namelist()[0])).splitlines()
     header = [h.strip() for h in rows[0].split("|")]
     return [dict(zip(header, (c.strip() for c in row.split("|")))) for row in rows[1:]]
 
 
 def csv_rows(cache, url, name):
-    return list(csv.DictReader(io.StringIO(text(fetch(url, cache, name)))))
+    return list(csv.DictReader(io.StringIO(text(source.fetch(url, cache, name)))))
 
 
 def dbf_rows(data, wanted):
@@ -111,7 +98,7 @@ def dbf_rows(data, wanted):
 
 
 def tiger_zip(cache, kind, county):
-    return fetch(TIGER.format(kind.upper(), county, kind), cache, f"tl_{county}_{kind}.zip", magic=b"PK")
+    return source.fetch(TIGER.format(kind.upper(), county, kind), cache, f"tl_{county}_{kind}.zip", magic=b"PK")
 
 
 def tiger(cache, kind, county, wanted):
@@ -128,39 +115,91 @@ def place_name(geoid, name):
     return stripped
 
 
-def write(path, columns, rows):
-    lines = ["\t".join(columns)]
-    for row in rows:
-        cells = [str(row[c]) for c in columns]
-        assert not any(re.search(r"[\t\n{}]", c) for c in cells), row
-        lines.append("\t".join(cells))
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    print(f"{path}: {len(rows)} rows", file=sys.stderr)
-
-
-def main():
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    p.add_argument("--cache", default=str(CACHE))
-    p.add_argument("--min-population", type=int, default=25000)
-    p.add_argument("--out", default=str(OUT))
-    p.add_argument("--streets-per-locality", type=int, default=10)
-    a = p.parse_args()
-    cache, out = Path(a.cache), Path(a.out)
-    cache.mkdir(parents=True, exist_ok=True)
-    out.mkdir(parents=True, exist_ok=True)
-
-    state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
-    county_population = {r["STATE"] + r["COUNTY"]: r[ESTIMATE] for r in csv_rows(cache, COUNTIES, "co-est2025.csv") if r["SUMLEV"] == "050"}
+def localities(cache, counties):
+    """Each incorporated place with its county, population and centroid."""
     place_population, county_part = {}, collections.defaultdict(list)
     for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
         if r["SUMLEV"] == "162":
             place_population[r["STATE"] + r["PLACE"]] = int(r[ESTIMATE])
         elif r["SUMLEV"] == "157":
             county_part[r["STATE"] + r["PLACE"]].append((int(r[ESTIMATE]), r["STATE"] + r["COUNTY"]))
+    out = {}
+    for r in gazetteer(cache, "place"):
+        geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
+        if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
+            continue
+        county = max(county_part[geoid])[1]
+        if county not in counties:
+            print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
+            continue
+        out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
+    return out
 
-    regions = {}
-    for r in gazetteer(cache, "state"):
-        regions[r["USPS"]] = {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]}
+
+def capitals(cache, places, counties):
+    """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
+    by_gnis = {p["gnis"]: code for code, p in places.items()}
+    named = collections.defaultdict(list)
+    for code, p in places.items():
+        named[(counties[p["municipality"]]["region"], p["name"])].append(code)
+    by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
+    gnis_of = collections.defaultdict(set)
+    for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
+        gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
+    out = set()
+    for (state, name), ids in sorted(gnis_of.items()):
+        code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
+        if code:
+            out.add(code)
+        else:
+            print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
+    return out
+
+
+def drawn(places, share):
+    """The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
+    sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
+    the sources moves only the places near the cut."""
+    peopled = [code for code, p in places.items() if p["population"] > 0]
+    key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
+    return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
+
+
+def zcta_localities(cache, localities):
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
+    parts = {}
+    for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
+        if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
+            parts.setdefault(r["GEOID_ZCTA5_20"], []).append((int(r["AREALAND_PART"]), r["GEOID_PLACE_20"]))
+    largest = {zcta: max(p)[1] for zcta, p in parts.items()}
+    return {zcta: place for zcta, place in largest.items() if place in localities}
+
+
+def streets(cache, counties, locality_of_zcta, share, minimum):
+    """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
+    with concurrent.futures.ThreadPoolExecutor(3) as pool:
+        list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
+    addresses, count = collections.Counter(), collections.Counter()
+    for county in counties:
+        zips = collections.defaultdict(set)
+        for r in tiger(cache, "addr", county, {"TLID", "ZIP"}):
+            if r["ZIP"] in locality_of_zcta:
+                zips[r["TLID"]].add(r["ZIP"])
+                addresses[r["ZIP"]] += 1
+        for r in tiger(cache, "featnames", county, {"TLID", "FULLNAME", "PAFLAG"}):
+            if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
+                for z in zips.get(r["TLID"], ()):
+                    count[(locality_of_zcta[z], r["FULLNAME"])] += 1
+    return geo.top_streets(count, share, minimum, "addresses"), addresses
+
+
+def main():
+    a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
+    cache, out = geo.directories(a)
+
+    state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
+    regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
+    county_population = {r["STATE"] + r["COUNTY"]: r[ESTIMATE] for r in csv_rows(cache, COUNTIES, "co-est2025.csv") if r["SUMLEV"] == "050"}
     counties, unestimated = {}, collections.Counter()
     for r in gazetteer(cache, "counties"):
         if r["GEOID"] not in county_population:
@@ -168,56 +207,30 @@ def main():
             continue
         counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
-    localities = {}
-    for r in gazetteer(cache, "place"):
-        geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
-        if r["FUNCSTAT"] not in "AFN" or r["LSAD"] == CDP or population < a.min_population or not county_part.get(geoid):
-            continue
-        county = max(county_part[geoid])[1]
-        if county not in counties:
-            print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
-            continue
-        localities[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
 
-    zcta_of = {}
-    for r in csv.DictReader(io.StringIO(text(fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
-        if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] in localities:
-            zcta_of.setdefault(r["GEOID_ZCTA5_20"], []).append((int(r["AREALAND_PART"]), r["GEOID_PLACE_20"]))
-    locality_of_zcta = {zcta: max(parts)[1] for zcta, parts in zcta_of.items()}
+    places = localities(cache, counties)
+    owner_of_zcta = zcta_localities(cache, places)
+    capital_codes = capitals(cache, places, counties)
+    owners = {p: places[p] for p in set(owner_of_zcta.values())}
+    for code in sorted(capital_codes - set(owners)):
+        print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
+    places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
+    locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
+    named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped_localities(dropped["locality"])
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    needed = sorted({l["municipality"] for l in localities.values()})
-    with concurrent.futures.ThreadPoolExecutor(3) as pool:
-        list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), needed))
-    addresses, streets_of = collections.Counter(), collections.Counter()
-    for county in needed:
-        zips = collections.defaultdict(set)
-        for r in tiger(cache, "addr", county, {"TLID", "ZIP"}):
-            if r["ZIP"] in locality_of_zcta:
-                zips[r["TLID"]].add(r["ZIP"])
-                addresses[r["ZIP"]] += 1
-        for r in tiger(cache, "featnames", county, {"TLID", "FULLNAME", "PAFLAG"}):
-            if r["PAFLAG"] == "P" and r["FULLNAME"]:
-                for z in zips.get(r["TLID"], ()):
-                    streets_of[(locality_of_zcta[z], r["FULLNAME"])] += 1
-    by_locality = collections.defaultdict(list)
-    for (locality, name), n in streets_of.items():
-        by_locality[locality].append((n, name))
-    streets = []
-    for locality in sorted(by_locality):
-        for n, name in sorted(by_locality[locality], key=lambda s: (-s[0], s[1]))[: a.streets_per_locality]:
-            streets.append({"name": name, "locality": locality, "addresses": n})
-    for geoid in [l for l in localities if l not in by_locality]:
-        print(f"{geoid} {localities[geoid]['name']}: no streets, dropped", file=sys.stderr)
-        del localities[geoid]
-    postal_codes = [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in localities]
-
-    kept_counties = {l["municipality"] for l in localities.values()}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
-    write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
-    write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
-    write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(localities.items())])
-    write(out / "postal-code.tsv", ["code", "locality", "addresses"], postal_codes)
-    write(out / "street.tsv", ["name", "locality", "addresses"], streets)
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
+    tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
 
 
 if __name__ == "__main__":

data-import/geo.py

@@ -0,0 +1,98 @@
+"""The steps a country's geo import may share with another's."""
+import argparse
+import collections
+import math
+import sys
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
+CACHE = Path(__file__).resolve().parent / "cache"
+
+
+def parser(doc, country, street_share, **own):
+    """A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
+    "_" standing for "-"."""
+    options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
+    p = argparse.ArgumentParser(description=doc.splitlines()[0])
+    for name in sorted(options):
+        p.add_argument("--" + name.replace("_", "-"), **options[name])
+    return p
+
+
+def directories(args):
+    """The cache and out directories of the parsed options, both created."""
+    cache, out = Path(args.cache), Path(args.out)
+    cache.mkdir(parents=True, exist_ok=True)
+    out.mkdir(parents=True, exist_ok=True)
+    return cache, out
+
+
+def centroid(rows):
+    """The mean lat and lon of the rows with a point, or None where no row has one."""
+    points = [r for r in rows if r["lat"] is not None]
+    if not points:
+        return None
+    return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
+
+
+class Nearest:
+    """Nearest point by an equirectangular distance, over a degree grid."""
+
+    def __init__(self, points, cell=0.05):
+        self.cell = cell
+        self.grid = collections.defaultdict(list)
+        for lat, lon, value in points:
+            self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
+
+    def find(self, lat, lon):
+        ci, cj = int(lat // self.cell), int(lon // self.cell)
+        best, best_d = None, math.inf
+        ring = 0
+        while ring < 400:
+            for i in range(ci - ring, ci + ring + 1):
+                for j in range(cj - ring, cj + ring + 1):
+                    if max(abs(i - ci), abs(j - cj)) != ring:
+                        continue
+                    for plat, plon, value in self.grid.get((i, j), ()):
+                        d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
+                        if d < best_d:
+                            best, best_d = value, d
+            if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
+                return best
+            ring += 1
+        return best
+
+
+def top_streets(count, share, minimum, column):
+    """count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
+    them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
+    of = collections.defaultdict(list)
+    for (locality, name), n in count.items():
+        of[locality].append((n, name))
+    return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
+
+
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
+
+
+def log_dropped_localities(rows):
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)

data-import/httpmethod.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/httpmethod.tsv from the IANA HTTP Method Registry.
+
+    data-import/httpmethod.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A method ships when the register cites RFC 9110 section 9.3 or RFC 5789 for it, which
+is the eight HTTP defines and PATCH; the register's other entries do not. The
+register's `yes` and `no` ship as `true` and `false`, which a consumer's boolean reads.
+"""
+import argparse
+import csv
+import io
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/http-methods/methods.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpmethod.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["idempotent", "method", "safe"]
+CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
+BOOLEAN = {"yes": "true", "no": "false"}
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+EXPECTED = 9
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        method = (r["Method Name"] or "").strip()
+        if not CORE.search(r["Reference"] or ""):
+            continue
+        if set(method) & set(SELECTOR):
+            sys.exit(f"{method}: a method holding one of {SELECTOR} is one a selector cannot spell")
+        answers = {}
+        for column in ("Safe", "Idempotent"):
+            answer = (r[column] or "").strip()
+            if answer not in BOOLEAN:
+                sys.exit(f"{method}: {column} {answer!r} is neither yes nor no")
+            answers[column] = BOOLEAN[answer]
+        yield {"idempotent": answers["Idempotent"], "method": method, "safe": answers["Safe"]}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "http-methods.csv").decode("utf-8")), key=lambda r: r["method"])
+    if len(table) != EXPECTED:
+        sys.exit(f"{len(table)} methods cite RFC 9110 section 9.3 or RFC 5789, not {EXPECTED}; the register's Reference column has moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/httpstatus.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/httpstatus.tsv from the IANA HTTP Status Code Registry.
+
+    data-import/httpstatus.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+What ships is the codes in use: a range, an unassigned code and a qualified reason are all skipped.
+"""
+import argparse
+import csv
+import io
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpstatus.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["code", "reason"]
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        code, reason = r["Value"].strip(), r["Description"].strip()
+        if code.isdigit() and reason and "(" not in reason and reason != "Unassigned":
+            yield {"code": code, "reason": reason}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(source.fetch(a.source, a.cache, "http-status-codes-1.csv").decode("utf-8"))
+    tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: int(r["code"])))
+
+
+if __name__ == "__main__":
+    main()

data-import/language.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/language.tsv from datasets/language-codes (PDDL), the LoC ISO 639-2 register.
+
+    data-import/language.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+Only the 639-2 entries carrying a 639-1 code ship, since the alpha-2 code is the key.
+"""
+import argparse
+import csv
+import io
+import re
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://raw.githubusercontent.com/datasets/language-codes/main/data/language-codes-full.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "language.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["code", "code3", "name"]
+
+
+def name(english):
+    """The register lists every synonym; the first, without its qualifier, is the one a selector spells."""
+    return re.sub(r"\s*\([^)]*\)", "", english.split(";")[0]).strip()
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        if r["alpha2"]:
+            yield {"code": r["alpha2"], "code3": r["alpha3-t"] or r["alpha3-b"], "name": name(r["English"])}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(source.fetch(a.source, a.cache, "language-codes-full.csv").decode("utf-8"))
+    tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["code"]))
+
+
+if __name__ == "__main__":
+    main()

data-import/loglevel.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/loglevel.tsv from RFC 5424 and POSIX <syslog.h>.
+
+    data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE]
+
+Every severity in Table 2 ships, paired to POSIX's keywords by the order each source
+lists them, which holds only because a keyword prefixes the severity RFC 5424 numbers
+alike.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt"
+POSIX = "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["code", "keyword", "severity"]
+TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S)
+ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M)
+SECTION = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S)
+MACRO = re.compile(r"\bLOG_([A-Z]+)\b")
+EXPECTED = 8
+
+
+def severities(text):
+    table = TABLE.search(text)
+    if not table:
+        sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved")
+    return ROW.findall(table.group(1))
+
+
+def keywords(page):
+    section = SECTION.search(html.unescape(re.sub(r"<[^>]+>", "", page)))
+    if not section:
+        sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved")
+    return [m.lower() for m in MACRO.findall(section.group(1))]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--posix", default=POSIX)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(severities(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")))
+    codes = [code for code, _ in table]
+    if codes != [str(i) for i in range(EXPECTED)]:
+        sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}")
+    words = keywords(source.fetch(a.posix, a.cache, "posix-syslog.h.html").decode("utf-8"))
+    if len(words) != EXPECTED:
+        sys.exit(f"POSIX lists {len(words)} severity macros, not {EXPECTED}: {words}")
+    rows = []
+    for (code, severity), keyword in zip(table, words):
+        if not severity.lower().startswith(keyword):
+            sys.exit(f"severity {code} is {severity} in the RFC and {keyword} in POSIX, which is no prefix of it")
+        rows.append({"code": code, "keyword": keyword, "severity": severity})
+    tsv.write(a.out, COLUMNS, rows)
+
+
+if __name__ == "__main__":
+    main()

data-import/mimetype.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/mimetype.tsv from mime-db (MIT), the IANA media type registry with extensions.
+
+    data-import/mimetype.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+Only types mime-db records as IANA-registered, with a filename extension, ship.
+"""
+import argparse
+import json
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "mimetype.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["ext", "type"]
+FIXUPS = {"application/mp4": "mp4s", "audio/mpeg": "mp3", "audio/ogg": "ogg", "video/quicktime": "mov"}
+
+
+def rows(db):
+    for mimetype, entry in db.items():
+        if entry.get("source") == "iana" and entry.get("extensions"):
+            yield {"ext": "." + FIXUPS.get(mimetype, entry["extensions"][0]), "type": mimetype}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(json.loads(source.fetch(a.source, a.cache, "mime-db.json").decode("utf-8")))
+    tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["type"]))
+
+
+if __name__ == "__main__":
+    main()

data-import/names-se.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Rebuild data/sv_SE/first-name.tsv and last-name.tsv from SCB's whole-population name counts (CC0, "Källa: SCB").
+
+    data-import/names-se.py [--source URL_OR_FILE] [--first N] [--last N] [--cache DIR] [--out DIR]
+
+A tilltalsnamn goes under each sex that carries it, so a name both sexes carry has two rows.
+"""
+import argparse
+import re
+from pathlib import Path
+
+import source
+import tsv
+import xlsx
+
+SOURCE = "https://www.scb.se/contentassets/9fe7dbb460994c72b835163dbc491ef9/namn-med-minst-tva-barare-31-december-2022.xlsx"
+OUT = Path(__file__).resolve().parent.parent / "data" / "sv_SE"
+CACHE = Path(__file__).resolve().parent / "cache"
+SHEETS = {"f": "Tilltalsnamn kvinnor", "m": "Tilltalsnamn män"}
+SURNAMES = "Efternamn"
+NAME = re.compile(r"^[^\W\d_]{2,}([ '-][^\W\d_]{2,})*$")
+PARTICLES = {"af", "av", "da", "de", "del", "den", "der", "di", "dos", "du", "la", "le", "van", "von"}
+
+
+def cased(name):
+    """SCB's uppercase name as it is written: each part capitalised, a particle before another part lowercased."""
+    parts = name.lower().split(" ")
+    return " ".join(w if w in PARTICLES and len(parts) > 1 else w.title() for w in parts)
+
+
+def counted(rows, top):
+    """The top names of a sheet by bearers, initials and single letters dropped."""
+    names = [(cells[0], int(cells[1])) for cells in rows if len(cells) >= 2 and cells[1].isdigit() and NAME.match(cells[0])]
+    return sorted(names, key=lambda n: (-n[1], n[0]))[:top]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--first", type=int, default=2000, help="names per sex")
+    p.add_argument("--last", type=int, default=5000)
+    p.add_argument("--out", default=str(OUT))
+    p.add_argument("--source", default=SOURCE)
+    a = p.parse_args()
+    data = source.fetch(a.source, a.cache, "scb-namn-2022.xlsx", magic=b"PK")
+    first = [{"name": cased(n), "sex": sex, "count": c} for sex, sheet in SHEETS.items() for n, c in counted(xlsx.rows(data, sheet), a.first)]
+    last = [{"name": cased(n), "count": c} for n, c in counted(xlsx.rows(data, SURNAMES), a.last)]
+    out = Path(a.out)
+    tsv.write(out / "first-name.tsv", ["name", "sex", "count"], sorted(first, key=lambda r: (r["name"], r["sex"])))
+    tsv.write(out / "last-name.tsv", ["name", "count"], sorted(last, key=lambda r: r["name"]))
+
+
+if __name__ == "__main__":
+    main()

data-import/names-us.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Rebuild data/en_US/first-name.tsv and last-name.tsv from SSA baby names and the Census 2010 surnames (public domain).
+
+    data-import/names-us.py [--names URL_OR_FILE] [--surnames URL_OR_FILE] [--from-year YEAR] [--first N] [--last N] [--cache DIR] [--out DIR]
+
+A given name is counted over the births of --from-year and later, under each sex it was given to, so a
+name both sexes carry has two rows. --names takes SSA's names.zip or a merged name,sex,count,year file;
+--surnames takes the Census names.zip or its Names_2010Census.csv.
+"""
+import argparse
+import collections
+import csv
+import io
+import re
+import zipfile
+from pathlib import Path
+
+import source
+import tsv
+
+NAMES = "https://raw.githubusercontent.com/hackerb9/ssa-baby-names/master/alldata.txt"
+SURNAMES = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip"
+OUT = Path(__file__).resolve().parent.parent / "data" / "en_US"
+CACHE = Path(__file__).resolve().parent / "cache"
+MC = re.compile(r"^Mc([a-z])")
+PLACEHOLDERS = {"Baby", "Female", "Infant", "Male", "Notnamed", "Unknown", "Unnamed"}
+
+
+def csv_or_zip_rows(data, member):
+    """The rows of a CSV, or of every member of a zip named like member, name,sex,count[,year]."""
+    if data.startswith(b"PK"):
+        z = zipfile.ZipFile(io.BytesIO(data))
+        for n in sorted(z.namelist()):
+            if re.fullmatch(member, n):
+                year = re.sub(r"\D", "", n)
+                for r in csv.reader(io.StringIO(z.read(n).decode("utf-8-sig"))):
+                    yield r + [year] if year else r
+        return
+    yield from csv.reader(io.StringIO(data.decode("utf-8-sig")))
+
+
+def given(data, from_year):
+    counts = collections.Counter()
+    for r in csv_or_zip_rows(data, r"yob\d{4}\.txt"):
+        if len(r) >= 4 and r[3].isdigit() and int(r[3]) >= from_year and r[0] not in PLACEHOLDERS:
+            counts[(r[0], r[1].lower())] += int(r[2])
+    return counts
+
+
+def surname(name):
+    """A Census uppercase surname as it is written: capitalised, and Mc before a capital."""
+    return MC.sub(lambda m: "Mc" + m.group(1).upper(), name.title())
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--first", type=int, default=2000, help="names per sex")
+    p.add_argument("--from-year", type=int, default=1930)
+    p.add_argument("--last", type=int, default=5000)
+    p.add_argument("--names", default=NAMES)
+    p.add_argument("--out", default=str(OUT))
+    p.add_argument("--surnames", default=SURNAMES)
+    a = p.parse_args()
+    counts = given(source.fetch(a.names, a.cache, "ssa-names.txt"), a.from_year)
+    first = []
+    for sex in ("f", "m"):
+        top = sorted(((n, c) for (n, s), c in counts.items() if s == sex), key=lambda n: (-n[1], n[0]))[:a.first]
+        first += [{"name": n, "sex": sex, "count": c} for n, c in top]
+    rows = csv_or_zip_rows(source.fetch(a.surnames, a.cache, "census-surnames-2010.zip"), r"Names_2010Census\.csv")
+    last = [{"name": surname(r[0]), "count": int(r[2])} for r in rows if len(r) >= 3 and r[2].isdigit() and r[0].isalpha()]
+    last = sorted(last, key=lambda r: (-r["count"], r["name"]))[:a.last]
+    out = Path(a.out)
+    tsv.write(out / "first-name.tsv", ["name", "sex", "count"], sorted(first, key=lambda r: (r["name"], r["sex"])))
+    tsv.write(out / "last-name.tsv", ["name", "count"], sorted(last, key=lambda r: r["name"]))
+
+
+if __name__ == "__main__":
+    main()

data-import/port.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/port.tsv from the IANA Service Name and Transport Protocol Port Number Registry.
+
+    data-import/port.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a service name and a numeric TCP port. A port the registry lists more than
+once keeps the first service it describes, or the first of them where it describes none,
+so a number selects one row.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "port.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["number", "service"]
+FLOOR = 4000
+
+
+def rows(text):
+    best, described = {}, set()
+    for r in csv.DictReader(io.StringIO(text)):
+        service, number = (r["Service Name"] or "").strip(), (r["Port Number"] or "").strip()
+        transport, description = (r["Transport Protocol"] or "").strip(), (r["Description"] or "").strip()
+        if transport != "tcp" or not service or not number.isdigit():
+            continue
+        if not 1 <= int(number) <= 65535:
+            sys.exit(f"{service}: port {number} is outside 1-65535")
+        if number not in best or (description and number not in described):
+            best[number] = {"number": number, "service": service}
+        if description:
+            described.add(number)
+    return sorted(best.values(), key=lambda r: int(r["number"]))
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = rows(source.fetch(a.source, a.cache, "service-names-port-numbers.csv").decode("utf-8"))
+    if len(table) < FLOOR:
+        sys.exit(f"only {len(table)} ports named a TCP service; the registry's columns have moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/protocol.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/protocol.tsv from the IANA Protocol Numbers registry.
+
+    data-import/protocol.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A row needs a number and a keyword, which drops the seven numbers assigned to a class of
+protocols rather than to one; a keyword the registry reserves or marks deprecated goes
+too. Where the registry names no protocol beside the keyword, the keyword is the name.
+"""
+import argparse
+import csv
+import io
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["keyword", "name", "number"]
+SKIP = ("Reserved", "deprecated")
+SELECTOR = '[]{}"|'  # mirrors inSelector in internal/grammar/path.go
+
+
+def rows(text):
+    for r in csv.DictReader(io.StringIO(text)):
+        number, keyword = (r["Decimal"] or "").strip(), (r["Keyword"] or "").strip()
+        if not number.isdigit() or not keyword or any(s in keyword for s in SKIP):
+            continue
+        if not 0 <= int(number) <= 255:
+            sys.exit(f"{keyword}: protocol number {number} is outside 0-255")
+        yield {"keyword": keyword, "name": " ".join((r["Protocol"] or "").split()) or keyword, "number": number}
+
+
+def refuse_an_unselectable_row(table):
+    """What the loader refuses at New, and a name that names two rows rather than one."""
+    keywords, names = {}, {}
+    for r in table:
+        for column in ("keyword", "name"):
+            if set(r[column]) & set(SELECTOR):
+                sys.exit(f"{r['keyword']}: {column} {r[column]!r} holds one of {SELECTOR}, which a selector cannot spell")
+        if r["keyword"] in keywords:
+            sys.exit(f"{r['keyword']}: repeats the keyword of protocol {keywords[r['keyword']]}")
+        keywords[r["keyword"]] = r["number"]
+    for r in table:
+        if r["name"] != r["keyword"] and r["name"] in keywords:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} is another protocol's keyword")
+        if r["name"] in names:
+            sys.exit(f"{r['keyword']}: name {r['name']!r} repeats {names[r['name']]}'s, so it selects neither")
+        names[r["name"]] = r["keyword"]
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "protocol-numbers-1.csv").decode("utf-8")), key=lambda r: int(r["number"]))
+    refuse_an_unselectable_row(table)
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/source.py

@@ -0,0 +1,29 @@
+"""A source fetched once into the cache."""
+import re
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+
+def fetch(source, cache, name, magic=b"", data=None, headers=None):
+    """The bytes of a URL, downloaded into cache/name once, or of a local file."""
+    if not re.match(r"^https?://", source):
+        return Path(source).read_bytes()
+    path = Path(cache) / name
+    if path.exists():
+        return path.read_bytes()
+    for attempt in range(1, 6):
+        req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
+        try:
+            with urllib.request.urlopen(req, timeout=600) as r:
+                body = r.read()
+        except OSError:
+            body = b""
+        if body and body.startswith(magic) and b"Request Rejected" not in body[:512]:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(body)
+            return body
+        if attempt < 5:
+            time.sleep(10 * attempt)
+    sys.exit(f"{source}: no valid download in 5 attempts")

data-import/territory.py

@@ -1,30 +1,28 @@
 #!/usr/bin/env python3
-"""Rebuild data/misc/country.tsv from datasets/country-codes (PDDL).
+"""Rebuild data/misc/territory.tsv from datasets/country-codes (PDDL).
 
-    data-import/country.py [--source URL_OR_FILE] [--out FILE]
+    data-import/territory.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
 """
 import argparse
 import csv
 import io
 import re
 import sys
-import urllib.request
 from pathlib import Path
 
+import source
+import tsv
+
 SOURCE = "https://raw.githubusercontent.com/datasets/country-codes/main/data/country-codes.csv"
-OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "country.tsv"
-COLUMNS = ["alpha2", "alpha3", "calling-code", "capital", "currency", "flag", "languages", "name", "numeric", "tld"]
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["alpha2", "alpha3", "calling-code", "capital", "country", "currency", "flag", "languages", "name", "numeric", "tld"]
+SOVEREIGN = re.compile(r"^(?:Part of|Territor(?:y|ies) of|Crown dependency of|Commonwealth of|Associated with) ([A-Z]{2})$")
+STANDALONE = {"In contention", "International"}
 # Gaps in the source, keyed by alpha2.
 FIXUPS = {"TR": {"currency": "TRY"}}
 
 
-def read(source):
-    if re.match(r"^https?://", source):
-        with urllib.request.urlopen(source, timeout=60) as r:
-            return r.read().decode("utf-8")
-    return Path(source).read_text(encoding="utf-8")
-
-
 def flag(alpha2):
     return "".join(chr(0x1F1E6 + ord(c) - ord("A")) for c in alpha2)
 
@@ -38,6 +36,16 @@ def languages(field):
     return ",".join(p for p in field.split(",") if p)
 
 
+def country(alpha2, independent):
+    """The sovereign state the register records; a territory it records none for stands alone."""
+    m = SOVEREIGN.match(independent)
+    if m:
+        return m.group(1)
+    if independent == "Yes" or independent in STANDALONE:
+        return alpha2
+    sys.exit(f"{alpha2}: is_independent {independent!r} names no sovereign this script can read")
+
+
 def rows(text):
     for r in csv.DictReader(io.StringIO(text)):
         alpha2 = r["ISO3166-1-Alpha-2"]
@@ -46,6 +54,7 @@ def rows(text):
             "alpha3": r["ISO3166-1-Alpha-3"],
             "calling-code": calling_code(r["Dial"]),
             "capital": r["Capital"],
+            "country": country(alpha2, r["is_independent"]),
             "currency": r["ISO4217-currency_alphabetic_code"].split(",")[0],
             "flag": flag(alpha2),
             "languages": languages(r["Languages"]),
@@ -58,23 +67,18 @@ def rows(text):
             yield row
 
 
-def write(out, table):
-    lines = ["\t".join(COLUMNS)]
-    for row in sorted(table, key=lambda r: r["alpha2"]):
-        cells = [row[c] for c in COLUMNS]
-        assert not any("\t" in c or "\n" in c for c in cells), row
-        lines.append("\t".join(cells))
-    Path(out).write_text("\n".join(lines) + "\n", encoding="utf-8")
-    return len(lines) - 1
-
-
 def main():
     p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
     p.add_argument("--source", default=SOURCE)
     p.add_argument("--out", default=str(OUT))
     a = p.parse_args()
-    n = write(a.out, rows(read(a.source)))
-    print(f"{a.out}: {n} rows", file=sys.stderr)
+    table = sorted(rows(source.fetch(a.source, a.cache, "country-codes.csv").decode("utf-8")), key=lambda r: r["alpha2"])
+    shipped = {r["alpha2"] for r in table}
+    for r in table:
+        if r["country"] not in shipped:
+            sys.exit(f"{r['alpha2']}: country {r['country']} is no shipped territory; a sovereign the set drops leaves its territories unreadable")
+    tsv.write(a.out, COLUMNS, table)
 
 
 if __name__ == "__main__":

data-import/timezone.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/timezone.tsv from the IANA tzdb tarball (public domain) and GeoNames (CC BY 4.0).
+
+    data-import/timezone.py [--source URL_OR_FILE] [--cities URL_OR_FILE] [--cache DIR] [--out FILE] [--territories FILE]
+
+The offset is the zone's standard offset, the first field of its Zone rule's last
+continuation line, with a Link resolved to its target. The weight is the population
+GeoNames records in the zone's cities of 15,000 or more, never below that threshold,
+so a drawn zone is one people live in.
+"""
+import argparse
+import csv
+import io
+import re
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
+CITIES = "https://download.geonames.org/export/dump/cities15000.zip"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "timezone.tsv"
+TERRITORIES = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["offset", "territory", "weight", "zone"]
+REGIONS = ["africa", "antarctica", "asia", "australasia", "backward", "etcetera", "europe", "northamerica", "southamerica"]
+# GeoNames' own cutoff, so a zone it records no city in weighs less than one would.
+FLOOR = 15000
+
+
+def member(tar, name):
+    try:
+        f = tar.extractfile(name)
+    except KeyError:
+        f = None
+    if f is None:
+        sys.exit(f"{name}: the tarball no longer holds it as a file; the tzdb layout has moved")
+    return f.read().decode("utf-8")
+
+
+def offsets(tar):
+    """Every zone's standard offset, and every link's target."""
+    std, links = {}, {}
+    for name in REGIONS:
+        zone = None
+        for raw in member(tar, name).splitlines():
+            line = raw.split("#")[0].rstrip()
+            if not line.strip():
+                continue
+            if line.startswith("Zone"):
+                f = line.split()
+                zone, std[f[1]] = f[1], f[2]
+            elif line.startswith("Link"):
+                f = line.split()
+                links[f[2]], zone = f[1], None
+            elif line[0] in " \t" and zone:
+                std[zone] = line.split()[0]
+            else:
+                zone = None
+    return std, links
+
+
+def resolve(zone, std, links):
+    for _ in range(10):
+        if zone in std:
+            return std[zone]
+        if zone not in links:
+            return None
+        zone = links[zone]
+    return None
+
+
+def utc_offset(raw):
+    """±HH:MM from a tzdb STDOFF field, which writes the minutes and seconds only when it has them."""
+    m = re.match(r"^(-)?(\d{1,2})(?::(\d{2})(?::(\d{2}))?)?$", raw)
+    if not m or (m.group(4) or "00") != "00":
+        return None
+    return f"{'-' if m.group(1) else '+'}{int(m.group(2)):02d}:{m.group(3) or '00'}"
+
+
+def population(body):
+    """The population GeoNames records per timezone, summed over its cities."""
+    with zipfile.ZipFile(io.BytesIO(body)) as z:
+        text = z.read("cities15000.txt").decode("utf-8")
+    per = {}
+    for city in csv.reader(io.StringIO(text), delimiter="\t", quoting=csv.QUOTE_NONE):
+        if len(city) > 17 and city[17] and city[14].isdigit():
+            per[city[17]] = per.get(city[17], 0) + int(city[14])
+    return per
+
+
+def rows(tar, territories, pop):
+    std, links = offsets(tar)
+    for line in member(tar, "zone.tab").splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        fields = line.split("\t")
+        if len(fields) < 3:
+            sys.exit(f"zone.tab: {line!r} has {len(fields)} fields; a row names a territory, a location and a zone")
+        territory, zone = fields[0], fields[2]
+        if territory not in territories:
+            continue
+        raw = resolve(zone, std, links)
+        if raw is None:
+            sys.exit(f"{zone}: the tarball gives it no Zone rule and no Link to one")
+        offset = utc_offset(raw)
+        if offset is None:
+            sys.exit(f"{zone}: standard offset {raw!r} is not a ±HH:MM the table can spell")
+        yield {"offset": offset, "territory": territory, "weight": max(pop.get(zone, 0), FLOOR), "zone": zone}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    p.add_argument("--cities", default=CITIES)
+    p.add_argument("--territories", default=str(TERRITORIES))
+    a = p.parse_args()
+    shipped = {r["alpha2"] for r in csv.DictReader(io.StringIO(Path(a.territories).read_text(encoding="utf-8")), delimiter="\t")}
+    pop = population(source.fetch(a.cities, a.cache, "cities15000.zip", magic=b"PK"))
+    body = source.fetch(a.source, a.cache, "tzdata-latest.tar.gz", magic=b"\x1f\x8b")
+    with tarfile.open(fileobj=io.BytesIO(body)) as tar:
+        table = sorted(rows(tar, shipped, pop), key=lambda r: r["zone"])
+        print(f"tzdb {tar.extractfile('version').read().decode('utf-8').strip()}", file=sys.stderr)
+    linked = {r["territory"] for r in table}
+    if missing := shipped - linked:
+        sys.exit(f"no zone for {sorted(missing)}; a parent row without a child is a load error")
+    if (floored := sum(1 for r in table if r["weight"] == FLOOR)) > len(table) // 2:
+        sys.exit(f"{floored} of {len(table)} zones fell to the floor; GeoNames' population or timezone column has moved")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/tld.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/tld.tsv from the IANA Root Zone Database.
+
+    data-import/tld.py [--source URL_OR_FILE] [--zone URL_OR_FILE] [--cache DIR] [--out FILE]
+
+A TLD ships when the database names a manager for it; one it marks "Not assigned" is a
+delegation the root zone no longer holds, and the zone file is fetched to prove the two
+agree. The key is the A-label the root zone holds; `unicode` is the form the database
+displays, less the bidi marks it wraps a right-to-left label in, and proved to punycode
+back to the key.
+"""
+import argparse
+import html
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://www.iana.org/domains/root/db"
+ZONE = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "tld.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["tld", "type", "unicode"]
+TYPES = ("country-code", "generic", "generic-restricted", "infrastructure", "sponsored")
+BIDI = {0x200E: None, 0x200F: None}
+ROW = re.compile(r'<a href="/domains/root/db/([^"]+)\.html">([^<]+)</a>.*?</td>\s*<td>([^<]*)</td>\s*<td>(.*?)</td>', re.S)
+
+
+def rows(page):
+    for a_label, shown, kind, manager in ROW.findall(page):
+        if "Not assigned" in manager:
+            continue
+        kind = kind.strip()
+        if kind not in TYPES:
+            sys.exit(f"{a_label}: the database types it {kind!r}, which is none of {TYPES}")
+        yield {"tld": "." + a_label, "type": kind, "unicode": html.unescape(shown).strip().translate(BIDI)}
+
+
+def prove_each_unicode_form_is_its_key(table):
+    """The displayed form is the key's own spelling, which punycode encodes back to."""
+    for r in table:
+        label = r["unicode"][1:]
+        encoded = label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
+        if "." + encoded != r["tld"]:
+            sys.exit(f"{r['tld']}: the database displays {r['unicode']!r}, which is no spelling of it")
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--zone", default=ZONE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    table = sorted(rows(source.fetch(a.source, a.cache, "root-db.html").decode("utf-8")), key=lambda r: r["tld"])
+    prove_each_unicode_form_is_its_key(table)
+    listing = source.fetch(a.zone, a.cache, "tlds-alpha-by-domain.txt").decode("utf-8")
+    delegated = {"." + line.strip().lower() for line in listing.splitlines() if line.strip() and not line.startswith("#")}
+    shipped = {r["tld"] for r in table}
+    if len(table) != len(delegated) or shipped != delegated:
+        sys.exit(f"the database assigns {len(table)} rows where the root zone holds {len(delegated)} TLDs: {sorted(shipped ^ delegated)[:10]}")
+    tsv.write(a.out, COLUMNS, table)
+
+
+if __name__ == "__main__":
+    main()

data-import/tsv.py

@@ -0,0 +1,18 @@
+"""A table written as the loader admits it."""
+import re
+import sys
+from pathlib import Path
+
+
+def write(path, columns, rows):
+    """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
+    lines = ["\t".join(columns)]
+    for row in rows:
+        cells = [str(row[c]).strip() for c in columns]
+        if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
+            raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
+        lines.append("\t".join(cells))
+    if len(lines) < 2:
+        raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
+    Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)

data-import/useragent.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Rebuild data/misc/useragent.tsv from microlinkhq/top-user-agents (MIT).
+
+    data-import/useragent.py [--source URL] [--cache DIR] [--out FILE]
+
+The desktop and mobile lists partition the whole, so `device` is read rather than
+parsed. A row ships when the string names a browser and an operating system both.
+"""
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+import source
+import tsv
+
+SOURCE = "https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src"
+OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "useragent.tsv"
+CACHE = Path(__file__).resolve().parent / "cache"
+COLUMNS = ["browser", "device", "os", "ua"]
+# First match wins: a Chromium fork this list does not name ships as Chrome.
+BROWSERS = [("Edge", r"Edg(A|iOS)?/"), ("Opera", r"OPR/"), ("Samsung Internet", r"SamsungBrowser/"),
+            ("Chrome", r"(Chrome|CriOS)/"), ("Firefox", r"(Firefox|FxiOS)/"), ("Safari", r"Version/[\d.]+ .*Safari")]
+SYSTEMS = [("iOS", r"iPhone|iPad|CPU OS "), ("Android", r"Android"), ("ChromeOS", r"CrOS"),
+           ("Windows", r"Windows NT"), ("macOS", r"Macintosh|Mac OS X"), ("Linux", r"X11.*Linux|Ubuntu")]
+
+
+def named(table, ua):
+    for name, pattern in table:
+        if re.search(pattern, ua):
+            return name
+    return ""
+
+
+def rows(desktop, mobile):
+    seen, kept = set(), []
+    for device, uas in (("desktop", desktop), ("mobile", mobile)):
+        for ua in uas:
+            browser, os = named(BROWSERS, ua), named(SYSTEMS, ua)
+            if not browser or not os:
+                print(f"dropped, naming no {'browser' if not browser else 'operating system'}: {ua}", file=sys.stderr)
+                continue
+            if ua in seen:
+                sys.exit(f"{ua}: listed twice, which would draw it twice as often")
+            seen.add(ua)
+            kept.append({"browser": browser, "device": device, "os": os, "ua": ua})
+    if len(kept) < 0.8 * (len(desktop) + len(mobile)):
+        sys.exit(f"only {len(kept)} of {len(desktop) + len(mobile)} strings named both; the tokens have moved")
+    return kept
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--cache", default=str(CACHE))
+    p.add_argument("--source", default=SOURCE)
+    p.add_argument("--out", default=str(OUT))
+    a = p.parse_args()
+    lists = [json.loads(source.fetch(f"{a.source}/{n}.json", a.cache, f"top-user-agents-{n}.json")) for n in ("desktop", "mobile")]
+    tsv.write(a.out, COLUMNS, sorted(rows(*lists), key=lambda r: (r["browser"], r["os"], r["ua"])))
+
+
+if __name__ == "__main__":
+    main()

data-import/xlsx.py

@@ -0,0 +1,21 @@
+"""The rows of an xlsx sheet."""
+import io
+import xml.etree.ElementTree as ET
+import zipfile
+
+NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main", "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships"}
+
+
+def rows(data, sheet=None):
+    """The rows of an xlsx sheet named sheet, the first sheet by default, each a list of cell texts."""
+    z = zipfile.ZipFile(io.BytesIO(data))
+    strings = ["".join(t.text or "" for t in si.iter("{%s}t" % NS["m"])) for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", NS)]
+    rels = {r.get("Id"): r.get("Target") for r in ET.fromstring(z.read("xl/_rels/workbook.xml.rels"))}
+    sheets = {s.get("name"): rels[s.get("{%s}id" % NS["r"])] for s in ET.fromstring(z.read("xl/workbook.xml")).iter("{%s}sheet" % NS["m"])}
+    target = sheets[sheet] if sheet else next(iter(sheets.values()))
+    for row in ET.fromstring(z.read("xl/" + target)).findall(".//m:row", NS):
+        cells = []
+        for c in row.findall("m:c", NS):
+            v = c.find("m:v", NS)
+            cells.append("" if v is None else strings[int(v.text)] if c.get("t") == "s" else v.text)
+        yield cells

data.go

@@ -1,194 +1,175 @@
 package fejkdata
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
-// label prefixes file names in errors; onDisk marks path as a directory that must
-// exist.
-type dataSource struct {
-	fsys   fs.FS
-	label  string
-	onDisk bool
-	path   string
-	root   string
-}
-
-func (s dataSource) name(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-// loadData loads every source into one namespace tree and returns its root
-// children. A directory becomes a folder; each *.json file in it compiles to a node
-// keyed by its base name (address.json -> "address"); each subdirectory becomes a
-// nested folder, so folders turn into dot-path segments. Sources merge left to right:
-// matching folders merge by their children, and any other clash is won by the last
-// source loaded. Once merged, linkRefs binds every reference against the
-// final tree.
-func loadData(sources []dataSource) (map[string]node, error) {
-	root := map[string]node{}
+// loadSources merges sources in order, the last winning a clash. A source whose manifest
+// carries an index leaves its categories unloaded; every other source loads here, with
+// the unloaded categories it needs.
+// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
+func loadSources(sources []datafiles.Source) (folder, error) {
+	root := folder{children: map[string]node{}}
 	for _, src := range sources {
-		if src.onDisk {
-			if src.path == "" {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.path)
+		m, err := src.Manifest()
 		if err != nil {
-				return nil, err
+			return folder{}, err
 		}
-			if !info.IsDir() {
-				return nil, fmt.Errorf("%s is not a directory", src.path)
+		if err := addSource(&root, src, m); err != nil {
+			return folder{}, err
 		}
 	}
-		dir := src.root
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
-		if err != nil {
-			return nil, err
-		}
-		mergeChildren(root, g.children)
-	}
-	if len(root) == 0 {
-		return nil, fmt.Errorf("no .json data found")
-	}
-	if err := linkTables(root); err != nil {
-		return nil, err
-	}
-	if err := linkRefs(root); err != nil {
-		return nil, err
-	}
-	if err := checkNoCycles(root); err != nil {
-		return nil, err
-	}
-	if err := checkScope(treeScope(root)); err != nil {
-		return nil, err
-	}
-	return root, nil
+	return root, loadReached(&root, categorySites(&root), nil)
 }
 
-// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
-// by name, so the tree is built deterministically. Empty subdirectories (no JSON
-// anywhere under them) are skipped rather than added as empty namespaces.
-func loadDir(src dataSource, dir string) (*folder, error) {
-	entries, err := fs.ReadDir(src.fsys, dir)
-	if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.name(dir), err)
+func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
+	src = src.Mounted(m.Namespace)
+	if m.Index != nil {
+		placeIndex(root, src, m.Index)
+		return nil
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
-	for _, e := range entries {
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
+	if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
+		return err
 	}
-	}
-	for _, e := range entries {
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
-		}
-		full := path.Join(dir, e.Name())
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
-		}
-	}
-	return g, nil
+	mergeFolder(root, g)
+	return nil
 }
 
-// categoryFiles is what a category may name beside itself: the rows files of its
-// directory, each marked once a category names it.
-type categoryFiles struct {
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-// rows reads the rows file a category names beside it.
-func (c *categoryFiles) rows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
+func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
+	return func(c datafiles.Category) error {
+		n, err := compileCategory(c)
 		if err != nil {
 			return err
 		}
-	if len(child.children) == 0 {
+		place(c.Folders).put(c.Name, n)
 		return nil
 	}
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.name(full), err)
-	}
-	g.children[name] = child
-	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
-// is skipped.
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
-	if !strings.HasSuffix(file, ".json") {
-		return nil
+// madeFolder is the folder that dir names under root, made where it is missing or a category.
+func madeFolder(root *folder, dir []string) *folder {
+	g := root
+	for _, seg := range dir {
+		sub, isFolder := g.children[seg].(*folder)
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.put(seg, sub)
 		}
-	name := strings.TrimSuffix(file, ".json")
-	if err := checkName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.name(full), err)
+		g = sub
 	}
-	b, err := fs.ReadFile(src.fsys, full)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.name(full), err)
-	}
-	g.children[name] = n
-	return nil
+	return g
 }
 
-// mergeChildren overlays src onto dst. Two folders under the same key merge
-// recursively (so locales/categories from several paths combine); every other
-// key is replaced, making the last-loaded directory win on a conflict.
-func mergeChildren(dst, src map[string]node) {
-	for k, v := range src {
-		if dg, ok := dst[k].(*folder); ok {
+func mergeFolder(dst, src *folder) {
+	for k, v := range src.children {
+		if dg, ok := dst.children[k].(*folder); ok {
 			if sg, ok := v.(*folder); ok {
-				mergeChildren(dg.children, sg.children)
+				mergeFolder(dg, sg)
 				continue
 			}
 		}
-		dst[k] = v
+		dst.put(k, v)
 	}
 }
+
+// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
+// single inline node.
+type nodeSet func(fn func(label string, n node) error) error
+
+// categorySite is a loaded category, where it sits and its own dot path.
+type categorySite struct {
+	categoryAt
+	n    node
+	path string
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, siteIn(dir, g, name))
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func siteIn(dir []string, in *folder, name string) categorySite {
+	return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
+}
+
+func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
+
+func siteNodes(sites []categorySite) nodeSet {
+	return func(fn func(label string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func inlineNodes(n node, label string) nodeSet {
+	return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
+}
+
+// pipeline is the one load pipeline, run over one nodeSet; its fields are where
+// entry points differ.
+type pipeline struct {
+	nodes   nodeSet
+	resolve func() error
+	// Set by a caller that proves the columns against their Go types itself.
+	typedByGo bool
+}
+
+// categoryPipeline is the pipeline over the categories of one load, their references
+// resolving against root.
+func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
+	return pipeline{
+		nodes: siteNodes(sites),
+		resolve: func() error {
+			if err := linkTables(sites); err != nil {
+				return err
+			}
+			return resolveCategoryTemplates(sites, root)
+		},
+	}
+}
+
+func (p pipeline) run() error {
+	if err := p.resolve(); err != nil {
+		return err
+	}
+	if err := checkNoCycles(p.nodes); err != nil {
+		return err
+	}
+	settleRecords(p.nodes)
+	if !p.typedByGo {
+		if err := checkColumns(p.nodes); err != nil {
+			return err
+		}
+	}
+	return checkNodeFences(p.nodes)
+}
+
+// checkNodeFences runs the per-node fences over s, each over every node before the next,
+// so which of several broken nodes is reported does not depend on the walk. Its walks
+// recurse unguarded, so pipeline.run refuses cycles first.
+func checkNodeFences(s nodeSet) error {
+	mem := renderCounts{}
+	if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
+		return err
+	}
+	return s((&valueProof{}).checkDatatype)
+}

data/en_US/.fejkdata.json

@@ -0,0 +1,27 @@
+{
+	"index": {
+		"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"en_US.color": {"paths":[""]},
+		"en_US.company": {"paths":["","base","suffix"]},
+		"en_US.date": {"paths":[""]},
+		"en_US.email": {"paths":["","domain","local","local.n"]},
+		"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
+		"en_US.ip": {"paths":[""]},
+		"en_US.itin": {"paths":["","area","group","serial"]},
+		"en_US.last-name": {"paths":["","count","name"]},
+		"en_US.person": {"paths":["","first","last","prefix","sex"]},
+		"en_US.phone": {"paths":["","area","exchange","line"]},
+		"en_US.phone-exchange": {"paths":[""]},
+		"en_US.price": {"paths":["","amt","cents"]},
+		"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
+		"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
+		"en_US.ssn": {"paths":["","area","group","serial"]},
+		"en_US.time": {"paths":[""]},
+		"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
+		"en_US.url": {"paths":["","host","path"]},
+		"en_US.username": {"paths":["","n"]},
+		"en_US.version": {"paths":["","n","pre","suffix"]},
+		"en_US.word": {"paths":[""]}
+	},
+	"namespace": "en_US"
+}

data/en_US/address.json

@@ -1,8 +1,8 @@
 {
-  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{/geo.US.address.street}",
-  "street-number": "{/geo.US.address.number}",
-  "locality": "{/geo.US.address.locality}",
-  "region": "{/geo.US.address.region}",
-  "postal-code": "{/geo.US.address.postal-code}"
+  "format": "{/geo.US.address as a}{a}",
+  "street": "{a.street}",
+  "street-number": "{a.street-number}",
+  "locality": "{a.locality}",
+  "region": "{a.region}",
+  "postal-code": "{a.postal-code}"
 }

data/en_US/date.json

@@ -1,13 +1 @@
-{
-  "format": "{month}/{day}/{year}",
-  "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
-  "day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"],
-  "year": [
-    "197{digits(1)}",
-    "198{digits(1)}",
-    { "format": "199{digits(1)}", "weight": 2 },
-    { "format": "200{digits(1)}", "weight": 3 },
-    { "format": "201{digits(1)}", "weight": 3 },
-    { "format": "202{digits(1)}", "weight": 2 }
-  ]
-}
+"{date(1970-01-01,2029-12-31,'01/02/2006')}"

data/en_US/first-name.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "first-name.tsv", "name": "name", "parent": "sex", "weight": "count" }

data/en_US/fs.go

@@ -0,0 +1,19 @@
+// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
+// geo.US, which package geous holds.
+package enus
+
+import (
+	"embed"
+	"io/fs"
+
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
+// data stands in for geo.US.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }

data/en_US/fs_test.go

@@ -0,0 +1,22 @@
+package enus_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	enus "github.com/larvit/fejkdata/data/en_US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/en_US/itin.json

@@ -0,0 +1,16 @@
+{
+  "format": "{area}-{group}-{serial}",
+  "area": "9{digits(2)}",
+  "group": [
+    { "format": "{int(50,65)}", "weight": 16 },
+    { "format": "{int(70,88)}", "weight": 19 },
+    { "format": "{int(90,92)}", "weight": 3 },
+    { "format": "{int(94,99)}", "weight": 6 }
+  ],
+  "serial": [
+    { "format": "000{int(1,9)}", "weight": 9 },
+    { "format": "00{int(10,99)}", "weight": 90 },
+    { "format": "0{int(100,999)}", "weight": 900 },
+    { "format": "{int(1000,9999)}", "weight": 9000 }
+  ]
+}

data/en_US/last-name.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "last-name.tsv", "key": "name", "weight": "count" }

data/en_US/person.json

@@ -1,7 +1,7 @@
 {
-  "format": "{prefix}{femalefirst|malefirst} {last}",
-  "femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"],
-  "malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"],
-  "last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"],
-  "prefix": ["", { "format": "{title} ", "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], "weight": 0.1 }]
+  "format": "{prefix}{first} {last}",
+  "first": "{.first-name as n}{n.name}",
+  "last": "{.last-name.name}",
+  "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
+  "sex": "{n..sex.name}"
 }

data/en_US/phone-exchange.json

@@ -0,0 +1,17 @@
+[
+  { "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
+  { "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
+  { "format": "{int(2,9)}10", "weight": 8 },
+  { "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
+  { "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
+  { "format": "55{int(0,4)}", "weight": 5 },
+  { "format": "55{int(6,9)}", "weight": 4 },
+  { "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
+  { "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
+  { "format": "95{int(1,7)}", "weight": 7 },
+  { "format": "96{digits(1)}", "weight": 10 },
+  { "format": "97{int(0,5)}", "weight": 6 },
+  { "format": "97{int(7,9)}", "weight": 3 }
+]

data/en_US/phone.json

@@ -1,15 +1,15 @@
 [
   {
-    "format": "({area}) {exch}-{line}",
+    "format": "({area}) {exchange}-{line}",
     "weight": 2,
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   },
   {
-    "format": "{area}-{exch}-{line}",
+    "format": "{area}-{exchange}-{line}",
     "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
-    "exch": "{int(100,999)}",
+    "exchange": "{.phone-exchange}",
     "line": "{digits(4)}"
   }
 ]

data/en_US/sex.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "sex.tsv", "key": "code", "name": "name" }

data/en_US/sex.tsv

@@ -0,0 +1,3 @@
+code	name
+f	female
+m	male

data/en_US/ssn.json

@@ -1 +1,19 @@
-"{int(100,999)}-{digits(2)}-{digits(4)}"
+{
+  "format": "{area}-{group}-{serial}",
+  "area": [
+    { "format": "00{int(1,9)}", "weight": 9 },
+    { "format": "0{int(10,99)}", "weight": 90 },
+    { "format": "{int(100,665)}", "weight": 566 },
+    { "format": "{int(667,899)}", "weight": 233 }
+  ],
+  "group": [
+    { "format": "0{int(1,9)}", "weight": 9 },
+    { "format": "{int(10,99)}", "weight": 90 }
+  ],
+  "serial": [
+    { "format": "000{int(1,9)}", "weight": 9 },
+    { "format": "00{int(10,99)}", "weight": 90 },
+    { "format": "0{int(100,999)}", "weight": 900 },
+    { "format": "{int(1000,9999)}", "weight": 9000 }
+  ]
+}

data/en_US/time.json

@@ -1,6 +1 @@
-{
-  "format": "{hour}:{minute} {ampm}",
-  "hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
-  "minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] },
-  "ampm": ["AM", "PM"]
-}
+"{time('3:04 PM')}"

data/en_US/title.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "title.tsv", "name": "name", "parent": "sex", "weight": "share" }

data/en_US/title.tsv

@@ -0,0 +1,11 @@
+name	sex	share
+Dr	f	6
+Dr	m	6
+Miss	f	4
+Mr	m	60
+Mrs	f	22
+Ms	f	30
+Mx	f	1
+Mx	m	1
+Prof	f	3
+Prof	m	3

data/geo/SE/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
+		"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
+		"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
+		"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
+		"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
+	},
+	"namespace": "geo.SE"
+}

data/geo/SE/address.json

@@ -1,7 +1,7 @@
 {
-  "format": "{street} {number}\n{postal-code} {locality}",
-  "street": "{.street.name}",
-  "number": [
+  "format": "{street} {street-number}\n{postal-code} {locality}",
+  "street": "{.locality as l}{l.street.name}",
+  "street-number": [
     "{int(1,9)}",
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 0.2 },
@@ -9,6 +9,6 @@
     { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
     { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
   ],
-  "postal-code": "{.postal-code.code}",
-  "locality": "{.locality.name}"
+  "postal-code": "{l.postal-code.code}",
+  "locality": "{l.name}"
 }

data/geo/SE/fs.go

@@ -0,0 +1,15 @@
+// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
+package geose
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/SE/fs_test.go

@@ -0,0 +1,22 @@
+package geose_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geose "github.com/larvit/fejkdata/data/geo/SE"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/geo/SE/locality.tsv

@@ -866,7 +866,6 @@ Nynäshamn	0192	16447	58.9034	17.9479
 Nyvång	1277	200	56.1333	12.9000
 Nyåker	2403	200	63.7500	19.3167
 Nälden	2309	986	63.3500	14.2500
-Nämdö	0120	200	59.1833	18.6833
 Nässjö	0682	18639	57.6531	14.6968
 Näsum	1272	1274	56.1667	14.4833
 Näsviken	2184	994	61.7667	16.8667

data/geo/SE/postal-code.tsv

@@ -949,7 +949,6 @@ code	locality
 130 25	Ingmarsö
 130 33	Gällnöby
 130 34	Skälvik
-130 36	Nämdö
 130 37	Stavsnäs
 130 38	Runmarö
 130 39	Sandhamn

data/geo/US/.fejkdata.json

@@ -0,0 +1,11 @@
+{
+	"index": {
+		"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
+		"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
+		"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
+		"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
+		"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
+		"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
+	},
+	"namespace": "geo.US"
+}

data/geo/US/address.json

@@ -1,13 +1,13 @@
 {
-  "format": "{number} {street}\n{locality}, {region} {postal-code}",
-  "street": "{.street.name}",
-  "number": [
+  "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
+  "street": "{.locality as l}{l.street.name}",
+  "street-number": [
     "{int(10,99)}",
     { "format": "{int(100,999)}", "weight": 2 },
     { "format": "{int(1000,9999)}", "weight": 0.5 },
     { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
   ],
-  "locality": "{.locality.name}",
-  "region": "{.region.abbr}",
-  "postal-code": "{.postal-code.code}"
+  "locality": "{l.name}",
+  "region": "{l..municipality..region.abbr}",
+  "postal-code": "{l.postal-code.code}"
 }

data/geo/US/fs.go

@@ -0,0 +1,15 @@
+// Package geous is fejkdata's US places module, whose categories are geo.US.*.
+package geous
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/geo/US/fs_test.go

@@ -0,0 +1,22 @@
+package geous_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	geous "github.com/larvit/fejkdata/data/geo/US"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/.fejkdata.json

@@ -0,0 +1,25 @@
+{
+	"index": {
+		"misc.car": {"paths":["","make","model"]},
+		"misc.coordinate": {"paths":["","lat","lon"]},
+		"misc.creditcard": {"paths":["","d"]},
+		"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
+		"misc.datetime": {"paths":[""]},
+		"misc.emoji": {"paths":[""]},
+		"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
+		"misc.httpstatus": {"paths":["","code","reason"]},
+		"misc.language": {"paths":["","code","code3","name"]},
+		"misc.loglevel": {"paths":["","code","keyword","severity"]},
+		"misc.mac": {"paths":[""]},
+		"misc.mimetype": {"paths":["","ext","type"]},
+		"misc.objectid": {"paths":[""]},
+		"misc.port": {"paths":["","number","service"]},
+		"misc.protocol": {"paths":["","keyword","name","number"]},
+		"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
+		"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
+		"misc.tld": {"paths":["","tld","type","unicode"]},
+		"misc.useragent": {"paths":["","browser","device","os","ua"]},
+		"misc.uuid": {"paths":["","variant"]}
+	},
+	"namespace": "misc"
+}

data/misc/car.json

@@ -1,10 +1 @@
-[
-  { "format": "{maker} {model}", "maker": "BMW", "model": ["3 Series", "5 Series", "X3", "X5"] },
-  { "format": "{maker} {model}", "maker": "Ford", "model": ["Fiesta", "Focus", "Mustang", "Explorer"] },
-  { "format": "{maker} {model}", "maker": "Honda", "model": ["Civic", "Accord", "CR-V", "Jazz"] },
-  { "format": "{maker} {model}", "maker": "Mercedes-Benz", "model": ["A-Class", "C-Class", "E-Class", "GLC"] },
-  { "format": "{maker} {model}", "maker": "Tesla", "model": ["Model 3", "Model S", "Model X", "Model Y"] },
-  { "format": "{maker} {model}", "maker": "Toyota", "model": ["Corolla", "Camry", "RAV4", "Yaris"] },
-  { "format": "{maker} {model}", "maker": "Volkswagen", "model": ["Golf", "Passat", "Polo", "Tiguan"] },
-  { "format": "{maker} {model}", "maker": "Volvo", "model": ["XC40", "XC60", "XC90", "V60"] }
-]
+{ "format": "{make} {model}", "rows": "car.tsv" }

data/misc/car.tsv

@@ -0,0 +1,33 @@
+make	model
+BMW	3 Series
+BMW	5 Series
+BMW	X3
+BMW	X5
+Ford	Explorer
+Ford	Fiesta
+Ford	Focus
+Ford	Mustang
+Honda	Accord
+Honda	CR-V
+Honda	Civic
+Honda	Jazz
+Mercedes-Benz	A-Class
+Mercedes-Benz	C-Class
+Mercedes-Benz	E-Class
+Mercedes-Benz	GLC
+Tesla	Model 3
+Tesla	Model S
+Tesla	Model X
+Tesla	Model Y
+Toyota	Camry
+Toyota	Corolla
+Toyota	RAV4
+Toyota	Yaris
+Volkswagen	Golf
+Volkswagen	Passat
+Volkswagen	Polo
+Volkswagen	Tiguan
+Volvo	V60
+Volvo	XC40
+Volvo	XC60
+Volvo	XC90

data/misc/country.json

@@ -1 +0,0 @@
-{ "format": "{name}", "rows": "country.tsv", "key": "alpha2", "name": "name" }

data/misc/country.tsv

@@ -1,242 +0,0 @@
-alpha2	alpha3	calling-code	capital	currency	flag	languages	name	numeric	tld
-AD	AND	376	Andorra la Vella	EUR	🇦🇩	ca	Andorra	020	.ad
-AE	ARE	971	Abu Dhabi	AED	🇦🇪	ar-AE,fa,en,hi,ur	United Arab Emirates	784	.ae
-AF	AFG	93	Kabul	AFN	🇦🇫	fa-AF,ps,uz-AF,tk	Afghanistan	004	.af
-AG	ATG	1-268	St. John's	XCD	🇦🇬	en-AG	Antigua & Barbuda	028	.ag
-AI	AIA	1-264	The Valley	XCD	🇦🇮	en-AI	Anguilla	660	.ai
-AL	ALB	355	Tirana	ALL	🇦🇱	sq,el	Albania	008	.al
-AM	ARM	374	Yerevan	AMD	🇦🇲	hy	Armenia	051	.am
-AO	AGO	244	Luanda	AOA	🇦🇴	pt-AO	Angola	024	.ao
-AR	ARG	54	Buenos Aires	ARS	🇦🇷	es-AR,en,it,de,fr,gn	Argentina	032	.ar
-AS	ASM	1-684	Pago Pago	USD	🇦🇸	en-AS,sm,to	American Samoa	016	.as
-AT	AUT	43	Vienna	EUR	🇦🇹	de-AT,hr,hu,sl	Austria	040	.at
-AU	AUS	61	Canberra	AUD	🇦🇺	en-AU	Australia	036	.au
-AW	ABW	297	Oranjestad	AWG	🇦🇼	nl-AW,pap,es,en	Aruba	533	.aw
-AX	ALA	358	Mariehamn	EUR	🇦🇽	sv-AX	Åland Islands	248	.ax
-AZ	AZE	994	Baku	AZN	🇦🇿	az,ru,hy	Azerbaijan	031	.az
-BA	BIH	387	Sarajevo	BAM	🇧🇦	bs,hr-BA,sr-BA	Bosnia	070	.ba
-BB	BRB	1-246	Bridgetown	BBD	🇧🇧	en-BB	Barbados	052	.bb
-BD	BGD	880	Dhaka	BDT	🇧🇩	bn-BD,en	Bangladesh	050	.bd
-BE	BEL	32	Brussels	EUR	🇧🇪	nl-BE,fr-BE,de-BE	Belgium	056	.be
-BF	BFA	226	Ouagadougou	XOF	🇧🇫	fr-BF,mos	Burkina Faso	854	.bf
-BG	BGR	359	Sofia	EUR	🇧🇬	bg,tr-BG,rom	Bulgaria	100	.bg
-BH	BHR	973	Manama	BHD	🇧🇭	ar-BH,en,fa,ur	Bahrain	048	.bh
-BI	BDI	257	Gitega	BIF	🇧🇮	fr-BI,rn	Burundi	108	.bi
-BJ	BEN	229	Porto-Novo	XOF	🇧🇯	fr-BJ	Benin	204	.bj
-BL	BLM	590	Gustavia	EUR	🇧🇱	fr	St. Barthélemy	652	.gp
-BM	BMU	1-441	Hamilton	BMD	🇧🇲	en-BM,pt	Bermuda	060	.bm
-BN	BRN	673	Bandar Seri Begawan	BND	🇧🇳	ms-BN,en-BN	Brunei	096	.bn
-BO	BOL	591	Sucre	BOB	🇧🇴	es-BO,qu,ay	Bolivia	068	.bo
-BR	BRA	55	Brasilia	BRL	🇧🇷	pt-BR,es,en,fr	Brazil	076	.br
-BS	BHS	1-242	Nassau	BSD	🇧🇸	en-BS	Bahamas	044	.bs
-BT	BTN	975	Thimphu	INR	🇧🇹	dz	Bhutan	064	.bt
-BW	BWA	267	Gaborone	BWP	🇧🇼	en-BW,tn-BW	Botswana	072	.bw
-BY	BLR	375	Minsk	BYN	🇧🇾	be,ru	Belarus	112	.by
-BZ	BLZ	501	Belmopan	BZD	🇧🇿	en-BZ,es	Belize	084	.bz
-CA	CAN	1	Ottawa	CAD	🇨🇦	en-CA,fr-CA,iu	Canada	124	.ca
-CC	CCK	61	West Island	AUD	🇨🇨	ms-CC,en	Cocos Islands	166	.cc
-CD	COD	243	Kinshasa	CDF	🇨🇩	fr-CD,ln,ktu,kg,sw,lua	Congo - Kinshasa	180	.cd
-CF	CAF	236	Bangui	XAF	🇨🇫	fr-CF,sg,ln,kg	Central African Republic	140	.cf
-CG	COG	242	Brazzaville	XAF	🇨🇬	fr-CG,kg,ln-CG	Congo - Brazzaville	178	.cg
-CH	CHE	41	Bern	CHF	🇨🇭	de-CH,fr-CH,it-CH,rm	Switzerland	756	.ch
-CI	CIV	225	Yamoussoukro	XOF	🇨🇮	fr-CI	Côte d’Ivoire	384	.ci
-CK	COK	682	Avarua	NZD	🇨🇰	en-CK,mi	Cook Islands	184	.ck
-CL	CHL	56	Santiago	CLP	🇨🇱	es-CL	Chile	152	.cl
-CM	CMR	237	Yaounde	XAF	🇨🇲	en-CM,fr-CM	Cameroon	120	.cm
-CN	CHN	86	Beijing	CNY	🇨🇳	zh-CN,yue,wuu,dta,ug,za	China	156	.cn
-CO	COL	57	Bogota	COP	🇨🇴	es-CO	Colombia	170	.co
-CR	CRI	506	San Jose	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
-CU	CUB	53	Havana	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
-CV	CPV	238	Praia	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
-CW	CUW	599	 Willemstad	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
-CX	CXR	61	Flying Fish Cove	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
-CY	CYP	357	Nicosia	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
-CZ	CZE	420	Prague	CZK	🇨🇿	cs,sk	Czechia	203	.cz
-DE	DEU	49	Berlin	EUR	🇩🇪	de	Germany	276	.de
-DJ	DJI	253	Djibouti	DJF	🇩🇯	fr-DJ,ar,so-DJ,aa	Djibouti	262	.dj
-DK	DNK	45	Copenhagen	DKK	🇩🇰	da-DK,en,fo,de-DK	Denmark	208	.dk
-DM	DMA	1-767	Roseau	XCD	🇩🇲	en-DM	Dominica	212	.dm
-DO	DOM	1-809	Santo Domingo	DOP	🇩🇴	es-DO	Dominican Republic	214	.do
-DZ	DZA	213	Algiers	DZD	🇩🇿	ar-DZ	Algeria	012	.dz
-EC	ECU	593	Quito	USD	🇪🇨	es-EC	Ecuador	218	.ec
-EE	EST	372	Tallinn	EUR	🇪🇪	et,ru	Estonia	233	.ee
-EG	EGY	20	Cairo	EGP	🇪🇬	ar-EG,en,fr	Egypt	818	.eg
-EH	ESH	212	El-Aaiun	MAD	🇪🇭	ar,mey	Western Sahara	732	.eh
-ER	ERI	291	Asmara	ERN	🇪🇷	aa-ER,ar,tig,kun,ti-ER	Eritrea	232	.er
-ES	ESP	34	Madrid	EUR	🇪🇸	es-ES,ca,gl,eu,oc	Spain	724	.es
-ET	ETH	251	Addis Ababa	ETB	🇪🇹	am,en-ET,om-ET,ti-ET,so-ET,sid	Ethiopia	231	.et
-FI	FIN	358	Helsinki	EUR	🇫🇮	fi-FI,sv-FI,smn	Finland	246	.fi
-FJ	FJI	679	Suva	FJD	🇫🇯	en-FJ,fj	Fiji	242	.fj
-FK	FLK	500	Stanley	FKP	🇫🇰	en-FK	Falkland Islands	238	.fk
-FM	FSM	691	Palikir	USD	🇫🇲	en-FM,chk,pon,yap,kos,uli,woe,nkr,kpg	Micronesia	583	.fm
-FO	FRO	298	Torshavn	DKK	🇫🇴	fo,da-FO	Faroe Islands	234	.fo
-FR	FRA	33	Paris	EUR	🇫🇷	fr-FR,frp,br,co,ca,eu,oc	France	250	.fr
-GA	GAB	241	Libreville	XAF	🇬🇦	fr-GA	Gabon	266	.ga
-GB	GBR	44	London	GBP	🇬🇧	en-GB,cy-GB,gd	UK	826	.uk
-GD	GRD	1-473	St. George's	XCD	🇬🇩	en-GD	Grenada	308	.gd
-GE	GEO	995	Tbilisi	GEL	🇬🇪	ka,ru,hy,az	Georgia	268	.ge
-GF	GUF	594	Cayenne	EUR	🇬🇫	fr-GF	French Guiana	254	.gf
-GG	GGY	44	St Peter Port	GBP	🇬🇬	en,nrf	Guernsey	831	.gg
-GH	GHA	233	Accra	GHS	🇬🇭	en-GH,ak,ee,tw	Ghana	288	.gh
-GI	GIB	350	Gibraltar	GIP	🇬🇮	en-GI,es,it,pt	Gibraltar	292	.gi
-GL	GRL	299	Nuuk	DKK	🇬🇱	kl,da-GL,en	Greenland	304	.gl
-GM	GMB	220	Banjul	GMD	🇬🇲	en-GM,mnk,wof,wo,ff	Gambia	270	.gm
-GN	GIN	224	Conakry	GNF	🇬🇳	fr-GN	Guinea	324	.gn
-GP	GLP	590	Basse-Terre	EUR	🇬🇵	fr-GP	Guadeloupe	312	.gp
-GQ	GNQ	240	Ciudad de la Paz	XAF	🇬🇶	es-GQ,fr,pt	Equatorial Guinea	226	.gq
-GR	GRC	30	Athens	EUR	🇬🇷	el-GR,en,fr	Greece	300	.gr
-GT	GTM	502	Guatemala City	GTQ	🇬🇹	es-GT	Guatemala	320	.gt
-GU	GUM	1-671	Hagatna	USD	🇬🇺	en-GU,ch-GU	Guam	316	.gu
-GW	GNB	245	Bissau	XOF	🇬🇼	pt-GW,pov	Guinea-Bissau	624	.gw
-GY	GUY	592	Georgetown	GYD	🇬🇾	en-GY	Guyana	328	.gy
-HK	HKG	852	Hong Kong	HKD	🇭🇰	zh-HK,yue,zh,en	Hong Kong	344	.hk
-HN	HND	504	Tegucigalpa	HNL	🇭🇳	es-HN,cab,miq	Honduras	340	.hn
-HR	HRV	385	Zagreb	EUR	🇭🇷	hr-HR,sr	Croatia	191	.hr
-HT	HTI	509	Port-au-Prince	HTG	🇭🇹	ht,fr-HT	Haiti	332	.ht
-HU	HUN	36	Budapest	HUF	🇭🇺	hu-HU	Hungary	348	.hu
-ID	IDN	62	Jakarta	IDR	🇮🇩	id,en,nl,jv	Indonesia	360	.id
-IE	IRL	353	Dublin	EUR	🇮🇪	en-IE,ga-IE	Ireland	372	.ie
-IL	ISR	972	Jerusalem	ILS	🇮🇱	he,ar-IL,en-IL	Israel	376	.il
-IM	IMN	44	Douglas	GBP	🇮🇲	en,gv	Isle of Man	833	.im
-IN	IND	91	New Delhi	INR	🇮🇳	en-IN,hi,bn,te,mr,ta,ur,gu,kn,ml,or,pa,as,bh,sat,ks,ne,sd,kok,doi,mni,sit,sa,fr,lus,inc	India	356	.in
-IO	IOT	246	Diego Garcia	USD	🇮🇴	en-IO	British Indian Ocean Territory	086	.io
-IQ	IRQ	964	Baghdad	IQD	🇮🇶	ar-IQ,ku,hy	Iraq	368	.iq
-IR	IRN	98	Tehran	IRR	🇮🇷	fa-IR,ku	Iran	364	.ir
-IS	ISL	354	Reykjavik	ISK	🇮🇸	is,en,de,da,sv,no	Iceland	352	.is
-IT	ITA	39	Rome	EUR	🇮🇹	it-IT,de-IT,fr-IT,sc,ca,co,sl	Italy	380	.it
-JE	JEY	44	Saint Helier	GBP	🇯🇪	en,fr,nrf	Jersey	832	.je
-JM	JAM	1-876	Kingston	JMD	🇯🇲	en-JM	Jamaica	388	.jm
-JO	JOR	962	Amman	JOD	🇯🇴	ar-JO,en	Jordan	400	.jo
-JP	JPN	81	Tokyo	JPY	🇯🇵	ja	Japan	392	.jp
-KE	KEN	254	Nairobi	KES	🇰🇪	en-KE,sw-KE	Kenya	404	.ke
-KG	KGZ	996	Bishkek	KGS	🇰🇬	ky,uz,ru	Kyrgyzstan	417	.kg
-KH	KHM	855	Phnom Penh	KHR	🇰🇭	km,fr,en	Cambodia	116	.kh
-KI	KIR	686	Tarawa	AUD	🇰🇮	en-KI,gil	Kiribati	296	.ki
-KM	COM	269	Moroni	KMF	🇰🇲	ar,fr-KM	Comoros	174	.km
-KN	KNA	1-869	Basseterre	XCD	🇰🇳	en-KN	St. Kitts & Nevis	659	.kn
-KP	PRK	850	Pyongyang	KPW	🇰🇵	ko-KP	North Korea	408	.kp
-KR	KOR	82	Seoul	KRW	🇰🇷	ko-KR,en	South Korea	410	.kr
-KW	KWT	965	Kuwait City	KWD	🇰🇼	ar-KW,en	Kuwait	414	.kw
-KY	CYM	1-345	George Town	KYD	🇰🇾	en-KY	Cayman Islands	136	.ky
-KZ	KAZ	7	Nur-Sultan	KZT	🇰🇿	kk,ru	Kazakhstan	398	.kz
-LA	LAO	856	Vientiane	LAK	🇱🇦	lo,fr,en	Laos	418	.la
-LB	LBN	961	Beirut	LBP	🇱🇧	ar-LB,fr-LB,en,hy	Lebanon	422	.lb
-LC	LCA	1-758	Castries	XCD	🇱🇨	en-LC	St. Lucia	662	.lc
-LI	LIE	423	Vaduz	CHF	🇱🇮	de-LI	Liechtenstein	438	.li
-LK	LKA	94	Colombo	LKR	🇱🇰	si,ta,en	Sri Lanka	144	.lk
-LR	LBR	231	Monrovia	LRD	🇱🇷	en-LR	Liberia	430	.lr
-LS	LSO	266	Maseru	LSL	🇱🇸	en-LS,st,zu,xh	Lesotho	426	.ls
-LT	LTU	370	Vilnius	EUR	🇱🇹	lt,ru,pl	Lithuania	440	.lt
-LU	LUX	352	Luxembourg	EUR	🇱🇺	lb,de-LU,fr-LU	Luxembourg	442	.lu
-LV	LVA	371	Riga	EUR	🇱🇻	lv,ru,lt	Latvia	428	.lv
-LY	LBY	218	Tripoli	LYD	🇱🇾	ar-LY,it,en	Libya	434	.ly
-MA	MAR	212	Rabat	MAD	🇲🇦	ar-MA,ber,fr	Morocco	504	.ma
-MC	MCO	377	Monaco	EUR	🇲🇨	fr-MC,en,it	Monaco	492	.mc
-MD	MDA	373	Chisinau	MDL	🇲🇩	ro,ru,gag,tr	Moldova	498	.md
-ME	MNE	382	Podgorica	EUR	🇲🇪	sr,hu,bs,sq,hr,rom	Montenegro	499	.me
-MF	MAF	590	Marigot	EUR	🇲🇫	fr	St. Martin	663	.gp
-MG	MDG	261	Antananarivo	MGA	🇲🇬	fr-MG,mg	Madagascar	450	.mg
-MH	MHL	692	Majuro	USD	🇲🇭	mh,en-MH	Marshall Islands	584	.mh
-MK	MKD	389	Skopje	MKD	🇲🇰	mk,sq,tr,rmm,sr	North Macedonia	807	.mk
-ML	MLI	223	Bamako	XOF	🇲🇱	fr-ML,bm	Mali	466	.ml
-MM	MMR	95	Nay Pyi Taw	MMK	🇲🇲	my	Myanmar	104	.mm
-MN	MNG	976	Ulaanbaatar	MNT	🇲🇳	mn,ru	Mongolia	496	.mn
-MO	MAC	853	Macao	MOP	🇲🇴	zh,zh-MO,pt	Macao	446	.mo
-MP	MNP	1-670	Saipan	USD	🇲🇵	fil,tl,zh,ch-MP,en-MP	Northern Mariana Islands	580	.mp
-MQ	MTQ	596	Fort-de-France	EUR	🇲🇶	fr-MQ	Martinique	474	.mq
-MR	MRT	222	Nouakchott	MRU	🇲🇷	ar-MR,fuc,snk,fr,mey,wo	Mauritania	478	.mr
-MS	MSR	1-664	Plymouth	XCD	🇲🇸	en-MS	Montserrat	500	.ms
-MT	MLT	356	Valletta	EUR	🇲🇹	mt,en-MT	Malta	470	.mt
-MU	MUS	230	Port Louis	MUR	🇲🇺	en-MU,bho,fr	Mauritius	480	.mu
-MV	MDV	960	Male	MVR	🇲🇻	dv,en	Maldives	462	.mv
-MW	MWI	265	Lilongwe	MWK	🇲🇼	ny,yao,tum,swk	Malawi	454	.mw
-MX	MEX	52	Mexico City	MXN	🇲🇽	es-MX	Mexico	484	.mx
-MY	MYS	60	Kuala Lumpur	MYR	🇲🇾	ms-MY,en,zh,ta,te,ml,pa,th	Malaysia	458	.my
-MZ	MOZ	258	Maputo	MZN	🇲🇿	pt-MZ,vmw	Mozambique	508	.mz
-NA	NAM	264	Windhoek	NAD	🇳🇦	en-NA,af,de,hz,naq	Namibia	516	.na
-NC	NCL	687	Noumea	XPF	🇳🇨	fr-NC	New Caledonia	540	.nc
-NE	NER	227	Niamey	XOF	🇳🇪	fr-NE,ha,kr,dje	Niger	562	.ne
-NF	NFK	672	Kingston	AUD	🇳🇫	en-NF	Norfolk Island	574	.nf
-NG	NGA	234	Abuja	NGN	🇳🇬	en-NG,ha,yo,ig,ff	Nigeria	566	.ng
-NI	NIC	505	Managua	NIO	🇳🇮	es-NI,en	Nicaragua	558	.ni
-NL	NLD	31	Amsterdam	EUR	🇳🇱	nl-NL,fy-NL	Netherlands	528	.nl
-NO	NOR	47	Oslo	NOK	🇳🇴	no,nb,nn,se,fi	Norway	578	.no
-NP	NPL	977	Kathmandu	NPR	🇳🇵	ne,en	Nepal	524	.np
-NR	NRU	674	Yaren	AUD	🇳🇷	na,en-NR	Nauru	520	.nr
-NU	NIU	683	Alofi	NZD	🇳🇺	niu,en-NU	Niue	570	.nu
-NZ	NZL	64	Wellington	NZD	🇳🇿	en-NZ,mi	New Zealand	554	.nz
-OM	OMN	968	Muscat	OMR	🇴🇲	ar-OM,en,bal,ur	Oman	512	.om
-PA	PAN	507	Panama City	PAB	🇵🇦	es-PA,en	Panama	591	.pa
-PE	PER	51	Lima	PEN	🇵🇪	es-PE,qu,ay	Peru	604	.pe
-PF	PYF	689	Papeete	XPF	🇵🇫	fr-PF,ty	French Polynesia	258	.pf
-PG	PNG	675	Port Moresby	PGK	🇵🇬	en-PG,ho,meu,tpi	Papua New Guinea	598	.pg
-PH	PHL	63	Manila	PHP	🇵🇭	tl,en-PH,fil,ceb,ilo,hil,war,pam,bik,bcl,pag,mrw,tsg,mdh,cbk,krj,sgd,msb,akl,ibg,yka,mta,abx	Philippines	608	.ph
-PK	PAK	92	Islamabad	PKR	🇵🇰	ur-PK,en-PK,pa,sd,ps,brh	Pakistan	586	.pk
-PL	POL	48	Warsaw	PLN	🇵🇱	pl	Poland	616	.pl
-PM	SPM	508	Saint-Pierre	EUR	🇵🇲	fr-PM	St. Pierre & Miquelon	666	.pm
-PN	PCN	870	Adamstown	NZD	🇵🇳	en-PN	Pitcairn	612	.pn
-PR	PRI	1	San Juan	USD	🇵🇷	en-PR,es-PR	Puerto Rico	630	.pr
-PT	PRT	351	Lisbon	EUR	🇵🇹	pt-PT,mwl	Portugal	620	.pt
-PW	PLW	680	Melekeok	USD	🇵🇼	pau,sov,en-PW,tox,ja,fil,zh	Palau	585	.pw
-PY	PRY	595	Asuncion	PYG	🇵🇾	es-PY,gn	Paraguay	600	.py
-QA	QAT	974	Doha	QAR	🇶🇦	ar-QA,es	Qatar	634	.qa
-RE	REU	262	Saint-Denis	EUR	🇷🇪	fr-RE	Réunion	638	.re
-RO	ROU	40	Bucharest	RON	🇷🇴	ro,hu,rom	Romania	642	.ro
-RS	SRB	381	Belgrade	RSD	🇷🇸	sr,hu,bs,rom	Serbia	688	.rs
-RU	RUS	7	Moscow	RUB	🇷🇺	ru,tt,xal,cau,ady,kv,ce,tyv,cv,udm,tut,mns,bua,myv,mdf,chm,ba,inh,kbd,krc,av,sah,nog	Russia	643	.ru
-RW	RWA	250	Kigali	RWF	🇷🇼	rw,en-RW,fr-RW,sw	Rwanda	646	.rw
-SA	SAU	966	Riyadh	SAR	🇸🇦	ar-SA	Saudi Arabia	682	.sa
-SB	SLB	677	Honiara	SBD	🇸🇧	en-SB,tpi	Solomon Islands	090	.sb
-SC	SYC	248	Victoria	SCR	🇸🇨	en-SC,fr-SC	Seychelles	690	.sc
-SD	SDN	249	Khartoum	SDG	🇸🇩	ar-SD,en,fia	Sudan	729	.sd
-SE	SWE	46	Stockholm	SEK	🇸🇪	sv-SE,se,sma,fi-SE	Sweden	752	.se
-SG	SGP	65	Singapore	SGD	🇸🇬	cmn,en-SG,ms-SG,ta-SG,zh-SG	Singapore	702	.sg
-SH	SHN	290	Jamestown	SHP	🇸🇭	en-SH	St. Helena	654	.sh
-SI	SVN	386	Ljubljana	EUR	🇸🇮	sl,sh	Slovenia	705	.si
-SJ	SJM	47	Longyearbyen	NOK	🇸🇯	no,ru	Svalbard & Jan Mayen	744	.sj
-SK	SVK	421	Bratislava	EUR	🇸🇰	sk,hu	Slovakia	703	.sk
-SL	SLE	232	Freetown	SLE	🇸🇱	en-SL,men,tem	Sierra Leone	694	.sl
-SM	SMR	378	San Marino	EUR	🇸🇲	it-SM	San Marino	674	.sm
-SN	SEN	221	Dakar	XOF	🇸🇳	fr-SN,wo,fuc,mnk	Senegal	686	.sn
-SO	SOM	252	Mogadishu	SOS	🇸🇴	so-SO,ar-SO,it,en-SO	Somalia	706	.so
-SR	SUR	597	Paramaribo	SRD	🇸🇷	nl-SR,en,srn,hns,jv	Suriname	740	.sr
-SS	SSD	211	Juba	SSP	🇸🇸	en	South Sudan	728	.ss
-ST	STP	239	Sao Tome	STN	🇸🇹	pt-ST	São Tomé & Príncipe	678	.st
-SV	SLV	503	San Salvador	SVC	🇸🇻	es-SV	El Salvador	222	.sv
-SX	SXM	1-721	Philipsburg	XCG	🇸🇽	nl,en	Sint Maarten	534	.sx
-SY	SYR	963	Damascus	SYP	🇸🇾	ar-SY,ku,hy,arc,fr,en	Syria	760	.sy
-SZ	SWZ	268	Mbabane	SZL	🇸🇿	en-SZ,ss-SZ	Eswatini	748	.sz
-TC	TCA	1-649	Cockburn Town	USD	🇹🇨	en-TC	Turks & Caicos Islands	796	.tc
-TD	TCD	235	N'Djamena	XAF	🇹🇩	fr-TD,ar-TD,sre	Chad	148	.td
-TF	ATF	262	Port-aux-Francais	EUR	🇹🇫	fr	French Southern Territories	260	.tf
-TG	TGO	228	Lome	XOF	🇹🇬	fr-TG,ee,hna,kbp,dag,ha	Togo	768	.tg
-TH	THA	66	Bangkok	THB	🇹🇭	th,en	Thailand	764	.th
-TJ	TJK	992	Dushanbe	TJS	🇹🇯	tg,ru	Tajikistan	762	.tj
-TL	TLS	670	Dili	USD	🇹🇱	tet,pt-TL,id,en	Timor-Leste	626	.tl
-TM	TKM	993	Ashgabat	TMT	🇹🇲	tk,ru,uz	Turkmenistan	795	.tm
-TN	TUN	216	Tunis	TND	🇹🇳	ar-TN,fr	Tunisia	788	.tn
-TO	TON	676	Nuku'alofa	TOP	🇹🇴	to,en-TO	Tonga	776	.to
-TR	TUR	90	Ankara	TRY	🇹🇷	tr-TR,ku,diq,az,av	Türkiye	792	.tr
-TT	TTO	1-868	Port of Spain	TTD	🇹🇹	en-TT,hns,fr,es,zh	Trinidad & Tobago	780	.tt
-TV	TUV	688	Funafuti	AUD	🇹🇻	tvl,en,sm,gil	Tuvalu	798	.tv
-TW	TWN	886	Taipei	TWD	🇹🇼	zh-TW,zh,nan,hak	Taiwan	158	.tw
-TZ	TZA	255	Dodoma	TZS	🇹🇿	sw-TZ,en,ar	Tanzania	834	.tz
-UA	UKR	380	Kyiv	UAH	🇺🇦	uk,ru-UA,rom,pl,hu	Ukraine	804	.ua
-UG	UGA	256	Kampala	UGX	🇺🇬	en-UG,lg,sw,ar	Uganda	800	.ug
-US	USA	1	Washington	USD	🇺🇸	en-US,es-US,haw,fr	US	840	.us
-UY	URY	598	Montevideo	UYU	🇺🇾	es-UY	Uruguay	858	.uy
-UZ	UZB	998	Tashkent	UZS	🇺🇿	uz,ru,tg	Uzbekistan	860	.uz
-VA	VAT	39	Vatican City	EUR	🇻🇦	la,it,fr	Vatican City	336	.va
-VC	VCT	1-784	Kingstown	XCD	🇻🇨	en-VC,fr	St. Vincent & Grenadines	670	.vc
-VE	VEN	58	Caracas	VES	🇻🇪	es-VE	Venezuela	862	.ve
-VG	VGB	1-284	Road Town	USD	🇻🇬	en-VG	British Virgin Islands	092	.vg
-VI	VIR	1-340	Charlotte Amalie	USD	🇻🇮	en-VI	U.S. Virgin Islands	850	.vi
-VN	VNM	84	Hanoi	VND	🇻🇳	vi,en,fr,zh,km	Vietnam	704	.vn
-VU	VUT	678	Port Vila	VUV	🇻🇺	bi,en-VU,fr-VU	Vanuatu	548	.vu
-WF	WLF	681	Mata Utu	XPF	🇼🇫	wls,fud,fr-WF	Wallis & Futuna	876	.wf
-WS	WSM	685	Apia	WST	🇼🇸	sm,en-WS	Samoa	882	.ws
-YE	YEM	967	Sanaa	YER	🇾🇪	ar-YE	Yemen	887	.ye
-YT	MYT	262	Mamoudzou	EUR	🇾🇹	fr-YT	Mayotte	175	.yt
-ZA	ZAF	27	Pretoria	ZAR	🇿🇦	zu,xh,af,nso,en-ZA,tn,st,ts,ss,ve,nr	South Africa	710	.za
-ZM	ZMB	260	Lusaka	ZMW	🇿🇲	en-ZM,bem,loz,lun,lue,ny,toi	Zambia	894	.zm
-ZW	ZWE	263	Harare	ZWG	🇿🇼	en-ZW,sn,nr,nd	Zimbabwe	716	.zw

data/misc/datetime.json

@@ -0,0 +1 @@
+"{date(2000-01-01,2029-12-31,'2006-01-02T15:04:05Z')}"

data/misc/fs.go

@@ -0,0 +1,15 @@
+// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
+package misc
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// FS holds the module's categories and its manifest.
+//
+//go:embed *.json *.tsv .fejkdata.json
+var FS embed.FS
+
+// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
+func Modules() []fs.FS { return []fs.FS{FS} }

data/misc/fs_test.go

@@ -0,0 +1,22 @@
+package misc_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/larvit/fejkdata"
+	"github.com/larvit/fejkdata/data/misc"
+)
+
+func TestFSLoadsByItsManifest(t *testing.T) {
+	if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
+		t.Fatalf("FS holds no manifest: %v", err)
+	}
+	f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
+	if err != nil {
+		t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
+	}
+	if len(f.List()) == 0 {
+		t.Error("List() is empty, want the module's categories")
+	}
+}

data/misc/httpmethod.json

@@ -0,0 +1 @@
+{ "format": "{method}", "rows": "httpmethod.tsv", "key": "method" }

data/misc/httpmethod.tsv

@@ -0,0 +1,10 @@
+idempotent	method	safe
+false	CONNECT	false
+true	DELETE	false
+true	GET	true
+true	HEAD	true
+true	OPTIONS	true
+false	PATCH	false
+false	POST	false
+true	PUT	false
+true	TRACE	true

data/misc/httpstatus.tsv

@@ -1,17 +1,61 @@
 code	reason
+100	Continue
+101	Switching Protocols
+102	Processing
+103	Early Hints
 200	OK
 201	Created
+202	Accepted
+203	Non-Authoritative Information
 204	No Content
+205	Reset Content
+206	Partial Content
+207	Multi-Status
+208	Already Reported
+226	IM Used
+300	Multiple Choices
 301	Moved Permanently
 302	Found
+303	See Other
 304	Not Modified
+305	Use Proxy
+307	Temporary Redirect
+308	Permanent Redirect
 400	Bad Request
 401	Unauthorized
+402	Payment Required
 403	Forbidden
 404	Not Found
+405	Method Not Allowed
+406	Not Acceptable
+407	Proxy Authentication Required
+408	Request Timeout
 409	Conflict
-422	Unprocessable Entity
+410	Gone
+411	Length Required
+412	Precondition Failed
+413	Content Too Large
+414	URI Too Long
+415	Unsupported Media Type
+416	Range Not Satisfiable
+417	Expectation Failed
+421	Misdirected Request
+422	Unprocessable Content
+423	Locked
+424	Failed Dependency
+425	Too Early
+426	Upgrade Required
+428	Precondition Required
 429	Too Many Requests
+431	Request Header Fields Too Large
+451	Unavailable For Legal Reasons
 500	Internal Server Error
+501	Not Implemented
 502	Bad Gateway
 503	Service Unavailable
+504	Gateway Timeout
+505	HTTP Version Not Supported
+506	Variant Also Negotiates
+507	Insufficient Storage
+508	Loop Detected
+511	Network Authentication Required

data/misc/language.tsv

@@ -1,17 +1,184 @@
-code	name
-ar	Arabic
-zh	Chinese
-da	Danish
-nl	Dutch
-en	English
-fi	Finnish
-fr	French
-de	German
-it	Italian
-ja	Japanese
-no	Norwegian
-pl	Polish
-pt	Portuguese
-ru	Russian
-es	Spanish
-sv	Swedish
+code	code3	name
+aa	aar	Afar
+ab	abk	Abkhazian
+ae	ave	Avestan
+af	afr	Afrikaans
+ak	aka	Akan
+am	amh	Amharic
+an	arg	Aragonese
+ar	ara	Arabic
+as	asm	Assamese
+av	ava	Avaric
+ay	aym	Aymara
+az	aze	Azerbaijani
+ba	bak	Bashkir
+be	bel	Belarusian
+bg	bul	Bulgarian
+bi	bis	Bislama
+bm	bam	Bambara
+bn	ben	Bengali
+bo	bod	Tibetan
+br	bre	Breton
+bs	bos	Bosnian
+ca	cat	Catalan
+ce	che	Chechen
+ch	cha	Chamorro
+co	cos	Corsican
+cr	cre	Cree
+cs	ces	Czech
+cu	chu	Church Slavic
+cv	chv	Chuvash
+cy	cym	Welsh
+da	dan	Danish
+de	deu	German
+dv	div	Divehi
+dz	dzo	Dzongkha
+ee	ewe	Ewe
+el	ell	Modern Greek
+en	eng	English
+eo	epo	Esperanto
+es	spa	Spanish
+et	est	Estonian
+eu	eus	Basque
+fa	fas	Persian
+ff	ful	Fulah
+fi	fin	Finnish
+fj	fij	Fijian
+fo	fao	Faroese
+fr	fra	French
+fy	fry	Western Frisian
+ga	gle	Irish
+gd	gla	Gaelic
+gl	glg	Galician
+gn	grn	Guarani
+gu	guj	Gujarati
+gv	glv	Manx
+ha	hau	Hausa
+he	heb	Hebrew
+hi	hin	Hindi
+ho	hmo	Hiri Motu
+hr	hrv	Croatian
+ht	hat	Haitian
+hu	hun	Hungarian
+hy	hye	Armenian
+hz	her	Herero
+ia	ina	Interlingua
+id	ind	Indonesian
+ie	ile	Interlingue
+ig	ibo	Igbo
+ii	iii	Sichuan Yi
+ik	ipk	Inupiaq
+io	ido	Ido
+is	isl	Icelandic
+it	ita	Italian
+iu	iku	Inuktitut
+ja	jpn	Japanese
+jv	jav	Javanese
+ka	kat	Georgian
+kg	kon	Kongo
+ki	kik	Kikuyu
+kj	kua	Kuanyama
+kk	kaz	Kazakh
+kl	kal	Kalaallisut
+km	khm	Central Khmer
+kn	kan	Kannada
+ko	kor	Korean
+kr	kau	Kanuri
+ks	kas	Kashmiri
+ku	kur	Kurdish
+kv	kom	Komi
+kw	cor	Cornish
+ky	kir	Kirghiz
+la	lat	Latin
+lb	ltz	Luxembourgish
+lg	lug	Ganda
+li	lim	Limburgan
+ln	lin	Lingala
+lo	lao	Lao
+lt	lit	Lithuanian
+lu	lub	Luba-Katanga
+lv	lav	Latvian
+mg	mlg	Malagasy
+mh	mah	Marshallese
+mi	mri	Maori
+mk	mkd	Macedonian
+ml	mal	Malayalam
+mn	mon	Mongolian
+mr	mar	Marathi
+ms	msa	Malay
+mt	mlt	Maltese
+my	mya	Burmese
+na	nau	Nauru
+nb	nob	Norwegian Bokmål
+nd	nde	North Ndebele
+ne	nep	Nepali
+ng	ndo	Ndonga
+nl	nld	Dutch
+nn	nno	Norwegian Nynorsk
+no	nor	Norwegian
+nr	nbl	South Ndebele
+nv	nav	Navajo
+ny	nya	Chichewa
+oc	oci	Occitan
+oj	oji	Ojibwa
+om	orm	Oromo
+or	ori	Oriya
+os	oss	Ossetian
+pa	pan	Panjabi
+pi	pli	Pali
+pl	pol	Polish
+ps	pus	Pushto
+pt	por	Portuguese
+qu	que	Quechua
+rm	roh	Romansh
+rn	run	Rundi
+ro	ron	Romanian
+ru	rus	Russian
+rw	kin	Kinyarwanda
+sa	san	Sanskrit
+sc	srd	Sardinian
+sd	snd	Sindhi
+se	sme	Northern Sami
+sg	sag	Sango
+si	sin	Sinhala
+sk	slk	Slovak
+sl	slv	Slovenian
+sm	smo	Samoan
+sn	sna	Shona
+so	som	Somali
+sq	sqi	Albanian
+sr	srp	Serbian
+ss	ssw	Swati
+st	sot	Sotho, Southern
+su	sun	Sundanese
+sv	swe	Swedish
+sw	swa	Swahili
+ta	tam	Tamil
+te	tel	Telugu
+tg	tgk	Tajik
+th	tha	Thai
+ti	tir	Tigrinya
+tk	tuk	Turkmen
+tl	tgl	Tagalog
+tn	tsn	Tswana
+to	ton	Tonga
+tr	tur	Turkish
+ts	tso	Tsonga
+tt	tat	Tatar
+tw	twi	Twi
+ty	tah	Tahitian
+ug	uig	Uighur
+uk	ukr	Ukrainian
+ur	urd	Urdu
+uz	uzb	Uzbek
+ve	ven	Venda
+vi	vie	Vietnamese
+vo	vol	Volapük
+wa	wln	Walloon
+wo	wol	Wolof
+xh	xho	Xhosa
+yi	yid	Yiddish
+yo	yor	Yoruba
+za	zha	Zhuang
+zh	zho	Chinese
+zu	zul	Zulu

data/misc/loglevel.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "loglevel.tsv", "key": "code", "name": "keyword" }

data/misc/loglevel.tsv

@@ -0,0 +1,9 @@
+code	keyword	severity
+0	emerg	Emergency
+1	alert	Alert
+2	crit	Critical
+3	err	Error
+4	warning	Warning
+5	notice	Notice
+6	info	Informational
+7	debug	Debug

data/misc/mimetype.tsv

@@ -1,16 +1,702 @@
 ext	type
+.ez	application/andrew-inset
+.atom	application/atom+xml
+.atomcat	application/atomcat+xml
+.atomdeleted	application/atomdeleted+xml
+.atomsvc	application/atomsvc+xml
+.dwd	application/atsc-dwd+xml
+.held	application/atsc-held+xml
+.rsat	application/atsc-rsat+xml
+.aml	application/automationml-aml+xml
+.amlx	application/automationml-amlx+zip
+.xcs	application/calendar+xml
+.ccxml	application/ccxml+xml
+.cdfx	application/cdfx+xml
+.cdmia	application/cdmi-capability
+.cdmic	application/cdmi-container
+.cdmid	application/cdmi-domain
+.cdmio	application/cdmi-object
+.cdmiq	application/cdmi-queue
+.cpl	application/cpl+xml
+.cwl	application/cwl
+.mpd	application/dash+xml
+.mpp	application/dash-patch+xml
+.davmount	application/davmount+xml
+.dcm	application/dicom
+.dssc	application/dssc+der
+.xdssc	application/dssc+xml
+.emma	application/emma+xml
+.emotionml	application/emotionml+xml
+.epub	application/epub+zip
+.exi	application/exi
+.exp	application/express
+.fdf	application/fdf
+.fdt	application/fdt+xml
+.pfr	application/font-tdpfr
+.geojson	application/geo+json
+.gml	application/gml+xml
 .gz	application/gzip
+.stk	application/hyperstudio
+.ink	application/inkml+xml
+.ipfix	application/ipfix
+.its	application/its+xml
+.jar	application/java-archive
 .json	application/json
+.kbl	application/kbl+xml
+.jsonld	application/ld+json
+.lgr	application/lgr+xml
+.lostxml	application/lost+xml
+.hqx	application/mac-binhex40
+.mads	application/mads+xml
+.webmanifest	application/manifest+json
+.mrc	application/marc
+.mrcx	application/marcxml+xml
+.ma	application/mathematica
+.mathml	application/mathml+xml
+.mbox	application/mbox
+.mpf	application/media-policy-dataset+xml
+.mscml	application/mediaservercontrol+xml
+.meta4	application/metalink4+xml
+.mets	application/mets+xml
+.maei	application/mmt-aei+xml
+.musd	application/mmt-usd+xml
+.mods	application/mods+xml
+.m21	application/mp21
+.mp4s	application/mp4
+.doc	application/msword
+.mxf	application/mxf
+.nq	application/n-quads
+.nt	application/n-triples
+.cjs	application/node
+.bin	application/octet-stream
+.oda	application/oda
+.opf	application/oebps-package+xml
+.ogx	application/ogg
+.oxps	application/oxps
+.relo	application/p2p-overlay+xml
+.xer	application/patch-ops-error+xml
 .pdf	application/pdf
+.pgp	application/pgp-encrypted
+.asc	application/pgp-keys
+.sig	application/pgp-signature
+.p10	application/pkcs10
+.p7m	application/pkcs7-mime
+.p7s	application/pkcs7-signature
+.p8	application/pkcs8
+.ac	application/pkix-attr-cert
+.cer	application/pkix-cert
+.crl	application/pkix-crl
+.pkipath	application/pkix-pkipath
+.pki	application/pkixcmp
+.pls	application/pls+xml
+.ai	application/postscript
+.provx	application/provenance+xml
+.cww	application/prs.cww
+.xsf	application/prs.xsf+xml
+.pskcxml	application/pskc+xml
+.rdf	application/rdf+xml
+.rif	application/reginfo+xml
+.rnc	application/relax-ng-compact-syntax
+.rl	application/resource-lists+xml
+.rld	application/resource-lists-diff+xml
+.rs	application/rls-services+xml
+.rapd	application/route-apd+xml
+.sls	application/route-s-tsid+xml
+.rusd	application/route-usd+xml
+.gbr	application/rpki-ghostbusters
+.mft	application/rpki-manifest
+.roa	application/rpki-roa
+.rtf	application/rtf
+.sbml	application/sbml+xml
+.scq	application/scvp-cv-request
+.scs	application/scvp-cv-response
+.spq	application/scvp-vp-request
+.spp	application/scvp-vp-response
+.sdp	application/sdp
+.senmlx	application/senml+xml
+.sensmlx	application/sensml+xml
+.setpay	application/set-payment-initiation
+.setreg	application/set-registration-initiation
+.shf	application/shf+xml
+.siv	application/sieve
+.smi	application/smil+xml
+.rq	application/sparql-query
+.srx	application/sparql-results+xml
+.sql	application/sql
+.gram	application/srgs
+.grxml	application/srgs+xml
+.sru	application/sru+xml
+.ssml	application/ssml+xml
+.swidtag	application/swid+xml
+.tei	application/tei+xml
+.tfi	application/thraud+xml
+.tsd	application/timestamped-data
+.toml	application/toml
+.trig	application/trig
+.ttml	application/ttml+xml
+.rsheet	application/urc-ressheet+xml
+.td	application/urc-targetdesc+xml
+.vec	application/vec+xml
+.1km	application/vnd.1000minds.decision-model+xml
+.plb	application/vnd.3gpp.pic-bw-large
+.psb	application/vnd.3gpp.pic-bw-small
+.pvb	application/vnd.3gpp.pic-bw-var
+.tcap	application/vnd.3gpp2.tcap
+.pwn	application/vnd.3m.post-it-notes
+.aso	application/vnd.accpac.simply.aso
+.imp	application/vnd.accpac.simply.imp
+.acu	application/vnd.acucobol
+.atc	application/vnd.acucorp
+.fcdt	application/vnd.adobe.formscentral.fcdt
+.fxp	application/vnd.adobe.fxp
+.xdp	application/vnd.adobe.xdp+xml
+.age	application/vnd.age
+.ahead	application/vnd.ahead.space
+.azf	application/vnd.airzip.filesecure.azf
+.azs	application/vnd.airzip.filesecure.azs
+.acc	application/vnd.americandynamics.acc
+.ami	application/vnd.amiga.ami
+.cii	application/vnd.anser-web-certificate-issue-initiation
+.atx	application/vnd.antix.game-component
+.parquet	application/vnd.apache.parquet
+.mpkg	application/vnd.apple.installer+xml
+.key	application/vnd.apple.keynote
+.m3u8	application/vnd.apple.mpegurl
+.numbers	application/vnd.apple.numbers
+.pages	application/vnd.apple.pages
+.swi	application/vnd.aristanetworks.swi
+.iota	application/vnd.astraea-software.iota
+.aep	application/vnd.audiograph
+.bmml	application/vnd.balsamiq.bmml+xml
+.mpm	application/vnd.blueice.multipass
+.bmi	application/vnd.bmi
+.rep	application/vnd.businessobjects
+.cdxml	application/vnd.chemdraw+xml
+.mmd	application/vnd.chipnuts.karaoke-mmd
+.cdy	application/vnd.cinderella
+.csl	application/vnd.citationstyles.style+xml
+.cla	application/vnd.claymore
+.rp9	application/vnd.cloanto.rp9
+.c4g	application/vnd.clonk.c4group
+.c11amc	application/vnd.cluetrust.cartomobile-config
+.c11amz	application/vnd.cluetrust.cartomobile-config-pkg
+.csp	application/vnd.commonspace
+.cdbcmsg	application/vnd.contact.cmsg
+.cmc	application/vnd.cosmocaller
+.clkx	application/vnd.crick.clicker
+.clkk	application/vnd.crick.clicker.keyboard
+.clkp	application/vnd.crick.clicker.palette
+.clkt	application/vnd.crick.clicker.template
+.clkw	application/vnd.crick.clicker.wordbank
+.wbs	application/vnd.criticaltools.wbs+xml
+.pml	application/vnd.ctc-posml
+.ppd	application/vnd.cups-ppd
+.dart	application/vnd.dart
+.rdz	application/vnd.data-vision.rdz
+.dbf	application/vnd.dbf
+.dcmp	application/vnd.dcmp+xml
+.uvf	application/vnd.dece.data
+.uvt	application/vnd.dece.ttml+xml
+.uvx	application/vnd.dece.unspecified
+.uvz	application/vnd.dece.zip
+.fe_launch	application/vnd.denovo.fcselayout-link
+.dna	application/vnd.dna
+.dpg	application/vnd.dpgraph
+.dfac	application/vnd.dreamfactory
+.ait	application/vnd.dvb.ait
+.svc	application/vnd.dvb.service
+.geo	application/vnd.dynageo
+.mag	application/vnd.ecowin.chart
+.nml	application/vnd.enliven
+.esf	application/vnd.epson.esf
+.msf	application/vnd.epson.msf
+.qam	application/vnd.epson.quickanime
+.slt	application/vnd.epson.salt
+.ssf	application/vnd.epson.ssf
+.es3	application/vnd.eszigno3+xml
+.ez2	application/vnd.ezpix-album
+.ez3	application/vnd.ezpix-package
+.mseed	application/vnd.fdsn.mseed
+.seed	application/vnd.fdsn.seed
+.gph	application/vnd.flographit
+.ftc	application/vnd.fluxtime.clip
+.fm	application/vnd.framemaker
+.fsc	application/vnd.fsc.weblaunch
+.oas	application/vnd.fujitsu.oasys
+.oa2	application/vnd.fujitsu.oasys2
+.oa3	application/vnd.fujitsu.oasys3
+.fg5	application/vnd.fujitsu.oasysgp
+.bh2	application/vnd.fujitsu.oasysprs
+.ddd	application/vnd.fujixerox.ddd
+.xdw	application/vnd.fujixerox.docuworks
+.xbd	application/vnd.fujixerox.docuworks.binder
+.fzs	application/vnd.fuzzysheet
+.txd	application/vnd.genomatix.tuxedo
+.ggb	application/vnd.geogebra.file
+.ggs	application/vnd.geogebra.slides
+.ggt	application/vnd.geogebra.tool
+.gex	application/vnd.geometry-explorer
+.gxt	application/vnd.geonext
+.g2w	application/vnd.geoplan
+.g3w	application/vnd.geospace
+.gmx	application/vnd.gmx
+.kml	application/vnd.google-earth.kml+xml
+.kmz	application/vnd.google-earth.kmz
+.xdcf	application/vnd.gov.sk.xmldatacontainer+xml
+.gqf	application/vnd.grafeq
+.gac	application/vnd.groove-account
+.ghf	application/vnd.groove-help
+.gim	application/vnd.groove-identity-message
+.grv	application/vnd.groove-injector
+.gtm	application/vnd.groove-tool-message
+.tpl	application/vnd.groove-tool-template
+.vcg	application/vnd.groove-vcard
+.hal	application/vnd.hal+xml
+.zmm	application/vnd.handheld-entertainment+xml
+.hbci	application/vnd.hbci
+.les	application/vnd.hhe.lesson-player
+.hpgl	application/vnd.hp-hpgl
+.hpid	application/vnd.hp-hpid
+.hps	application/vnd.hp-hps
+.jlt	application/vnd.hp-jlyt
+.pcl	application/vnd.hp-pcl
+.pclxl	application/vnd.hp-pclxl
+.sfd-hdstx	application/vnd.hydrostatix.sof-data
+.mpy	application/vnd.ibm.minipay
+.irm	application/vnd.ibm.rights-management
+.sc	application/vnd.ibm.secure-container
+.icc	application/vnd.iccprofile
+.igl	application/vnd.igloader
+.ivp	application/vnd.immervision-ivp
+.ivu	application/vnd.immervision-ivu
+.igm	application/vnd.insors.igm
+.xpw	application/vnd.intercon.formnet
+.i2g	application/vnd.intergeo
+.qbo	application/vnd.intu.qbo
+.qfx	application/vnd.intu.qfx
+.rcprofile	application/vnd.ipunplugged.rcprofile
+.irp	application/vnd.irepository.package+xml
+.xpr	application/vnd.is-xpr
+.fcs	application/vnd.isac.fcs
+.jam	application/vnd.jam
+.rms	application/vnd.jcp.javame.midlet-rms
+.jisp	application/vnd.jisp
+.joda	application/vnd.joost.joda-archive
+.ktz	application/vnd.kahootz
+.karbon	application/vnd.kde.karbon
+.chrt	application/vnd.kde.kchart
+.kfo	application/vnd.kde.kformula
+.flw	application/vnd.kde.kivio
+.kon	application/vnd.kde.kontour
+.kpr	application/vnd.kde.kpresenter
+.ksp	application/vnd.kde.kspread
+.kwd	application/vnd.kde.kword
+.htke	application/vnd.kenameaapp
+.kia	application/vnd.kidspiration
+.kne	application/vnd.kinar
+.skp	application/vnd.koan
+.sse	application/vnd.kodak-descriptor
+.lasxml	application/vnd.las.las+xml
+.lbd	application/vnd.llamagraphics.life-balance.desktop
+.lbe	application/vnd.llamagraphics.life-balance.exchange+xml
+.123	application/vnd.lotus-1-2-3
+.apr	application/vnd.lotus-approach
+.pre	application/vnd.lotus-freelance
+.nsf	application/vnd.lotus-notes
+.org	application/vnd.lotus-organizer
+.scm	application/vnd.lotus-screencam
+.lwp	application/vnd.lotus-wordpro
+.portpkg	application/vnd.macports.portpkg
+.mvt	application/vnd.mapbox-vector-tile
+.mcd	application/vnd.mcd
+.mc1	application/vnd.medcalcdata
+.cdkey	application/vnd.mediastation.cdkey
+.mwf	application/vnd.mfer
+.mfm	application/vnd.mfmp
+.flo	application/vnd.micrografx.flo
+.igx	application/vnd.micrografx.igx
+.mif	application/vnd.mif
+.daf	application/vnd.mobius.daf
+.dis	application/vnd.mobius.dis
+.mbk	application/vnd.mobius.mbk
+.mqy	application/vnd.mobius.mqy
+.msl	application/vnd.mobius.msl
+.plc	application/vnd.mobius.plc
+.txf	application/vnd.mobius.txf
+.mpn	application/vnd.mophun.application
+.mpc	application/vnd.mophun.certificate
+.xul	application/vnd.mozilla.xul+xml
+.cil	application/vnd.ms-artgalry
+.cab	application/vnd.ms-cab-compressed
+.xls	application/vnd.ms-excel
+.xlam	application/vnd.ms-excel.addin.macroenabled.12
+.xlsb	application/vnd.ms-excel.sheet.binary.macroenabled.12
+.xlsm	application/vnd.ms-excel.sheet.macroenabled.12
+.xltm	application/vnd.ms-excel.template.macroenabled.12
+.eot	application/vnd.ms-fontobject
+.chm	application/vnd.ms-htmlhelp
+.ims	application/vnd.ms-ims
+.lrm	application/vnd.ms-lrm
+.thmx	application/vnd.ms-officetheme
+.ppt	application/vnd.ms-powerpoint
+.ppam	application/vnd.ms-powerpoint.addin.macroenabled.12
+.pptm	application/vnd.ms-powerpoint.presentation.macroenabled.12
+.sldm	application/vnd.ms-powerpoint.slide.macroenabled.12
+.ppsm	application/vnd.ms-powerpoint.slideshow.macroenabled.12
+.potm	application/vnd.ms-powerpoint.template.macroenabled.12
+.mpp	application/vnd.ms-project
+.docm	application/vnd.ms-word.document.macroenabled.12
+.dotm	application/vnd.ms-word.template.macroenabled.12
+.wps	application/vnd.ms-works
+.wpl	application/vnd.ms-wpl
+.xps	application/vnd.ms-xpsdocument
+.mseq	application/vnd.mseq
+.mus	application/vnd.musician
+.msty	application/vnd.muvee.style
+.taglet	application/vnd.mynfc
+.bdo	application/vnd.nato.bindingdataobject+xml
+.nlu	application/vnd.neurolanguage.nlu
+.ntf	application/vnd.nitf
+.nnd	application/vnd.noblenet-directory
+.nns	application/vnd.noblenet-sealer
+.nnw	application/vnd.noblenet-web
+.ac	application/vnd.nokia.n-gage.ac+xml
+.ngdat	application/vnd.nokia.n-gage.data
+.rpst	application/vnd.nokia.radio-preset
+.rpss	application/vnd.nokia.radio-presets
+.edm	application/vnd.novadigm.edm
+.edx	application/vnd.novadigm.edx
+.ext	application/vnd.novadigm.ext
+.odc	application/vnd.oasis.opendocument.chart
+.otc	application/vnd.oasis.opendocument.chart-template
+.odf	application/vnd.oasis.opendocument.formula
+.odft	application/vnd.oasis.opendocument.formula-template
+.odg	application/vnd.oasis.opendocument.graphics
+.otg	application/vnd.oasis.opendocument.graphics-template
+.odi	application/vnd.oasis.opendocument.image
+.oti	application/vnd.oasis.opendocument.image-template
+.odp	application/vnd.oasis.opendocument.presentation
+.otp	application/vnd.oasis.opendocument.presentation-template
+.ods	application/vnd.oasis.opendocument.spreadsheet
+.ots	application/vnd.oasis.opendocument.spreadsheet-template
+.odt	application/vnd.oasis.opendocument.text
+.odm	application/vnd.oasis.opendocument.text-master
+.ott	application/vnd.oasis.opendocument.text-template
+.oth	application/vnd.oasis.opendocument.text-web
+.xo	application/vnd.olpc-sugar
+.dd2	application/vnd.oma.dd2+xml
+.obgx	application/vnd.openblox.game+xml
+.osm	application/vnd.openstreetmap.data+xml
+.pptx	application/vnd.openxmlformats-officedocument.presentationml.presentation
+.sldx	application/vnd.openxmlformats-officedocument.presentationml.slide
+.ppsx	application/vnd.openxmlformats-officedocument.presentationml.slideshow
+.potx	application/vnd.openxmlformats-officedocument.presentationml.template
+.xlsx	application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+.xltx	application/vnd.openxmlformats-officedocument.spreadsheetml.template
+.docx	application/vnd.openxmlformats-officedocument.wordprocessingml.document
+.dotx	application/vnd.openxmlformats-officedocument.wordprocessingml.template
+.mgp	application/vnd.osgeo.mapguide.package
+.dp	application/vnd.osgi.dp
+.esa	application/vnd.osgi.subsystem
+.pdb	application/vnd.palm
+.paw	application/vnd.pawaafile
+.str	application/vnd.pg.format
+.ei6	application/vnd.pg.osasli
+.efif	application/vnd.picsel
+.wg	application/vnd.pmi.widget
+.plf	application/vnd.pocketlearn
+.pbd	application/vnd.powerbuilder6
+.systemverify	application/vnd.pp.systemverify+xml
+.box	application/vnd.previewsystems.box
+.mgz	application/vnd.proteus.magazine
+.qps	application/vnd.publishare-delta-tree
+.ptid	application/vnd.pvi.ptid1
+.xhtm	application/vnd.pwg-xhtml-print+xml
+.qxd	application/vnd.quark.quarkxpress
+.rar	application/vnd.rar
+.bed	application/vnd.realvnc.bed
+.mxl	application/vnd.recordare.musicxml
+.musicxml	application/vnd.recordare.musicxml+xml
+.cryptonote	application/vnd.rig.cryptonote
+.link66	application/vnd.route66.link66+xml
+.st	application/vnd.sailingtracker.track
+.see	application/vnd.seemail
+.sema	application/vnd.sema
+.semd	application/vnd.semd
+.semf	application/vnd.semf
+.ifm	application/vnd.shana.informed.formdata
+.itp	application/vnd.shana.informed.formtemplate
+.iif	application/vnd.shana.informed.interchange
+.ipk	application/vnd.shana.informed.package
+.twd	application/vnd.simtech-mindmapper
+.mmf	application/vnd.smaf
+.teacher	application/vnd.smart.teacher
+.fo	application/vnd.software602.filler.form+xml
+.sdkm	application/vnd.solent.sdkm+xml
+.dxp	application/vnd.spotfire.dxp
+.sfs	application/vnd.spotfire.sfs
+.sqlite	application/vnd.sqlite3
+.smzip	application/vnd.stepmania.package
+.sm	application/vnd.stepmania.stepchart
+.wadl	application/vnd.sun.wadl+xml
+.sus	application/vnd.sus-calendar
+.svd	application/vnd.svd
+.xsm	application/vnd.syncml+xml
+.bdm	application/vnd.syncml.dm+wbxml
+.xdm	application/vnd.syncml.dm+xml
+.ddf	application/vnd.syncml.dmddf+xml
+.tao	application/vnd.tao.intent-module-archive
+.pcap	application/vnd.tcpdump.pcap
+.tmo	application/vnd.tmobile-livetv
+.tpt	application/vnd.trid.tpt
+.mxs	application/vnd.triscape.mxs
+.tra	application/vnd.trueapp
+.ufd	application/vnd.ufdl
+.utz	application/vnd.uiq.theme
+.umj	application/vnd.umajin
+.unityweb	application/vnd.unity
+.uoml	application/vnd.uoml+xml
+.vcx	application/vnd.vcx
+.vsd	application/vnd.visio
+.vis	application/vnd.visionary
+.vsf	application/vnd.vsf
+.wbxml	application/vnd.wap.wbxml
+.wmlc	application/vnd.wap.wmlc
+.wmlsc	application/vnd.wap.wmlscriptc
+.wtb	application/vnd.webturbo
+.nbp	application/vnd.wolfram.player
+.wpd	application/vnd.wordperfect
+.wqd	application/vnd.wqd
+.stf	application/vnd.wt.stf
+.xar	application/vnd.xara
+.xfdl	application/vnd.xfdl
+.hvd	application/vnd.yamaha.hv-dic
+.hvs	application/vnd.yamaha.hv-script
+.hvp	application/vnd.yamaha.hv-voice
+.osf	application/vnd.yamaha.openscoreformat
+.osfpvg	application/vnd.yamaha.openscoreformat.osfpvg+xml
+.saf	application/vnd.yamaha.smaf-audio
+.spf	application/vnd.yamaha.smaf-phrase
+.cmp	application/vnd.yellowriver-custom-menu
+.zir	application/vnd.zul
+.zaz	application/vnd.zzazz.deck+xml
+.vxml	application/voicexml+xml
+.wasm	application/wasm
+.wif	application/watcherinfo+xml
+.wgt	application/widget
+.wsdl	application/wsdl+xml
+.wspolicy	application/wspolicy+xml
+.der	application/x-x509-ca-cert
+.xav	application/xcap-att+xml
+.xca	application/xcap-caps+xml
+.xdf	application/xcap-diff+xml
+.xel	application/xcap-el+xml
+.xns	application/xcap-ns+xml
+.xenc	application/xenc+xml
+.xfdf	application/xfdf
+.xhtml	application/xhtml+xml
+.xlf	application/xliff+xml
 .xml	application/xml
+.dtd	application/xml-dtd
+.xop	application/xop+xml
+.xsl	application/xslt+xml
+.mxml	application/xv+xml
+.yang	application/yang
+.yin	application/yin+xml
 .zip	application/zip
+.3gpp	audio/3gpp
+.adts	audio/aac
+.amr	audio/amr
+.au	audio/basic
+.mka	audio/matroska
+.mxmf	audio/mobile-xmf
+.m4a	audio/mp4
 .mp3	audio/mpeg
+.ogg	audio/ogg
+.uva	audio/vnd.dece.audio
+.eol	audio/vnd.digital-winds
+.dra	audio/vnd.dra
+.dts	audio/vnd.dts
+.dtshd	audio/vnd.dts.hd
+.lvp	audio/vnd.lucent.voice
+.pya	audio/vnd.ms-playready.media.pya
+.ecelp4800	audio/vnd.nuera.ecelp4800
+.ecelp7470	audio/vnd.nuera.ecelp7470
+.ecelp9600	audio/vnd.nuera.ecelp9600
+.rip	audio/vnd.rip
+.ttc	font/collection
+.otf	font/otf
+.ttf	font/ttf
+.woff	font/woff
+.woff2	font/woff2
+.exr	image/aces
+.apng	image/apng
+.avci	image/avci
+.avcs	image/avcs
+.avif	image/avif
+.bmp	image/bmp
+.cgm	image/cgm
+.drle	image/dicom-rle
+.dpx	image/dpx
+.emf	image/emf
+.fits	image/fits
+.g3	image/g3fax
 .gif	image/gif
+.heic	image/heic
+.heics	image/heic-sequence
+.heif	image/heif
+.heifs	image/heif-sequence
+.hej2	image/hej2k
+.ief	image/ief
+.jaii	image/jaii
+.jais	image/jais
+.jls	image/jls
+.jp2	image/jp2
 .jpg	image/jpeg
+.jph	image/jph
+.jhc	image/jphc
+.jpm	image/jpm
+.jpx	image/jpx
+.jxl	image/jxl
+.jxr	image/jxr
+.jxra	image/jxra
+.jxrs	image/jxrs
+.jxs	image/jxs
+.jxsc	image/jxsc
+.jxsi	image/jxsi
+.jxss	image/jxss
+.ktx	image/ktx
+.ktx2	image/ktx2
 .png	image/png
+.btif	image/prs.btif
+.pti	image/prs.pti
 .svg	image/svg+xml
+.t38	image/t38
+.tif	image/tiff
+.tfx	image/tiff-fx
+.psd	image/vnd.adobe.photoshop
+.azv	image/vnd.airzip.accelerator.azv
+.facti	image/vnd.blockfact.facti
+.uvi	image/vnd.dece.graphic
+.djvu	image/vnd.djvu
+.sub	image/vnd.dvb.subtitle
+.dwg	image/vnd.dwg
+.dxf	image/vnd.dxf
+.fbs	image/vnd.fastbidsheet
+.fpx	image/vnd.fpx
+.fst	image/vnd.fst
+.mmr	image/vnd.fujixerox.edmics-mmr
+.rlc	image/vnd.fujixerox.edmics-rlc
+.ico	image/vnd.microsoft.icon
+.mdi	image/vnd.ms-modi
+.npx	image/vnd.net-fpx
+.b16	image/vnd.pco.b16
+.tap	image/vnd.tencent.tap
+.vtf	image/vnd.valve.source.texture
+.wbmp	image/vnd.wap.wbmp
+.xif	image/vnd.xiff
+.pcx	image/vnd.zbrush.pcx
 .webp	image/webp
+.wmf	image/wmf
+.disposition-notification	message/disposition-notification
+.u8msg	message/global
+.u8dsn	message/global-delivery-status
+.u8mdn	message/global-disposition-notification
+.u8hdr	message/global-headers
+.eml	message/rfc822
+.wsc	message/vnd.wfa.wsc
+.3mf	model/3mf
+.gltf	model/gltf+json
+.glb	model/gltf-binary
+.igs	model/iges
+.jt	model/jt
+.msh	model/mesh
+.mtl	model/mtl
+.obj	model/obj
+.prc	model/prc
+.step	model/step
+.stpx	model/step+xml
+.stpz	model/step+zip
+.stpxz	model/step-xml+zip
+.stl	model/stl
+.u3d	model/u3d
+.bary	model/vnd.bary
+.cld	model/vnd.cld
+.dae	model/vnd.collada+xml
+.dwf	model/vnd.dwf
+.gdl	model/vnd.gdl
+.gtw	model/vnd.gtw
+.mts	model/vnd.mts
+.ogex	model/vnd.opengex
+.x_b	model/vnd.parasolid.transmit.binary
+.x_t	model/vnd.parasolid.transmit.text
+.pyo	model/vnd.pytha.pyox
+.vds	model/vnd.sap.vds
+.usda	model/vnd.usda
+.usdz	model/vnd.usdz+zip
+.bsp	model/vnd.valve.source.compiled-map
+.vtu	model/vnd.vtu
+.wrl	model/vrml
+.x3db	model/x3d+fastinfoset
+.x3d	model/x3d+xml
+.x3dv	model/x3d-vrml
+.appcache	text/cache-manifest
+.ics	text/calendar
+.css	text/css
 .csv	text/csv
 .html	text/html
+.js	text/javascript
+.md	text/markdown
+.n3	text/n3
 .txt	text/plain
+.dsc	text/prs.lines.tag
+.rtx	text/richtext
+.rtf	text/rtf
+.sgml	text/sgml
+.shex	text/shex
+.spdx	text/spdx
+.tsv	text/tab-separated-values
+.t	text/troff
+.ttl	text/turtle
+.uri	text/uri-list
+.vcard	text/vcard
+.curl	text/vnd.curl
+.sub	text/vnd.dvb.subtitle
+.ged	text/vnd.familysearch.gedcom
+.fly	text/vnd.fly
+.flx	text/vnd.fmi.flexstor
+.gv	text/vnd.graphviz
+.3dml	text/vnd.in3d.3dml
+.spot	text/vnd.in3d.spot
+.jad	text/vnd.sun.j2me.app-descriptor
+.wml	text/vnd.wap.wml
+.wmls	text/vnd.wap.wmlscript
+.vtt	text/vtt
+.wgsl	text/wgsl
+.xml	text/xml
+.3gp	video/3gpp
+.3g2	video/3gpp2
+.h261	video/h261
+.h263	video/h263
+.h264	video/h264
+.m4s	video/iso.segment
+.jpgv	video/jpeg
+.mkv	video/matroska
+.mk3d	video/matroska-3d
+.mj2	video/mj2
+.ts	video/mp2t
 .mp4	video/mp4
+.mpeg	video/mpeg
+.ogv	video/ogg
+.mov	video/quicktime
+.uvh	video/vnd.dece.hd
+.uvm	video/vnd.dece.mobile
+.uvp	video/vnd.dece.pd
+.uvs	video/vnd.dece.sd
+.uvv	video/vnd.dece.video
+.dvb	video/vnd.dvb.file
+.fvt	video/vnd.fvt
+.mxu	video/vnd.mpegurl
+.pyv	video/vnd.ms-playready.media.pyv
+.uvu	video/vnd.uvvu.mp4
+.viv	video/vnd.vivo

data/misc/port.json

@@ -0,0 +1 @@
+{ "format": "{number}", "rows": "port.tsv", "key": "number" }

data/misc/protocol.json

@@ -0,0 +1 @@
+{ "format": "{keyword}", "rows": "protocol.tsv", "key": "keyword", "name": "name" }

data/misc/protocol.tsv

@@ -0,0 +1,140 @@
+keyword	name	number
+HOPOPT	IPv6 Hop-by-Hop Option	0
+ICMP	Internet Control Message	1
+IGMP	Internet Group Management	2
+GGP	Gateway-to-Gateway	3
+IPv4	IPv4 encapsulation	4
+ST	Stream	5
+TCP	Transmission Control	6
+CBT	CBT	7
+EGP	Exterior Gateway Protocol	8
+IGP	any private interior gateway (used by Cisco for their IGRP)	9
+BBN-RCC-MON	BBN RCC Monitoring	10
+NVP-II	Network Voice Protocol	11
+PUP	PUP	12
+EMCON	EMCON	14
+XNET	Cross Net Debugger	15
+CHAOS	Chaos	16
+UDP	User Datagram	17
+MUX	Multiplexing	18
+DCN-MEAS	DCN Measurement Subsystems	19
+HMP	Host Monitoring	20
+PRM	Packet Radio Measurement	21
+XNS-IDP	XEROX NS IDP	22
+TRUNK-1	Trunk-1	23
+TRUNK-2	Trunk-2	24
+LEAF-1	Leaf-1	25
+LEAF-2	Leaf-2	26
+RDP	Reliable Data Protocol	27
+IRTP	Internet Reliable Transaction	28
+ISO-TP4	ISO Transport Protocol Class 4	29
+NETBLT	Bulk Data Transfer Protocol	30
+MFE-NSP	MFE Network Services Protocol	31
+MERIT-INP	MERIT Internodal Protocol	32
+DCCP	Datagram Congestion Control Protocol	33
+3PC	Third Party Connect Protocol	34
+IDPR	Inter-Domain Policy Routing Protocol	35
+XTP	XTP	36
+DDP	Datagram Delivery Protocol	37
+IDPR-CMTP	IDPR Control Message Transport Proto	38
+TP++	TP++ Transport Protocol	39
+IL	IL Transport Protocol	40
+IPv6	IPv6 encapsulation	41
+SDRP	Source Demand Routing Protocol	42
+IPv6-Route	Routing Header for IPv6	43
+IPv6-Frag	Fragment Header for IPv6	44
+IDRP	Inter-Domain Routing Protocol	45
+RSVP	Reservation Protocol	46
+GRE	Generic Routing Encapsulation	47
+DSR	Dynamic Source Routing Protocol	48
+BNA	BNA	49
+ESP	Encap Security Payload	50
+AH	Authentication Header	51
+I-NLSP	Integrated Net Layer Security TUBA	52
+NARP	NBMA Address Resolution Protocol	54
+Min-IPv4	Minimal IPv4 Encapsulation	55
+TLSP	Transport Layer Security Protocol using Kryptonet key management	56
+SKIP	SKIP	57
+IPv6-ICMP	ICMP for IPv6	58
+IPv6-NoNxt	No Next Header for IPv6	59
+IPv6-Opts	Destination Options for IPv6	60
+CFTP	CFTP	62
+SAT-EXPAK	SATNET and Backroom EXPAK	64
+KRYPTOLAN	Kryptolan	65
+RVD	MIT Remote Virtual Disk Protocol	66
+IPPC	Internet Pluribus Packet Core	67
+SAT-MON	SATNET Monitoring	69
+VISA	VISA Protocol	70
+IPCV	Internet Packet Core Utility	71
+CPNX	Computer Protocol Network Executive	72
+CPHB	Computer Protocol Heart Beat	73
+WSN	Wang Span Network	74
+PVP	Packet Video Protocol	75
+BR-SAT-MON	Backroom SATNET Monitoring	76
+SUN-ND	SUN ND PROTOCOL-Temporary	77
+WB-MON	WIDEBAND Monitoring	78
+WB-EXPAK	WIDEBAND EXPAK	79
+ISO-IP	ISO Internet Protocol	80
+VMTP	VMTP	81
+SECURE-VMTP	SECURE-VMTP	82
+VINES	VINES	83
+IPTM	Internet Protocol Traffic Manager	84
+NSFNET-IGP	NSFNET-IGP	85
+DGP	Dissimilar Gateway Protocol	86
+TCF	TCF	87
+EIGRP	EIGRP	88
+OSPFIGP	OSPFIGP	89
+Sprite-RPC	Sprite RPC Protocol	90
+LARP	Locus Address Resolution Protocol	91
+MTP	Multicast Transport Protocol	92
+AX.25	AX.25 Frames	93
+IPIP	IP-within-IP Encapsulation Protocol	94
+SCC-SP	Semaphore Communications Sec. Pro.	96
+ETHERIP	Ethernet-within-IP Encapsulation	97
+ENCAP	Encapsulation Header	98
+GMTP	GMTP	100
+IFMP	Ipsilon Flow Management Protocol	101
+PNNI	PNNI over IP	102
+PIM	Protocol Independent Multicast	103
+ARIS	ARIS	104
+SCPS	SCPS	105
+QNX	QNX	106
+A/N	Active Networks	107
+IPComp	IP Payload Compression Protocol	108
+SNP	Sitara Networks Protocol	109
+Compaq-Peer	Compaq Peer Protocol	110
+IPX-in-IP	IPX in IP	111
+VRRP	Virtual Router Redundancy Protocol	112
+PGM	PGM Reliable Transport Protocol	113
+L2TP	Layer Two Tunneling Protocol	115
+DDX	D-II Data Exchange (DDX)	116
+IATP	Interactive Agent Transfer Protocol	117
+STP	Schedule Transfer Protocol	118
+SRP	SpectraLink Radio Protocol	119
+UTI	UTI	120
+SMP	Simple Message Protocol	121
+PTP	Performance Transparency Protocol	123
+ISIS over IPv4	ISIS over IPv4	124
+FIRE	FIRE	125
+CRTP	Combat Radio Transport Protocol	126
+CRUDP	Combat Radio User Datagram	127
+SSCOPMCE	SSCOPMCE	128
+IPLT	IPLT	129
+SPS	Secure Packet Shield	130
+PIPE	Private IP Encapsulation within IP	131
+SCTP	Stream Control Transmission Protocol	132
+FC	Fibre Channel	133
+RSVP-E2E-IGNORE	RSVP-E2E-IGNORE	134
+Mobility Header	Mobility Header	135
+UDPLite	UDPLite	136
+MPLS-in-IP	MPLS-in-IP	137
+manet	MANET Protocols	138
+HIP	Host Identity Protocol	139
+Shim6	Shim6 Protocol	140
+WESP	Wrapped Encapsulating Security Payload	141
+ROHC	Robust Header Compression	142
+Ethernet	Ethernet	143
+AGGFRAG	AGGFRAG encapsulation payload for ESP	144
+NSH	Network Service Header	145
+Homa	Homa	146
+BIT-EMU	Bit-stream Emulation	147

data/misc/territory.json

@@ -0,0 +1 @@
+{ "format": "{name}", "rows": "territory.tsv", "key": "alpha2", "name": "name" }

data/misc/territory.tsv

@@ -0,0 +1,242 @@
+alpha2	alpha3	calling-code	capital	country	currency	flag	languages	name	numeric	tld
+AD	AND	376	Andorra la Vella	AD	EUR	🇦🇩	ca	Andorra	020	.ad
+AE	ARE	971	Abu Dhabi	AE	AED	🇦🇪	ar-AE,fa,en,hi,ur	United Arab Emirates	784	.ae
+AF	AFG	93	Kabul	AF	AFN	🇦🇫	fa-AF,ps,uz-AF,tk	Afghanistan	004	.af
+AG	ATG	1-268	St. John's	AG	XCD	🇦🇬	en-AG	Antigua & Barbuda	028	.ag
+AI	AIA	1-264	The Valley	GB	XCD	🇦🇮	en-AI	Anguilla	660	.ai
+AL	ALB	355	Tirana	AL	ALL	🇦🇱	sq,el	Albania	008	.al
+AM	ARM	374	Yerevan	AM	AMD	🇦🇲	hy	Armenia	051	.am
+AO	AGO	244	Luanda	AO	AOA	🇦🇴	pt-AO	Angola	024	.ao
+AR	ARG	54	Buenos Aires	AR	ARS	🇦🇷	es-AR,en,it,de,fr,gn	Argentina	032	.ar
+AS	ASM	1-684	Pago Pago	US	USD	🇦🇸	en-AS,sm,to	American Samoa	016	.as
+AT	AUT	43	Vienna	AT	EUR	🇦🇹	de-AT,hr,hu,sl	Austria	040	.at
+AU	AUS	61	Canberra	AU	AUD	🇦🇺	en-AU	Australia	036	.au
+AW	ABW	297	Oranjestad	NL	AWG	🇦🇼	nl-AW,pap,es,en	Aruba	533	.aw
+AX	ALA	358	Mariehamn	FI	EUR	🇦🇽	sv-AX	Åland Islands	248	.ax
+AZ	AZE	994	Baku	AZ	AZN	🇦🇿	az,ru,hy	Azerbaijan	031	.az
+BA	BIH	387	Sarajevo	BA	BAM	🇧🇦	bs,hr-BA,sr-BA	Bosnia	070	.ba
+BB	BRB	1-246	Bridgetown	BB	BBD	🇧🇧	en-BB	Barbados	052	.bb
+BD	BGD	880	Dhaka	BD	BDT	🇧🇩	bn-BD,en	Bangladesh	050	.bd
+BE	BEL	32	Brussels	BE	EUR	🇧🇪	nl-BE,fr-BE,de-BE	Belgium	056	.be
+BF	BFA	226	Ouagadougou	BF	XOF	🇧🇫	fr-BF,mos	Burkina Faso	854	.bf
+BG	BGR	359	Sofia	BG	EUR	🇧🇬	bg,tr-BG,rom	Bulgaria	100	.bg
+BH	BHR	973	Manama	BH	BHD	🇧🇭	ar-BH,en,fa,ur	Bahrain	048	.bh
+BI	BDI	257	Gitega	BI	BIF	🇧🇮	fr-BI,rn	Burundi	108	.bi
+BJ	BEN	229	Porto-Novo	BJ	XOF	🇧🇯	fr-BJ	Benin	204	.bj
+BL	BLM	590	Gustavia	FR	EUR	🇧🇱	fr	St. Barthélemy	652	.gp
+BM	BMU	1-441	Hamilton	GB	BMD	🇧🇲	en-BM,pt	Bermuda	060	.bm
+BN	BRN	673	Bandar Seri Begawan	BN	BND	🇧🇳	ms-BN,en-BN	Brunei	096	.bn
+BO	BOL	591	Sucre	BO	BOB	🇧🇴	es-BO,qu,ay	Bolivia	068	.bo
+BR	BRA	55	Brasilia	BR	BRL	🇧🇷	pt-BR,es,en,fr	Brazil	076	.br
+BS	BHS	1-242	Nassau	BS	BSD	🇧🇸	en-BS	Bahamas	044	.bs
+BT	BTN	975	Thimphu	BT	INR	🇧🇹	dz	Bhutan	064	.bt
+BW	BWA	267	Gaborone	BW	BWP	🇧🇼	en-BW,tn-BW	Botswana	072	.bw
+BY	BLR	375	Minsk	BY	BYN	🇧🇾	be,ru	Belarus	112	.by
+BZ	BLZ	501	Belmopan	BZ	BZD	🇧🇿	en-BZ,es	Belize	084	.bz
+CA	CAN	1	Ottawa	CA	CAD	🇨🇦	en-CA,fr-CA,iu	Canada	124	.ca
+CC	CCK	61	West Island	AU	AUD	🇨🇨	ms-CC,en	Cocos Islands	166	.cc
+CD	COD	243	Kinshasa	CD	CDF	🇨🇩	fr-CD,ln,ktu,kg,sw,lua	Congo - Kinshasa	180	.cd
+CF	CAF	236	Bangui	CF	XAF	🇨🇫	fr-CF,sg,ln,kg	Central African Republic	140	.cf
+CG	COG	242	Brazzaville	CG	XAF	🇨🇬	fr-CG,kg,ln-CG	Congo - Brazzaville	178	.cg
+CH	CHE	41	Bern	CH	CHF	🇨🇭	de-CH,fr-CH,it-CH,rm	Switzerland	756	.ch
+CI	CIV	225	Yamoussoukro	CI	XOF	🇨🇮	fr-CI	Côte d’Ivoire	384	.ci
+CK	COK	682	Avarua	NZ	NZD	🇨🇰	en-CK,mi	Cook Islands	184	.ck
+CL	CHL	56	Santiago	CL	CLP	🇨🇱	es-CL	Chile	152	.cl
+CM	CMR	237	Yaounde	CM	XAF	🇨🇲	en-CM,fr-CM	Cameroon	120	.cm
+CN	CHN	86	Beijing	CN	CNY	🇨🇳	zh-CN,yue,wuu,dta,ug,za	China	156	.cn
+CO	COL	57	Bogota	CO	COP	🇨🇴	es-CO	Colombia	170	.co
+CR	CRI	506	San Jose	CR	CRC	🇨🇷	es-CR,en	Costa Rica	188	.cr
+CU	CUB	53	Havana	CU	CUP	🇨🇺	es-CU,pap	Cuba	192	.cu
+CV	CPV	238	Praia	CV	CVE	🇨🇻	pt-CV	Cape Verde	132	.cv
+CW	CUW	599	Willemstad	NL	XCG	🇨🇼	nl,pap	Curaçao	531	.cw
+CX	CXR	61	Flying Fish Cove	AU	AUD	🇨🇽	en,zh,ms-CX	Christmas Island	162	.cx
+CY	CYP	357	Nicosia	CY	EUR	🇨🇾	el-CY,tr-CY,en	Cyprus	196	.cy
+CZ	CZE	420	Prague	CZ	CZK	🇨🇿	cs,sk	Czechia	203	.cz
+DE	DEU	49	Berlin	DE	EUR	🇩🇪	de	Germany	276	.de
+DJ	DJI	253	Djibouti	DJ	DJF	🇩🇯	fr-DJ,ar,so-DJ,aa	Djibouti	262	.dj
+DK	DNK	45	Copenhagen	DK	DKK	🇩🇰	da-DK,en,fo,de-DK	Denmark	208	.dk
+DM	DMA	1-767	Roseau	DM	XCD	🇩🇲	en-DM	Dominica	212	.dm
+DO	DOM	1-809	Santo Domingo	DO	DOP	🇩🇴	es-DO	Dominican Republic	214	.do
+DZ	DZA	213	Algiers	DZ	DZD	🇩🇿	ar-DZ	Algeria	012	.dz
+EC	ECU	593	Quito	EC	USD	🇪🇨	es-EC	Ecuador	218	.ec
+EE	EST	372	Tallinn	EE	EUR	🇪🇪	et,ru	Estonia	233	.ee
+EG	EGY	20	Cairo	EG	EGP	🇪🇬	ar-EG,en,fr	Egypt	818	.eg
+EH	ESH	212	El-Aaiun	EH	MAD	🇪🇭	ar,mey	Western Sahara	732	.eh
+ER	ERI	291	Asmara	ER	ERN	🇪🇷	aa-ER,ar,tig,kun,ti-ER	Eritrea	232	.er
+ES	ESP	34	Madrid	ES	EUR	🇪🇸	es-ES,ca,gl,eu,oc	Spain	724	.es
+ET	ETH	251	Addis Ababa	ET	ETB	🇪🇹	am,en-ET,om-ET,ti-ET,so-ET,sid	Ethiopia	231	.et
+FI	FIN	358	Helsinki	FI	EUR	🇫🇮	fi-FI,sv-FI,smn	Finland	246	.fi
+FJ	FJI	679	Suva	FJ	FJD	🇫🇯	en-FJ,fj	Fiji	242	.fj
+FK	FLK	500	Stanley	GB	FKP	🇫🇰	en-FK	Falkland Islands	238	.fk
+FM	FSM	691	Palikir	FM	USD	🇫🇲	en-FM,chk,pon,yap,kos,uli,woe,nkr,kpg	Micronesia	583	.fm
+FO	FRO	298	Torshavn	DK	DKK	🇫🇴	fo,da-FO	Faroe Islands	234	.fo
+FR	FRA	33	Paris	FR	EUR	🇫🇷	fr-FR,frp,br,co,ca,eu,oc	France	250	.fr
+GA	GAB	241	Libreville	GA	XAF	🇬🇦	fr-GA	Gabon	266	.ga
+GB	GBR	44	London	GB	GBP	🇬🇧	en-GB,cy-GB,gd	UK	826	.uk
+GD	GRD	1-473	St. George's	GD	XCD	🇬🇩	en-GD	Grenada	308	.gd
+GE	GEO	995	Tbilisi	GE	GEL	🇬🇪	ka,ru,hy,az	Georgia	268	.ge
+GF	GUF	594	Cayenne	FR	EUR	🇬🇫	fr-GF	French Guiana	254	.gf
+GG	GGY	44	St Peter Port	GB	GBP	🇬🇬	en,nrf	Guernsey	831	.gg
+GH	GHA	233	Accra	GH	GHS	🇬🇭	en-GH,ak,ee,tw	Ghana	288	.gh
+GI	GIB	350	Gibraltar	GB	GIP	🇬🇮	en-GI,es,it,pt	Gibraltar	292	.gi
+GL	GRL	299	Nuuk	DK	DKK	🇬🇱	kl,da-GL,en	Greenland	304	.gl
+GM	GMB	220	Banjul	GM	GMD	🇬🇲	en-GM,mnk,wof,wo,ff	Gambia	270	.gm
+GN	GIN	224	Conakry	GN	GNF	🇬🇳	fr-GN	Guinea	324	.gn
+GP	GLP	590	Basse-Terre	FR	EUR	🇬🇵	fr-GP	Guadeloupe	312	.gp
+GQ	GNQ	240	Ciudad de la Paz	GQ	XAF	🇬🇶	es-GQ,fr,pt	Equatorial Guinea	226	.gq
+GR	GRC	30	Athens	GR	EUR	🇬🇷	el-GR,en,fr	Greece	300	.gr
+GT	GTM	502	Guatemala City	GT	GTQ	🇬🇹	es-GT	Guatemala	320	.gt
+GU	GUM	1-671	Hagatna	US	USD	🇬🇺	en-GU,ch-GU	Guam	316	.gu
+GW	GNB	245	Bissau	GW	XOF	🇬🇼	pt-GW,pov	Guinea-Bissau	624	.gw
+GY	GUY	592	Georgetown	GY	GYD	🇬🇾	en-GY	Guyana	328	.gy
+HK	HKG	852	Hong Kong	CN	HKD	🇭🇰	zh-HK,yue,zh,en	Hong Kong	344	.hk
+HN	HND	504	Tegucigalpa	HN	HNL	🇭🇳	es-HN,cab,miq	Honduras	340	.hn
+HR	HRV	385	Zagreb	HR	EUR	🇭🇷	hr-HR,sr	Croatia	191	.hr
+HT	HTI	509	Port-au-Prince	HT	HTG	🇭🇹	ht,fr-HT	Haiti	332	.ht
+HU	HUN	36	Budapest	HU	HUF	🇭🇺	hu-HU	Hungary	348	.hu
+ID	IDN	62	Jakarta	ID	IDR	🇮🇩	id,en,nl,jv	Indonesia	360	.id
+IE	IRL	353	Dublin	IE	EUR	🇮🇪	en-IE,ga-IE	Ireland	372	.ie
+IL	ISR	972	Jerusalem	IL	ILS	🇮🇱	he,ar-IL,en-IL	Israel	376	.il
+IM	IMN	44	Douglas	GB	GBP	🇮🇲	en,gv	Isle of Man	833	.im
+IN	IND	91	New Delhi	IN	INR	🇮🇳	en-IN,hi,bn,te,mr,ta,ur,gu,kn,ml,or,pa,as,bh,sat,ks,ne,sd,kok,doi,mni,sit,sa,fr,lus,inc	India	356	.in
+IO	IOT	246	Diego Garcia	GB	USD	🇮🇴	en-IO	British Indian Ocean Territory	086	.io
+IQ	IRQ	964	Baghdad	IQ	IQD	🇮🇶	ar-IQ,ku,hy	Iraq	368	.iq
+IR	IRN	98	Tehran	IR	IRR	🇮🇷	fa-IR,ku	Iran	364	.ir
+IS	ISL	354	Reykjavik	IS	ISK	🇮🇸	is,en,de,da,sv,no	Iceland	352	.is
+IT	ITA	39	Rome	IT	EUR	🇮🇹	it-IT,de-IT,fr-IT,sc,ca,co,sl	Italy	380	.it
+JE	JEY	44	Saint Helier	GB	GBP	🇯🇪	en,fr,nrf	Jersey	832	.je
+JM	JAM	1-876	Kingston	JM	JMD	🇯🇲	en-JM	Jamaica	388	.jm
+JO	JOR	962	Amman	JO	JOD	🇯🇴	ar-JO,en	Jordan	400	.jo
+JP	JPN	81	Tokyo	JP	JPY	🇯🇵	ja	Japan	392	.jp
+KE	KEN	254	Nairobi	KE	KES	🇰🇪	en-KE,sw-KE	Kenya	404	.ke
+KG	KGZ	996	Bishkek	KG	KGS	🇰🇬	ky,uz,ru	Kyrgyzstan	417	.kg
+KH	KHM	855	Phnom Penh	KH	KHR	🇰🇭	km,fr,en	Cambodia	116	.kh
+KI	KIR	686	Tarawa	KI	AUD	🇰🇮	en-KI,gil	Kiribati	296	.ki
+KM	COM	269	Moroni	KM	KMF	🇰🇲	ar,fr-KM	Comoros	174	.km
+KN	KNA	1-869	Basseterre	KN	XCD	🇰🇳	en-KN	St. Kitts & Nevis	659	.kn
+KP	PRK	850	Pyongyang	KP	KPW	🇰🇵	ko-KP	North Korea	408	.kp
+KR	KOR	82	Seoul	KR	KRW	🇰🇷	ko-KR,en	South Korea	410	.kr
+KW	KWT	965	Kuwait City	KW	KWD	🇰🇼	ar-KW,en	Kuwait	414	.kw
+KY	CYM	1-345	George Town	GB	KYD	🇰🇾	en-KY	Cayman Islands	136	.ky
+KZ	KAZ	7	Nur-Sultan	KZ	KZT	🇰🇿	kk,ru	Kazakhstan	398	.kz
+LA	LAO	856	Vientiane	LA	LAK	🇱🇦	lo,fr,en	Laos	418	.la
+LB	LBN	961	Beirut	LB	LBP	🇱🇧	ar-LB,fr-LB,en,hy	Lebanon	422	.lb
+LC	LCA	1-758	Castries	LC	XCD	🇱🇨	en-LC	St. Lucia	662	.lc
+LI	LIE	423	Vaduz	LI	CHF	🇱🇮	de-LI	Liechtenstein	438	.li
+LK	LKA	94	Colombo	LK	LKR	🇱🇰	si,ta,en	Sri Lanka	144	.lk
+LR	LBR	231	Monrovia	LR	LRD	🇱🇷	en-LR	Liberia	430	.lr
+LS	LSO	266	Maseru	LS	LSL	🇱🇸	en-LS,st,zu,xh	Lesotho	426	.ls
+LT	LTU	370	Vilnius	LT	EUR	🇱🇹	lt,ru,pl	Lithuania	440	.lt
+LU	LUX	352	Luxembourg	LU	EUR	🇱🇺	lb,de-LU,fr-LU	Luxembourg	442	.lu
+LV	LVA	371	Riga	LV	EUR	🇱🇻	lv,ru,lt	Latvia	428	.lv
+LY	LBY	218	Tripoli	LY	LYD	🇱🇾	ar-LY,it,en	Libya	434	.ly
+MA	MAR	212	Rabat	MA	MAD	🇲🇦	ar-MA,ber,fr	Morocco	504	.ma
+MC	MCO	377	Monaco	MC	EUR	🇲🇨	fr-MC,en,it	Monaco	492	.mc
+MD	MDA	373	Chisinau	MD	MDL	🇲🇩	ro,ru,gag,tr	Moldova	498	.md
+ME	MNE	382	Podgorica	ME	EUR	🇲🇪	sr,hu,bs,sq,hr,rom	Montenegro	499	.me
+MF	MAF	590	Marigot	FR	EUR	🇲🇫	fr	St. Martin	663	.gp
+MG	MDG	261	Antananarivo	MG	MGA	🇲🇬	fr-MG,mg	Madagascar	450	.mg
+MH	MHL	692	Majuro	MH	USD	🇲🇭	mh,en-MH	Marshall Islands	584	.mh
+MK	MKD	389	Skopje	MK	MKD	🇲🇰	mk,sq,tr,rmm,sr	North Macedonia	807	.mk
+ML	MLI	223	Bamako	ML	XOF	🇲🇱	fr-ML,bm	Mali	466	.ml
+MM	MMR	95	Nay Pyi Taw	MM	MMK	🇲🇲	my	Myanmar	104	.mm
+MN	MNG	976	Ulaanbaatar	MN	MNT	🇲🇳	mn,ru	Mongolia	496	.mn
+MO	MAC	853	Macao	CN	MOP	🇲🇴	zh,zh-MO,pt	Macao	446	.mo
+MP	MNP	1-670	Saipan	US	USD	🇲🇵	fil,tl,zh,ch-MP,en-MP	Northern Mariana Islands	580	.mp
+MQ	MTQ	596	Fort-de-France	FR	EUR	🇲🇶	fr-MQ	Martinique	474	.mq
+MR	MRT	222	Nouakchott	MR	MRU	🇲🇷	ar-MR,fuc,snk,fr,mey,wo	Mauritania	478	.mr
+MS	MSR	1-664	Plymouth	GB	XCD	🇲🇸	en-MS	Montserrat	500	.ms
+MT	MLT	356	Valletta	MT	EUR	🇲🇹	mt,en-MT	Malta	470	.mt
+MU	MUS	230	Port Louis	MU	MUR	🇲🇺	en-MU,bho,fr	Mauritius	480	.mu
+MV	MDV	960	Male	MV	MVR	🇲🇻	dv,en	Maldives	462	.mv
+MW	MWI	265	Lilongwe	MW	MWK	🇲🇼	ny,yao,tum,swk	Malawi	454	.mw
+MX	MEX	52	Mexico City	MX	MXN	🇲🇽	es-MX	Mexico	484	.mx
+MY	MYS	60	Kuala Lumpur	MY	MYR	🇲🇾	ms-MY,en,zh,ta,te,ml,pa,th	Malaysia	458	.my
+MZ	MOZ	258	Maputo	MZ	MZN	🇲🇿	pt-MZ,vmw	Mozambique	508	.mz
+NA	NAM	264	Windhoek	NA	NAD	🇳🇦	en-NA,af,de,hz,naq	Namibia	516	.na
+NC	NCL	687	Noumea	FR	XPF	🇳🇨	fr-NC	New Caledonia	540	.nc
+NE	NER	227	Niamey	NE	XOF	🇳🇪	fr-NE,ha,kr,dje	Niger	562	.ne
+NF	NFK	672	Kingston	AU	AUD	🇳🇫	en-NF	Norfolk Island	574	.nf
+NG	NGA	234	Abuja	NG	NGN	🇳🇬	en-NG,ha,yo,ig,ff	Nigeria	566	.ng
+NI	NIC	505	Managua	NI	NIO	🇳🇮	es-NI,en	Nicaragua	558	.ni
+NL	NLD	31	Amsterdam	NL	EUR	🇳🇱	nl-NL,fy-NL	Netherlands	528	.nl
+NO	NOR	47	Oslo	NO	NOK	🇳🇴	no,nb,nn,se,fi	Norway	578	.no
+NP	NPL	977	Kathmandu	NP	NPR	🇳🇵	ne,en	Nepal	524	.np
+NR	NRU	674	Yaren	NR	AUD	🇳🇷	na,en-NR	Nauru	520	.nr
+NU	NIU	683	Alofi	NZ	NZD	🇳🇺	niu,en-NU	Niue	570	.nu
+NZ	NZL	64	Wellington	NZ	NZD	🇳🇿	en-NZ,mi	New Zealand	554	.nz
+OM	OMN	968	Muscat	OM	OMR	🇴🇲	ar-OM,en,bal,ur	Oman	512	.om
+PA	PAN	507	Panama City	PA	PAB	🇵🇦	es-PA,en	Panama	591	.pa
+PE	PER	51	Lima	PE	PEN	🇵🇪	es-PE,qu,ay	Peru	604	.pe
+PF	PYF	689	Papeete	FR	XPF	🇵🇫	fr-PF,ty	French Polynesia	258	.pf
+PG	PNG	675	Port Moresby	PG	PGK	🇵🇬	en-PG,ho,meu,tpi	Papua New Guinea	598	.pg
+PH	PHL	63	Manila	PH	PHP	🇵🇭	tl,en-PH,fil,ceb,ilo,hil,war,pam,bik,bcl,pag,mrw,tsg,mdh,cbk,krj,sgd,msb,akl,ibg,yka,mta,abx	Philippines	608	.ph
+PK	PAK	92	Islamabad	PK	PKR	🇵🇰	ur-PK,en-PK,pa,sd,ps,brh	Pakistan	586	.pk
+PL	POL	48	Warsaw	PL	PLN	🇵🇱	pl	Poland	616	.pl
+PM	SPM	508	Saint-Pierre	FR	EUR	🇵🇲	fr-PM	St. Pierre & Miquelon	666	.pm
+PN	PCN	870	Adamstown	GB	NZD	🇵🇳	en-PN	Pitcairn	612	.pn
+PR	PRI	1	San Juan	US	USD	🇵🇷	en-PR,es-PR	Puerto Rico	630	.pr
+PT	PRT	351	Lisbon	PT	EUR	🇵🇹	pt-PT,mwl	Portugal	620	.pt
+PW	PLW	680	Melekeok	PW	USD	🇵🇼	pau,sov,en-PW,tox,ja,fil,zh	Palau	585	.pw
+PY	PRY	595	Asuncion	PY	PYG	🇵🇾	es-PY,gn	Paraguay	600	.py
+QA	QAT	974	Doha	QA	QAR	🇶🇦	ar-QA,es	Qatar	634	.qa
+RE	REU	262	Saint-Denis	FR	EUR	🇷🇪	fr-RE	Réunion	638	.re
+RO	ROU	40	Bucharest	RO	RON	🇷🇴	ro,hu,rom	Romania	642	.ro
+RS	SRB	381	Belgrade	RS	RSD	🇷🇸	sr,hu,bs,rom	Serbia	688	.rs
+RU	RUS	7	Moscow	RU	RUB	🇷🇺	ru,tt,xal,cau,ady,kv,ce,tyv,cv,udm,tut,mns,bua,myv,mdf,chm,ba,inh,kbd,krc,av,sah,nog	Russia	643	.ru
+RW	RWA	250	Kigali	RW	RWF	🇷🇼	rw,en-RW,fr-RW,sw	Rwanda	646	.rw
+SA	SAU	966	Riyadh	SA	SAR	🇸🇦	ar-SA	Saudi Arabia	682	.sa
+SB	SLB	677	Honiara	SB	SBD	🇸🇧	en-SB,tpi	Solomon Islands	090	.sb
+SC	SYC	248	Victoria	SC	SCR	🇸🇨	en-SC,fr-SC	Seychelles	690	.sc
+SD	SDN	249	Khartoum	SD	SDG	🇸🇩	ar-SD,en,fia	Sudan	729	.sd
+SE	SWE	46	Stockholm	SE	SEK	🇸🇪	sv-SE,se,sma,fi-SE	Sweden	752	.se
+SG	SGP	65	Singapore	SG	SGD	🇸🇬	cmn,en-SG,ms-SG,ta-SG,zh-SG	Singapore	702	.sg
+SH	SHN	290	Jamestown	GB	SHP	🇸🇭	en-SH	St. Helena	654	.sh
+SI	SVN	386	Ljubljana	SI	EUR	🇸🇮	sl,sh	Slovenia	705	.si
+SJ	SJM	47	Longyearbyen	NO	NOK	🇸🇯	no,ru	Svalbard & Jan Mayen	744	.sj
+SK	SVK	421	Bratislava	SK	EUR	🇸🇰	sk,hu	Slovakia	703	.sk
+SL	SLE	232	Freetown	SL	SLE	🇸🇱	en-SL,men,tem	Sierra Leone	694	.sl
+SM	SMR	378	San Marino	SM	EUR	🇸🇲	it-SM	San Marino	674	.sm
+SN	SEN	221	Dakar	SN	XOF	🇸🇳	fr-SN,wo,fuc,mnk	Senegal	686	.sn
+SO	SOM	252	Mogadishu	SO	SOS	🇸🇴	so-SO,ar-SO,it,en-SO	Somalia	706	.so
+SR	SUR	597	Paramaribo	SR	SRD	🇸🇷	nl-SR,en,srn,hns,jv	Suriname	740	.sr
+SS	SSD	211	Juba	SS	SSP	🇸🇸	en	South Sudan	728	.ss
+ST	STP	239	Sao Tome	ST	STN	🇸🇹	pt-ST	São Tomé & Príncipe	678	.st
+SV	SLV	503	San Salvador	SV	SVC	🇸🇻	es-SV	El Salvador	222	.sv
+SX	SXM	1-721	Philipsburg	NL	XCG	🇸🇽	nl,en	Sint Maarten	534	.sx
+SY	SYR	963	Damascus	SY	SYP	🇸🇾	ar-SY,ku,hy,arc,fr,en	Syria	760	.sy
+SZ	SWZ	268	Mbabane	SZ	SZL	🇸🇿	en-SZ,ss-SZ	Eswatini	748	.sz
+TC	TCA	1-649	Cockburn Town	GB	USD	🇹🇨	en-TC	Turks & Caicos Islands	796	.tc
+TD	TCD	235	N'Djamena	TD	XAF	🇹🇩	fr-TD,ar-TD,sre	Chad	148	.td
+TF	ATF	262	Port-aux-Francais	FR	EUR	🇹🇫	fr	French Southern Territories	260	.tf
+TG	TGO	228	Lome	TG	XOF	🇹🇬	fr-TG,ee,hna,kbp,dag,ha	Togo	768	.tg
+TH	THA	66	Bangkok	TH	THB	🇹🇭	th,en	Thailand	764	.th
+TJ	TJK	992	Dushanbe	TJ	TJS	🇹🇯	tg,ru	Tajikistan	762	.tj
+TL	TLS	670	Dili	TL	USD	🇹🇱	tet,pt-TL,id,en	Timor-Leste	626	.tl
+TM	TKM	993	Ashgabat	TM	TMT	🇹🇲	tk,ru,uz	Turkmenistan	795	.tm
+TN	TUN	216	Tunis	TN	TND	🇹🇳	ar-TN,fr	Tunisia	788	.tn
+TO	TON	676	Nuku'alofa	TO	TOP	🇹🇴	to,en-TO	Tonga	776	.to
+TR	TUR	90	Ankara	TR	TRY	🇹🇷	tr-TR,ku,diq,az,av	Türkiye	792	.tr
+TT	TTO	1-868	Port of Spain	TT	TTD	🇹🇹	en-TT,hns,fr,es,zh	Trinidad & Tobago	780	.tt
+TV	TUV	688	Funafuti	TV	AUD	🇹🇻	tvl,en,sm,gil	Tuvalu	798	.tv
+TW	TWN	886	Taipei	TW	TWD	🇹🇼	zh-TW,zh,nan,hak	Taiwan	158	.tw
+TZ	TZA	255	Dodoma	TZ	TZS	🇹🇿	sw-TZ,en,ar	Tanzania	834	.tz
+UA	UKR	380	Kyiv	UA	UAH	🇺🇦	uk,ru-UA,rom,pl,hu	Ukraine	804	.ua
+UG	UGA	256	Kampala	UG	UGX	🇺🇬	en-UG,lg,sw,ar	Uganda	800	.ug
+US	USA	1	Washington	US	USD	🇺🇸	en-US,es-US,haw,fr	US	840	.us
+UY	URY	598	Montevideo	UY	UYU	🇺🇾	es-UY	Uruguay	858	.uy
+UZ	UZB	998	Tashkent	UZ	UZS	🇺🇿	uz,ru,tg	Uzbekistan	860	.uz
+VA	VAT	39	Vatican City	VA	EUR	🇻🇦	la,it,fr	Vatican City	336	.va
+VC	VCT	1-784	Kingstown	VC	XCD	🇻🇨	en-VC,fr	St. Vincent & Grenadines	670	.vc
+VE	VEN	58	Caracas	VE	VES	🇻🇪	es-VE	Venezuela	862	.ve
+VG	VGB	1-284	Road Town	GB	USD	🇻🇬	en-VG	British Virgin Islands	092	.vg
+VI	VIR	1-340	Charlotte Amalie	US	USD	🇻🇮	en-VI	U.S. Virgin Islands	850	.vi
+VN	VNM	84	Hanoi	VN	VND	🇻🇳	vi,en,fr,zh,km	Vietnam	704	.vn
+VU	VUT	678	Port Vila	VU	VUV	🇻🇺	bi,en-VU,fr-VU	Vanuatu	548	.vu
+WF	WLF	681	Mata Utu	FR	XPF	🇼🇫	wls,fud,fr-WF	Wallis & Futuna	876	.wf
+WS	WSM	685	Apia	WS	WST	🇼🇸	sm,en-WS	Samoa	882	.ws
+YE	YEM	967	Sanaa	YE	YER	🇾🇪	ar-YE	Yemen	887	.ye
+YT	MYT	262	Mamoudzou	FR	EUR	🇾🇹	fr-YT	Mayotte	175	.yt
+ZA	ZAF	27	Pretoria	ZA	ZAR	🇿🇦	zu,xh,af,nso,en-ZA,tn,st,ts,ss,ve,nr	South Africa	710	.za
+ZM	ZMB	260	Lusaka	ZM	ZMW	🇿🇲	en-ZM,bem,loz,lun,lue,ny,toi	Zambia	894	.zm
+ZW	ZWE	263	Harare	ZW	ZWG	🇿🇼	en-ZW,sn,nr,nd	Zimbabwe	716	.zw

data/misc/timezone.json

@@ -1 +1 @@
-["UTC", "Africa/Cairo", "Africa/Johannesburg", "America/Chicago", "America/Denver", "America/Los_Angeles", "America/Mexico_City", "America/New_York", "America/Sao_Paulo", "America/Toronto", "Asia/Dubai", "Asia/Hong_Kong", "Asia/Kolkata", "Asia/Shanghai", "Asia/Singapore", "Asia/Tokyo", "Australia/Sydney", "Europe/Amsterdam", "Europe/Berlin", "Europe/London", "Europe/Madrid", "Europe/Moscow", "Europe/Oslo", "Europe/Paris", "Europe/Stockholm", "Europe/Warsaw", "Pacific/Auckland"]
+{ "format": "{zone}", "rows": "timezone.tsv", "key": "zone", "parent": "territory", "weight": "weight" }

data/misc/timezone.tsv

@@ -0,0 +1,402 @@
+offset	territory	weight	zone
++00:00	CI	17496811	Africa/Abidjan
++00:00	GH	11277813	Africa/Accra
++03:00	ET	18690487	Africa/Addis_Ababa
++01:00	DZ	20111539	Africa/Algiers
++03:00	ER	798575	Africa/Asmara
++00:00	ML	7673493	Africa/Bamako
++01:00	CF	2582076	Africa/Bangui
++00:00	GM	1305981	Africa/Banjul
++00:00	GW	904776	Africa/Bissau
++02:00	MW	3030905	Africa/Blantyre
++01:00	CG	3748808	Africa/Brazzaville
++02:00	BI	1251675	Africa/Bujumbura
++02:00	EG	47242427	Africa/Cairo
++00:00	MA	22808531	Africa/Casablanca
++01:00	ES	85985	Africa/Ceuta
++00:00	GN	6679241	Africa/Conakry
++00:00	SN	10055356	Africa/Dakar
++03:00	TZ	21350525	Africa/Dar_es_Salaam
++03:00	DJ	1316108	Africa/Djibouti
++01:00	CM	8330801	Africa/Douala
++00:00	EH	374107	Africa/El_Aaiun
++00:00	SL	1752243	Africa/Freetown
++02:00	BW	1262880	Africa/Gaborone
++02:00	ZW	4508424	Africa/Harare
++02:00	ZA	47939803	Africa/Johannesburg
++02:00	SS	1679674	Africa/Juba
++03:00	UG	9664318	Africa/Kampala
++02:00	SD	11781926	Africa/Khartoum
++02:00	RW	2910190	Africa/Kigali
++01:00	CD	20651243	Africa/Kinshasa
++01:00	NG	75725576	Africa/Lagos
++01:00	GA	1731692	Africa/Libreville
++00:00	TG	3089219	Africa/Lome
++01:00	AO	23567341	Africa/Luanda
++02:00	CD	16660579	Africa/Lubumbashi
++02:00	ZM	8422414	Africa/Lusaka
++01:00	GQ	355840	Africa/Malabo
++02:00	MZ	8598836	Africa/Maputo
++02:00	LS	739682	Africa/Maseru
++02:00	SZ	191312	Africa/Mbabane
++03:00	SO	6165052	Africa/Mogadishu
++00:00	LR	2127307	Africa/Monrovia
++03:00	KE	16920989	Africa/Nairobi
++01:00	TD	3363418	Africa/Ndjamena
++01:00	NE	3689982	Africa/Niamey
++00:00	MR	2157288	Africa/Nouakchott
++00:00	BF	6125427	Africa/Ouagadougou
++01:00	BJ	4001576	Africa/Porto-Novo
++00:00	ST	53300	Africa/Sao_Tome
++02:00	LY	6105792	Africa/Tripoli
++01:00	TN	7314557	Africa/Tunis
++02:00	NA	983097	Africa/Windhoek
+-10:00	US	15000	America/Adak
+-09:00	US	366200	America/Anchorage
+-04:00	AI	15000	America/Anguilla
+-04:00	AG	51737	America/Antigua
+-03:00	BR	858988	America/Araguaina
+-03:00	AR	11817784	America/Argentina/Buenos_Aires
+-03:00	AR	580969	America/Argentina/Catamarca
+-03:00	AR	8998034	America/Argentina/Cordoba
+-03:00	AR	413866	America/Argentina/Jujuy
+-03:00	AR	237745	America/Argentina/La_Rioja
+-03:00	AR	695484	America/Argentina/Mendoza
+-03:00	AR	241619	America/Argentina/Rio_Gallegos
+-03:00	AR	2120781	America/Argentina/Salta
+-03:00	AR	518624	America/Argentina/San_Juan
+-03:00	AR	266728	America/Argentina/San_Luis
+-03:00	AR	788916	America/Argentina/Tucuman
+-03:00	AR	109506	America/Argentina/Ushuaia
+-04:00	AW	90927	America/Aruba
+-03:00	PY	3653883	America/Asuncion
+-05:00	CA	15000	America/Atikokan
+-03:00	BR	13131907	America/Bahia
+-06:00	MX	36253	America/Bahia_Banderas
+-04:00	BB	98511	America/Barbados
+-03:00	BR	7977349	America/Belem
+-06:00	BZ	132559	America/Belize
+-04:00	CA	15000	America/Blanc-Sablon
+-04:00	BR	550434	America/Boa_Vista
+-05:00	CO	37243567	America/Bogota
+-07:00	US	748802	America/Boise
+-07:00	CA	15000	America/Cambridge_Bay
+-04:00	BR	2429053	America/Campo_Grande
+-05:00	MX	1328961	America/Cancun
+-04:00	VE	28324666	America/Caracas
+-03:00	GF	155245	America/Cayenne
+-05:00	KY	44705	America/Cayman
+-06:00	US	55510713	America/Chicago
+-06:00	MX	1533843	America/Chihuahua
+-07:00	MX	1512450	America/Ciudad_Juarez
+-06:00	CR	1458629	America/Costa_Rica
+-03:00	CL	62723	America/Coyhaique
+-07:00	CA	15000	America/Creston
+-04:00	BR	2960376	America/Cuiaba
+-04:00	CW	145838	America/Curacao
++00:00	GL	15000	America/Danmarkshavn
+-07:00	CA	15000	America/Dawson
+-07:00	CA	21465	America/Dawson_Creek
+-07:00	US	9069711	America/Denver
+-05:00	US	4344725	America/Detroit
+-04:00	DM	16571	America/Dominica
+-06:00	CA	3518059	America/Edmonton
+-05:00	BR	142900	America/Eirunepe
+-06:00	SV	2571611	America/El_Salvador
+-07:00	CA	15000	America/Fort_Nelson
+-03:00	BR	21824546	America/Fortaleza
+-04:00	CA	122883	America/Glace_Bay
+-04:00	CA	15000	America/Goose_Bay
+-05:00	TC	27489	America/Grand_Turk
+-04:00	GD	15000	America/Grenada
+-04:00	GP	250260	America/Guadeloupe
+-06:00	GT	7302205	America/Guatemala
+-05:00	EC	11564603	America/Guayaquil
+-04:00	GY	314746	America/Guyana
+-04:00	CA	843888	America/Halifax
+-05:00	CU	11972822	America/Havana
+-07:00	MX	2215048	America/Hermosillo
+-05:00	US	2672398	America/Indiana/Indianapolis
+-06:00	US	15000	America/Indiana/Knox
+-05:00	US	15000	America/Indiana/Marengo
+-05:00	US	15000	America/Indiana/Petersburg
+-06:00	US	15000	America/Indiana/Tell_City
+-05:00	US	15000	America/Indiana/Vevay
+-05:00	US	15451	America/Indiana/Vincennes
+-05:00	US	15000	America/Indiana/Winamac
+-06:00	CA	15000	America/Inuvik
+-05:00	CA	15000	America/Iqaluit
+-05:00	JM	2069735	America/Jamaica
+-09:00	US	31555	America/Juneau
+-05:00	US	909379	America/Kentucky/Louisville
+-05:00	US	15000	America/Kentucky/Monticello
+-04:00	BO	6958595	America/La_Paz
+-05:00	PE	21406340	America/Lima
+-08:00	US	46301242	America/Los_Angeles
+-04:00	SX	15000	America/Lower_Princes
+-03:00	BR	4805317	America/Maceio
+-06:00	NI	2621115	America/Managua
+-04:00	BR	3918958	America/Manaus
+-04:00	MF	15000	America/Marigot
+-04:00	MQ	243860	America/Martinique
+-06:00	MX	1899165	America/Matamoros
+-07:00	MX	2951216	America/Mazatlan
+-06:00	US	15000	America/Menominee
+-06:00	MX	4154924	America/Merida
+-09:00	US	15000	America/Metlakatla
+-06:00	MX	61603325	America/Mexico_City
+-03:00	PM	15000	America/Miquelon
+-04:00	CA	381536	America/Moncton
+-06:00	MX	9714766	America/Monterrey
+-03:00	UY	2457462	America/Montevideo
+-04:00	MS	15000	America/Montserrat
+-05:00	BS	319054	America/Nassau
+-05:00	US	88988896	America/New_York
+-09:00	US	15000	America/Nome
+-02:00	BR	15000	America/Noronha
+-06:00	US	15000	America/North_Dakota/Beulah
+-06:00	US	15000	America/North_Dakota/Center
+-06:00	US	21382	America/North_Dakota/New_Salem
+-02:00	GL	15000	America/Nuuk
+-06:00	MX	50784	America/Ojinaga
+-05:00	PA	1692726	America/Panama
+-03:00	SR	454064	America/Paramaribo
+-07:00	US	6541994	America/Phoenix
+-05:00	HT	5328623	America/Port-au-Prince
+-04:00	TT	456832	America/Port_of_Spain
+-04:00	BR	1551818	America/Porto_Velho
+-04:00	PR	1605968	America/Puerto_Rico
+-03:00	CL	135935	America/Punta_Arenas
+-06:00	CA	15000	America/Rankin_Inlet
+-03:00	BR	8832691	America/Recife
+-06:00	CA	982810	America/Regina
+-06:00	CA	15000	America/Resolute
+-05:00	BR	794527	America/Rio_Branco
+-03:00	BR	1403419	America/Santarem
+-04:00	CL	15912038	America/Santiago
+-04:00	DO	7521391	America/Santo_Domingo
+-03:00	BR	129186469	America/Sao_Paulo
+-02:00	GL	15000	America/Scoresbysund
+-09:00	US	15000	America/Sitka
+-04:00	BL	15000	America/St_Barthelemy
+-03:30	CA	202443	America/St_Johns
+-04:00	KN	15000	America/St_Kitts
+-04:00	LC	45210	America/St_Lucia
+-04:00	VI	70601	America/St_Thomas
+-04:00	VC	48723	America/St_Vincent
+-06:00	CA	16604	America/Swift_Current
+-06:00	HN	4808928	America/Tegucigalpa
+-04:00	GL	15000	America/Thule
+-08:00	MX	3797218	America/Tijuana
+-05:00	CA	24763654	America/Toronto
+-04:00	VG	15000	America/Tortola
+-07:00	CA	6340886	America/Vancouver
+-07:00	CA	28201	America/Whitehorse
+-06:00	CA	922566	America/Winnipeg
+-09:00	US	15000	America/Yakutat
++10:00	AU	15000	Antarctica/Macquarie
++01:00	SJ	15000	Arctic/Longyearbyen
++03:00	YE	6904010	Asia/Aden
++05:00	KZ	9055499	Asia/Almaty
++03:00	JO	4950859	Asia/Amman
++12:00	RU	15604	Asia/Anadyr
++05:00	KZ	282409	Asia/Aqtau
++05:00	KZ	598827	Asia/Aqtobe
++05:00	TM	2706037	Asia/Ashgabat
++05:00	KZ	388248	Asia/Atyrau
++03:00	IQ	29681856	Asia/Baghdad
++03:00	BH	815447	Asia/Bahrain
++04:00	AZ	5491302	Asia/Baku
++07:00	TH	49896297	Asia/Bangkok
++07:00	RU	1413356	Asia/Barnaul
++02:00	LB	4467748	Asia/Beirut
++06:00	KG	2233412	Asia/Bishkek
++08:00	BN	230416	Asia/Brunei
++09:00	RU	486311	Asia/Chita
++05:30	LK	4297175	Asia/Colombo
++03:00	SY	10939175	Asia/Damascus
++06:00	BD	35955225	Asia/Dhaka
++09:00	TL	311685	Asia/Dili
++04:00	AE	12931222	Asia/Dubai
++05:00	TJ	2578933	Asia/Dushanbe
++02:00	CY	109959	Asia/Famagusta
++07:00	VN	42129036	Asia/Ho_Chi_Minh
++08:00	HK	24678261	Asia/Hong_Kong
++07:00	MN	122174	Asia/Hovd
++08:00	RU	2231790	Asia/Irkutsk
++07:00	ID	68794489	Asia/Jakarta
++09:00	ID	2512221	Asia/Jayapura
++02:00	IL	7901151	Asia/Jerusalem
++04:30	AF	8764027	Asia/Kabul
++12:00	RU	247112	Asia/Kamchatka
++05:00	PK	88744953	Asia/Karachi
++05:45	NP	5913313	Asia/Kathmandu
++09:00	RU	15000	Asia/Khandyga
++05:30	IN	365373326	Asia/Kolkata
++07:00	RU	2615869	Asia/Krasnoyarsk
++08:00	MY	33322164	Asia/Kuala_Lumpur
++08:00	MY	4013140	Asia/Kuching
++03:00	KW	1662180	Asia/Kuwait
++08:00	MO	1236695	Asia/Macau
++11:00	RU	92782	Asia/Magadan
++08:00	ID	12458359	Asia/Makassar
++08:00	PH	64227523	Asia/Manila
++04:00	OM	3290021	Asia/Muscat
++02:00	CY	679420	Asia/Nicosia
++07:00	RU	2355475	Asia/Novokuznetsk
++07:00	RU	2119272	Asia/Novosibirsk
++06:00	RU	1243165	Asia/Omsk
++05:00	KZ	364799	Asia/Oral
++07:00	KH	4111780	Asia/Phnom_Penh
++07:00	ID	1819584	Asia/Pontianak
++09:00	KP	12876883	Asia/Pyongyang
++03:00	QA	1566088	Asia/Qatar
++05:00	KZ	529104	Asia/Qostanay
++05:00	KZ	511391	Asia/Qyzylorda
++03:00	SA	23913997	Asia/Riyadh
++11:00	RU	326982	Asia/Sakhalin
++05:00	UZ	3864475	Asia/Samarkand
++09:00	KR	47400555	Asia/Seoul
++08:00	CN	861293141	Asia/Shanghai
++08:00	SG	11621053	Asia/Singapore
++11:00	RU	15000	Asia/Srednekolymsk
++08:00	TW	26394699	Asia/Taipei
++05:00	UZ	7452125	Asia/Tashkent
++04:00	GE	1827911	Asia/Tbilisi
++03:30	IR	50648371	Asia/Tehran
++06:00	BT	170210	Asia/Thimphu
++09:00	JP	141330654	Asia/Tokyo
++07:00	RU	783576	Asia/Tomsk
++08:00	MN	1438738	Asia/Ulaanbaatar
++06:00	CN	12667986	Asia/Urumqi
++10:00	RU	15000	Asia/Ust-Nera
++07:00	LA	1561716	Asia/Vientiane
++10:00	RU	2534711	Asia/Vladivostok
++09:00	RU	902983	Asia/Yakutsk
++06:30	MM	18836338	Asia/Yangon
++05:00	RU	15407217	Asia/Yekaterinburg
++04:00	AM	2880620	Asia/Yerevan
+-01:00	PT	20056	Atlantic/Azores
+-04:00	BM	15000	Atlantic/Bermuda
++00:00	ES	2054431	Atlantic/Canary
+-01:00	CV	252678	Atlantic/Cape_Verde
++00:00	FO	15000	Atlantic/Faroe
++00:00	PT	213794	Atlantic/Madeira
++00:00	IS	245356	Atlantic/Reykjavik
++00:00	SH	15000	Atlantic/St_Helena
+-03:00	FK	15000	Atlantic/Stanley
++09:30	AU	1706678	Australia/Adelaide
++10:00	AU	5807246	Australia/Brisbane
++09:30	AU	17456	Australia/Broken_Hill
++09:30	AU	199509	Australia/Darwin
++08:45	AU	15000	Australia/Eucla
++10:00	AU	392450	Australia/Hobart
++10:00	AU	15000	Australia/Lindeman
++10:30	AU	15000	Australia/Lord_Howe
++10:00	AU	9304158	Australia/Melbourne
++08:00	AU	3086441	Australia/Perth
++10:00	AU	9352313	Australia/Sydney
++01:00	NL	13892513	Europe/Amsterdam
++01:00	AD	36283	Europe/Andorra
++04:00	RU	644812	Europe/Astrakhan
++02:00	GR	6286690	Europe/Athens
++01:00	RS	5756549	Europe/Belgrade
++01:00	DE	62740682	Europe/Berlin
++01:00	SK	2548401	Europe/Bratislava
++01:00	BE	9038306	Europe/Brussels
++02:00	RO	11070040	Europe/Bucharest
++01:00	HU	9366333	Europe/Budapest
++01:00	DE	15000	Europe/Busingen
++02:00	MD	1460347	Europe/Chisinau
++01:00	DK	3885672	Europe/Copenhagen
++01:00	IE	2720758	Europe/Dublin
++01:00	GI	26544	Europe/Gibraltar
++00:00	GG	16488	Europe/Guernsey
++02:00	FI	5265184	Europe/Helsinki
++00:00	IM	26218	Europe/Isle_of_Man
++03:00	TR	77554557	Europe/Istanbul
++00:00	JE	28000	Europe/Jersey
++02:00	RU	664545	Europe/Kaliningrad
++03:00	RU	782604	Europe/Kirov
++02:00	UA	35633582	Europe/Kyiv
++00:00	PT	6372176	Europe/Lisbon
++01:00	SI	552127	Europe/Ljubljana
++00:00	GB	57817786	Europe/London
++01:00	LU	131322	Europe/Luxembourg
++01:00	ES	42918247	Europe/Madrid
++01:00	MT	141714	Europe/Malta
++02:00	AX	15000	Europe/Mariehamn
++03:00	BY	6105410	Europe/Minsk
++01:00	MC	48977	Europe/Monaco
++03:00	RU	69342062	Europe/Moscow
++01:00	NO	3296711	Europe/Oslo
++01:00	FR	33093827	Europe/Paris
++01:00	ME	398494	Europe/Podgorica
++01:00	CZ	5924565	Europe/Prague
++02:00	LV	1206422	Europe/Riga
++01:00	IT	34033666	Europe/Rome
++04:00	RU	3546813	Europe/Samara
++01:00	SM	15000	Europe/San_Marino
++01:00	BA	1905798	Europe/Sarajevo
++04:00	RU	1705529	Europe/Saratov
++03:00	UA	1620906	Europe/Simferopol
++01:00	MK	1754303	Europe/Skopje
++02:00	BG	4111057	Europe/Sofia
++01:00	SE	7149541	Europe/Stockholm
++02:00	EE	1106943	Europe/Tallinn
++01:00	AL	1629643	Europe/Tirane
++04:00	RU	814189	Europe/Ulyanovsk
++01:00	LI	15000	Europe/Vaduz
++01:00	VA	15000	Europe/Vatican
++01:00	AT	5274040	Europe/Vienna
++02:00	LT	2103133	Europe/Vilnius
++03:00	RU	1837053	Europe/Volgograd
++01:00	PL	22720110	Europe/Warsaw
++01:00	HR	1650334	Europe/Zagreb
++01:00	CH	3461664	Europe/Zurich
++03:00	MG	6115943	Indian/Antananarivo
++06:00	IO	15000	Indian/Chagos
++07:00	CX	15000	Indian/Christmas
++06:30	CC	15000	Indian/Cocos
++03:00	KM	118353	Indian/Comoro
++05:00	TF	15000	Indian/Kerguelen
++04:00	SC	22881	Indian/Mahe
++05:00	MV	103693	Indian/Maldives
++04:00	MU	701493	Indian/Mauritius
++03:00	YT	138967	Indian/Mayotte
++04:00	RE	804329	Indian/Reunion
++13:00	WS	40407	Pacific/Apia
++12:00	NZ	5203989	Pacific/Auckland
++11:00	PG	40266	Pacific/Bougainville
++12:45	NZ	15000	Pacific/Chatham
++10:00	FM	15000	Pacific/Chuuk
+-06:00	CL	15000	Pacific/Easter
++11:00	VU	35901	Pacific/Efate
++12:00	FJ	334403	Pacific/Fiji
++12:00	TV	15000	Pacific/Funafuti
+-06:00	EC	15000	Pacific/Galapagos
+-09:00	PF	15000	Pacific/Gambier
++11:00	SB	194707	Pacific/Guadalcanal
++10:00	GU	121094	Pacific/Guam
+-10:00	US	1569453	Pacific/Honolulu
++13:00	KI	15000	Pacific/Kanton
++14:00	KI	15000	Pacific/Kiritimati
++11:00	FM	15000	Pacific/Kosrae
++12:00	MH	15000	Pacific/Kwajalein
++12:00	MH	45701	Pacific/Majuro
+-09:30	PF	15000	Pacific/Marquesas
++12:00	NR	15000	Pacific/Nauru
+-11:00	NU	15000	Pacific/Niue
++11:00	NF	15000	Pacific/Norfolk
++11:00	NC	137086	Pacific/Noumea
+-11:00	AS	15000	Pacific/Pago_Pago
++09:00	PW	15000	Pacific/Palau
+-08:00	PN	15000	Pacific/Pitcairn
++11:00	FM	15000	Pacific/Pohnpei
++10:00	PG	588589	Pacific/Port_Moresby
+-10:00	CK	15000	Pacific/Rarotonga
++10:00	MP	48220	Pacific/Saipan
+-10:00	PF	81958	Pacific/Tahiti
++12:00	KI	40311	Pacific/Tarawa
++13:00	TO	22400	Pacific/Tongatapu
++12:00	WF	15000	Pacific/Wallis

data/misc/tld.json

@@ -0,0 +1 @@
+{ "format": "{tld}", "rows": "tld.tsv", "key": "tld", "name": "unicode" }